#!/usr/bin/env python3
"""
铅山县生态环境局 - 行政许可 爬虫
http://www.jxyanshan.gov.cn/ysxsthjj/xzxk/yszwgk_xxgklist.shtml
"""
import requests
from bs4 import BeautifulSoup
import re, os, sys, time
from urllib.parse import urljoin

BASE_URL = "http://www.jxyanshan.gov.cn"
LIST_URL = "http://www.jxyanshan.gov.cn/ysxsthjj/xzxk/yszwgk_xxgklist"
SITE_NAME = "铅山县生态环境局"
MAX_PAGES = 5  # 前5页
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)
def fetch(url, encoding='utf-8'):
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        r.encoding = encoding
        return r.text
    except Exception as e:
        print(f"  [ERROR] fetch failed: {url} - {e}")
        return None

def extract_list_items(html):
    """Extract article URLs from list page."""
    soup = BeautifulSoup(html, 'html.parser')
    items = []
    for a in soup.find_all('a', href=True):
        href = a['href']
        # Detail pages: /ysxsthjj/xzxk/YYYYMM/<uuid>.shtml
        if re.search(r'/ysxsthjj/xzxk/\d{6}/[a-f0-9]+\.shtml', href):
            full_url = urljoin(BASE_URL, href)
            list_title = a.get_text(strip=True)
            items.append((full_url, list_title))
    return items

def extract_detail(html, page_url):
    """Extract title, content, attachments from detail page."""
    soup = BeautifulSoup(html, 'html.parser')
    
    # 1. Full title from zwtit
    title_el = soup.find('div', class_='zwtit')
    title = title_el.get_text(strip=True) if title_el else ""
    # Clean title from <title> tag if empty
    if not title:
        title_tag = soup.find('title')
        if title_tag:
            title = title_tag.get_text(strip=True)
            # Clean: remove site/channel name prefix
            title = re.sub(r'^[^_]+_[^_]+_', '', title)
    
    # 2. Content from zwcon
    content_parts = []
    attachments = []
    
    zwcon = soup.find('div', class_='zwcon')
    if not zwcon:
        zwcon = soup.find('div', class_='zwcon clearfix')
    
    if zwcon:
        # Find all attachments in the page (relative to detail page)
        base_detail_dir = page_url.rsplit('/', 1)[0] if '/' in page_url else ''
        
        for a in zwcon.find_all('a', href=True):
            href = a['href']
            if any(href.endswith(ext) for ext in ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.rar', '.zip']):
                full_url = urljoin(page_url, href)
                attach_name = a.get_text(strip=True) or href.split('/')[-1]
                attachments.append({'name': attach_name, 'url': full_url})
        
        # Get paragraphs, tables, images
        # Title may also appear as first <p> in zwcon - skip exact matches only
        for el in zwcon.find_all(['p', 'table', 'img']):
            if el.name == 'p':
                # Skip <p> that's inside a <table> — table cells rendered
                # as individual <p> are already handled by the table->markdown below
                if el.find_parent('table'):
                    continue
                txt = el.get_text(strip=True)
                if txt:
                    # Only skip if <p> text exactly matches the title
                    if title and txt == title:
                        continue
                    content_parts.append(txt)
            elif el.name == 'table':
                tbl_html = html_table_to_html(el, page_url)
                if tbl_html:
                    cell_texts.append(tbl_html)
            elif el.name == 'img':
                src = el.get('src', '')
                alt = el.get('alt', '')
                if src:
                    full_src = urljoin(page_url, src)
                    content_parts.append(f'![{alt}]({full_src})')
    
    content = '\n\n'.join(content_parts)
    
    # 3. Date: try to find in the page
    date = ""
    date_match = re.search(r'(\d{4})[-年](\d{1,2})[-月](\d{1,2})[日]?', html)
    if date_match:
        date = f"{date_match.group(1)}-{date_match.group(2).zfill(2)}-{date_match.group(3).zfill(2)}"
    
    return title, content, attachments, date

def main():
    items_all = []
    
    # Step 1: Collect all items from pages 1-5
    for page in range(1, MAX_PAGES + 1):
        if page == 1:
            url = f"{LIST_URL}.shtml"
        else:
            url = f"{LIST_URL}_{page}.shtml"
        
        print(f"Fetching list page {page}: {url}")
        html = fetch(url)
        if not html:
            print(f"  [SKIP] Page {page} failed")
            continue
        
        items = extract_list_items(html)
        print(f"  Found {len(items)} items")
        items_all.extend(items)
        time.sleep(0.3)
    
    print(f"\nTotal items to process: {len(items_all)}")
    
    # Step 2: Fetch each detail page
    results = []
    for i, (page_url, list_title) in enumerate(items_all):
        print(f"[{i+1}/{len(items_all)}] {list_title[:50]}...")
        
        # Always fetch detail for full title and content
        html = fetch(page_url)
        if not html:
            print(f"  [SKIP] Detail page failed")
            results.append({
                'title': list_title,
                'content': '',
                'page_url': page_url,
                'attachments': [],
                'date': '',
            })
            continue
        
        title, content, attachments, date = extract_detail(html, page_url)
        
        # Use list title if detail title is empty
        if not title:
            title = list_title
        
        # Clean title: remove "行政许可_铅山县生态环境局" suffix from <title>
        title = re.sub(r'\s*[_-]\s*[^_-]+$', '', title)
        
        results.append({
            'title': title,
            'content': content,
            'page_url': page_url,
            'attachments': attachments,
            'date': date,
        })
        
        print(f"  -> {title[:50]} | date={date} | attach={len(attachments)} | content_len={len(content)}")
        time.sleep(0.2)
    
    # Step 3: Push to search DB
    SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
    
    try:
        import sqlite3
        conn = sqlite3.connect(SEARCH_DB, timeout=60)
        c = conn.cursor()
        
        inserted = 0
        for r in results:
            summary = r['content'][:200] if r['content'] else ''
            attachments_json = str(r['attachments']) if r['attachments'] else ''
            has_table = 1 if '| --- |' in r['content'] else 0
            c.execute(
                """INSERT OR REPLACE INTO gov_raw 
                   (title, content, page_url, source_url, site_name, publish_date, summary, attachments, has_table, category)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, '行政许可')""",
                (r['title'], r['content'], r['page_url'], r['page_url'], SITE_NAME,
                 r['date'], summary, attachments_json, has_table)
            )
            inserted += 1
        
        conn.commit()
        conn.close()
        print(f"\n✅ Pushed {inserted} records to search DB")
        
    except Exception as e:
        print(f"\n[ERROR] DB push failed: {e}")
        # Print results for manual inspection
        for r in results:
            print(f"\n--- {r['title'][:60]} ---")
            print(f"URL: {r['page_url']}")
            print(f"Date: {r['date']}")
            print(f"Content: {r['content'][:200]}...")
            if r['attachments']:
                for a in r['attachments']:
                    print(f"  Attach: {a['name']} -> {a['url']}")
    
    # Summary
    print(f"\n{'='*50}")
    print(f"站点: {SITE_NAME}")
    print(f"栏目: 行政许可")
    print(f"采集页数: {MAX_PAGES}")
    print(f"采集条数: {len(results)}")
    print(f"{'='*50}")

if __name__ == '__main__':
    main()