#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
寻乌县人民政府-公示公告爬虫
crawl_xunwu_tzgg.py
列表: div.pageList > ul > li > a[title] 标题 + span.time 日期
分页: list.shtml + list_2~5.shtml (5页, 12条/页)
详情: div.article-content-body#zoomcon, h1.article-title标题, meta[PubDate]日期
"""
import re, json, time, sys, os, sqlite3
from datetime import datetime, timedelta
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://www.xunwu.gov.cn"
LIST_PATH = "/xwxrmzf/c103707"
LIST_URL = BASE_URL + LIST_PATH + "/list.shtml"
SITE_NAME = "寻乌县-公示公告"
GROUP = "江西"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
INCREMENTAL_DAYS = 7
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TIMEOUT = 20
session = requests.Session()
session.headers.update(HEADERS)

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)
def fetch(url):
    resp = session.get(url, timeout=TIMEOUT)
    resp.encoding = "utf-8"
    return resp.text

def parse_list(html):
    """解析列表页 — div.pageList > ul > li > a[title] + span.time"""
    soup = BeautifulSoup(html, "html.parser")
    items = []
    page_list = soup.find("div", class_="pageList")
    if not page_list:
        return items
    ul = page_list.find("ul")
    if not ul:
        return items
    for li in ul.find_all("li", recursive=False):
        a = li.find("a", href=True)
        if not a:
            continue
        title = a.get("title") or a.get_text(strip=True)
        if not title:
            continue
        href = a["href"].strip()
        if not href.startswith("http"):
            href = urljoin(BASE_URL, href)
        date_span = li.find("span", class_="time")
        date_str = date_span.get_text(strip=True)[:10] if date_span else ""
        items.append((title.strip(), href, date_str))
    return items

def parse_detail(html, url):
    """解析详情页 — h1.article-title + div.article-content-body#zoomcon"""
    soup = BeautifulSoup(html, "html.parser")
    
    # 标题
    title = ""
    meta_t = soup.find("meta", attrs={"name": re.compile(r"ArticleTitle", re.I)})
    if meta_t and meta_t.get("content"):
        title = meta_t["content"].strip()
    if not title:
        h1 = soup.find("h1", class_="article-title")
        if h1:
            title = h1.get_text(strip=True)
    if not title:
        title_m = re.search(r'<UCAPTITLE>(.*?)</UCAPTITLE>', html, re.S)
        if title_m:
            title = title_m.group(1).strip()
    
    # 日期
    pub_date = ""
    meta_d = soup.find("meta", attrs={"name": re.compile(r"PubDate", re.I)})
    if meta_d and meta_d.get("content"):
        pub_date = meta_d["content"].strip()[:10]
    if not pub_date:
        dm = re.search(r'<PUBLISHTIME>(.*?)</PUBLISHTIME>', html, re.S)
        if dm:
            pd = dm.group(1).strip()[:10]
            pub_date = pd
    if not pub_date:
        dm2 = re.search(r"(\d{4}-\d{2}-\d{2})", html)
        if dm2:
            pub_date = dm2.group(1)
    
    # 正文
    content = ""
    content_div = soup.find("div", class_="article-content-body") or soup.find("div", id="zoomcon")
    if not content_div:
        content_div = soup.find("div", class_=re.compile(r"content|zoom", re.I))
    if content_div:
        paragraphs = []

        def extract_paragraphs(container):
            """递归提取container内的<p>段落和表格"""
            for child in container.children:
                if child.name == "p":
                    p_html = str(child)
                    p_html = re.sub(r'<a[^>]*href="([^"]*)"[^>]*>([^<]*)</a>', r'[\2](\1)', p_html)
                    p_html = re.sub(r'<img[^>]*src="([^"]*)"[^>]*>', r'![\1]', p_html)
                    text = re.sub(r'<[^>]+>', '', p_html).strip()
                    text = re.sub(r'\s+', ' ', text)
                    text = re.sub(r'&nbsp;', ' ', text).strip()
                    if text:
                        paragraphs.append(text)
                elif child.name == "div":
                    # 跳过附件div（style/fj-a等）
                    cls = " ".join(child.get("class", [])) if child.get("class") else ""
                    if "fj-a" in cls or "fj" in cls.lower():
                        # 检查是否有实际附件链接
                        att_links = child.find_all("a", href=True)
                        if not att_links:
                            continue
                    extract_paragraphs(child)
                elif child.name == 'table':
                    tbl_html = html_table_to_html(child, url)
                    if tbl_html:
                        paragraphs.append(tbl_html)
                elif child.name and child.name not in ("style", "script"):
                    # 未知包裹标签（ucapcontent等）— 递归进去提取
                    extract_paragraphs(child)

        extract_paragraphs(content_div)

        if paragraphs:
            content = "\n\n".join(paragraphs)
        else:
            content = re.sub(r'<[^>]+>', '', str(content_div)).strip()
            content = re.sub(r'\s+', ' ', content).strip()
    
    # 附件
    attachments = []
    scope = content_div or soup
    for a_tag in scope.find_all("a", href=True):
        ahref = a_tag["href"].strip().lower()
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", ahref):
            full_url = urljoin(url, a_tag["href"].strip())
            attachments.append({
                "name": a_tag.get_text(strip=True) or full_url.split("/")[-1],
                "url": full_url
            })
            content += f'\n\n附件：<p><a href="{full_url}">{a_tag.get_text(strip=True)}</a></p>'
    
    # 空内容回退：正文过短（< 20字符）时回退为 [标题](原文URL)
    if len(content.strip()) < 20:
        fallback_title = title.strip() or "内容"
        content = f'<p><a href="{url}">{fallback_title}</a></p>'

    return title, pub_date, content, attachments


def incremental_filter(items):
    cutoff = datetime.now() - timedelta(days=INCREMENTAL_DAYS)
    filtered = []
    for title, url, date_str in items:
        try:
            if date_str:
                item_date = datetime.strptime(date_str, "%Y-%m-%d")
                if item_date >= cutoff:
                    filtered.append((title, url, date_str))
            else:
                filtered.append((title, url, date_str))
        except:
            filtered.append((title, url, date_str))
    return filtered


def main():
    is_incremental = any(arg in sys.argv for arg in ["--incremental", "incremental", "inc"])
    
    page_urls = [LIST_URL]
    for i in range(2, MAX_PAGES + 1):
        page_urls.append(BASE_URL + LIST_PATH + f"/list_{i}.shtml")
    
    all_items = []
    for idx, url in enumerate(page_urls):
        try:
            html = fetch(url)
            items = parse_list(html)
            print(f"  Page {idx+1}/{MAX_PAGES}: {len(items)} items", file=sys.stderr)
            all_items.extend(items)
            time.sleep(0.3)
        except Exception as e:
            print(f"  Page {idx+1} error: {e}", file=sys.stderr)
    
    print(f"  Total: {len(all_items)}", file=sys.stderr)
    
    if is_incremental:
        all_items = incremental_filter(all_items)
        print(f"  Incremental ({INCREMENTAL_DAYS}d): {len(all_items)}", file=sys.stderr)
    
    seen = set()
    unique = []
    for item in all_items:
        if item[1] not in seen:
            seen.add(item[1])
            unique.append(item)
    print(f"  Unique: {len(unique)}", file=sys.stderr)
    
    saved = 0
    skipped = 0
    errors = 0
    for idx, (title, url, list_date) in enumerate(unique):
        try:
            html = fetch(url)
            det_title, det_date, content, attachments = parse_detail(html, url)
            final_title = det_title or title
            final_date = det_date or list_date
            
            content_text = content[:50000] if content else ''
            summary = re.sub(r'\s+', ' ', content_text[:200]).strip() or final_title
            att_json = json.dumps(attachments, ensure_ascii=False) if attachments else ''
            
            conn = sqlite3.connect(DB_PATH, timeout=60)
            c = conn.cursor()
            c.execute('''INSERT OR IGNORE INTO gov_raw 
                (page_url, title, publish_date, site_name, group_name, summary, content, attachments, source_url)
                VALUES (?,?,?,?,?,?,?,?,?)''',
                (url, final_title, final_date, SITE_NAME, GROUP,
                 summary, content_text, att_json, url))
            if c.rowcount > 0:
                saved += 1
            else:
                skipped += 1
            conn.commit()
            conn.close()
            if (idx + 1) % 10 == 0:
                print(f"  [{idx+1}/{len(unique)}] saved {saved}, skipped {skipped}, errors {errors}", file=sys.stderr)
            time.sleep(0.3)
        except Exception as e:
            print(f"  [{idx+1}] ERROR {final_title[:30]}: {e}", file=sys.stderr)
            errors += 1
            time.sleep(0.5)
    
    print(f"\n{'='*50}", file=sys.stderr)
    print(f"[{SITE_NAME}] Done! Saved: {saved}, Skipped: {skipped}, Errors: {errors}", file=sys.stderr)
    print(f"{'='*50}", file=sys.stderr)


if __name__ == "__main__":
    main()