#!/usr/bin/env python3
"""
德安县人民政府-公告公示
数据源: XML files (data_2025.xml, data_2026.xml) - via Playwright
详情: div#detailCont (zhengwen TRS_Editor)
"""
import os
import sys
import sqlite3
from bs4 import BeautifulSoup
import requests
from playwright.sync_api import sync_playwright

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "德安县-公告公示"
BASE_URL = "https://www.dean.gov.cn"
LIST_URL = BASE_URL + "/xwzx/zdgg/"
DAYS = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else 365

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)


def fetch_xml_via_playwright(year):
    """Fetch XML data via Playwright (bypasses anti-curl protection)"""
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(LIST_URL, wait_until="domcontentloaded", timeout=15000)
        page.wait_for_timeout(2000)
        xml = page.evaluate("""
            async (yr) => {
                const resp = await fetch('/xwzx/zdgg/data_' + yr + '.xml');
                return await resp.text();
            }
        """, year)
        browser.close()
        return xml


def parse_xml_items(xml_text):
    """Parse XML to extract items"""
    soup = BeautifulSoup(xml_text, "xml")
    items = []
    for item in soup.find_all("ITEM"):
        title_el = item.find("TITLE")
        url_el = item.find("PUBURL")
        date_el = item.find("RELTIME")
        
        title = title_el.get_text(strip=True) if title_el else ""
        puburl = url_el.get_text(strip=True) if url_el else ""
        date_str = date_el.get_text(strip=True) if date_el else ""
        
        if not title or not puburl:
            continue
        
        # Resolve URL
        if puburl.startswith("http"):
            page_url = puburl
        elif puburl.startswith("./"):
            page_url = LIST_URL + puburl[2:]
        else:
            page_url = LIST_URL + puburl
        
        items.append({"page_url": page_url, "title": title, "publish_date": date_str})
    
    return items


def fetch_detail(page_url):
    try:
        r = session.get(page_url, timeout=15)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
        
        content = ""
        # Try multiple content selectors
        for sel in [
            ("id", "detailCont"),
            ("class_", "TRS_Editor"),
            ("class_", "trs_editor_view"),
            ("class_", "zhengwen"),
        ]:
            d = soup.find(sel[0], sel[1])
            if d:
                for tag in d.find_all(["script", "style"]):
                    tag.decompose()
                content = str(d)
                break
        
        return content
    except Exception as e:
        print("  [ERROR] " + str(e), flush=True)
        return ""


def main():
    print("[{}] Starting crawl (days={})".format(SITE_NAME, DAYS), flush=True)
    
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()
    
    # Fetch XML data via Playwright
    print("  Fetching data_2026.xml...", flush=True)
    xml_2026 = fetch_xml_via_playwright("2026")
    items_2026 = parse_xml_items(xml_2026)
    print("  2026 items: {}".format(len(items_2026)), flush=True)
    
    print("  Fetching data_2025.xml...", flush=True)
    xml_2025 = fetch_xml_via_playwright("2025")
    items_2025 = parse_xml_items(xml_2025)
    print("  2025 items: {}".format(len(items_2025)), flush=True)
    
    all_items = items_2026 + items_2025
    print("  Total items: {}".format(len(all_items)), flush=True)
    
    # Get existing
    c.execute("SELECT page_url FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    existing = set(row[0] for row in c.fetchall())
    
    new_count = 0
    for i, item in enumerate(all_items):
        if item["page_url"] in existing:
            continue
        
        content = fetch_detail(item["page_url"])
        item["content"] = content
        
        c.execute(
            "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content) VALUES (?, ?, ?, ?, ?, ?)",
            (SITE_NAME, LIST_URL, item["page_url"], item["title"], item["publish_date"], item["content"])
        )
        if c.rowcount > 0:
            new_count += 1
        conn.commit()
        
        if (i + 1) % 20 == 0:
            print("  Progress: {}/{} processed, {} new".format(i + 1, len(all_items), new_count), flush=True)
    
    conn.close()
    total = sqlite3.connect(SEARCH_DB, timeout=60).execute(
        "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)
    ).fetchone()[0]
    print("  New: {}, Total in DB: {}".format(new_count, total), flush=True)
    print("[{}] Done!".format(SITE_NAME), flush=True)


if __name__ == "__main__":
    main()