#!/usr/bin/env python3
"""
掇刀区人民政府 - 通知公告 (crawl_duodao_tzgg.py)
TRS jPage (dataproxy.jsp AJAX)
总计1407条, 94页(15条/页)
"""
import json, os, sys, time, re, urllib.request, sqlite3

SITE_NAME = "duodao_tzgg"
BASE_URL = "https://www.duodao.gov.cn"
PROXY_URL = f"{BASE_URL}/module/web/jpage/dataproxy.jsp"
TEMP_DB = "/root/search.db"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Content-Type": "application/x-www-form-urlencoded",
    "Referer": f"{BASE_URL}/col/col7563/index.html"
}
TOTAL_RECORDS = 1407
PER_PAGE = 15
TOTAL_PAGES = (TOTAL_RECORDS + PER_PAGE - 1) // PER_PAGE  # 94


def fetch_list(startrecord, endrecord):
    """Fetch one page from dataproxy API"""
    params = urllib.parse.urlencode({
        "startrecord": startrecord, "endrecord": endrecord,
        "perpage": PER_PAGE, "unitid": "72809", "webid": "39",
        "path": "/", "columnid": "7563",
        "webname": "掇刀区人民政府"
    }).encode("utf-8")
    req = urllib.request.Request(PROXY_URL, data=params, headers=HEADERS)
    resp = urllib.request.urlopen(req, timeout=30)
    return resp.read().decode("utf-8", errors="replace")


def parse_list(xml):
    """Extract articles from XML response"""
    articles = []
    for m in re.finditer(
        r'<a[^>]*href="([^"]*)"[^>]*title="([^"]*)"[^>]*>.*?</a><span>([^<]*)</span>',
        xml, re.DOTALL
    ):
        href = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()
        if not href.startswith("http"):
            href = BASE_URL + href
        articles.append({"title": title, "url": href, "date": date})
    return articles


def fetch_detail(url):
    """Fetch article detail content"""
    try:
        req = urllib.request.Request(url, headers={"User-Agent": HEADERS["User-Agent"]})
        resp = urllib.request.urlopen(req, timeout=30)
        html = resp.read().decode("utf-8", errors="replace")
        content = ""
        m = re.search(r'<div[^>]*class="con zoom"[^>]*>.*?<div[^>]*class="main-txt"[^>]*id="content"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
        if m:
            content = m.group(1)
        else:
            m = re.search(r'<div[^>]*class="main-txt"[^>]*id="content"[^>]*>(.*?)</div>', html, re.DOTALL)
            if m:
                content = m.group(1)
        if content:
            content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL)
            content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL)
        text_len = len(re.sub(r'<[^>]+>', '', content).strip())
        return content if text_len >= 30 else ""
    except urllib.error.HTTPError as e:
        return "" if e.code == 404 else ""
    except Exception:
        return ""


def main():
    os.chdir("/root")
    incremental = "--incremental" in sys.argv

    # Phase 1: Collect URLs
    print("=== Phase 1: Collecting URLs ===")
    if incremental:
        pages = [(1, PER_PAGE)]
        print(f"Mode: INCREMENTAL (1 page, {PER_PAGE} items)")
    else:
        pages = [(s, min(s + PER_PAGE - 1, TOTAL_RECORDS))
                 for s in range(1, TOTAL_RECORDS + 1, PER_PAGE)]
        print(f"Mode: FULL ({len(pages)} pages)")

    all_articles = []
    seen_urls = set()

    for start, end in pages:
        try:
            xml = fetch_list(start, end)
            articles = parse_list(xml)
            new = 0
            for art in articles:
                if art["url"] not in seen_urls:
                    seen_urls.add(art["url"])
                    all_articles.append(art)
                    new += 1
            print(f"  {start}-{end}: {len(articles)} items, +{new}, total={len(all_articles)}")
        except Exception as e:
            print(f"  {start}-{end} ERROR: {e}")
        time.sleep(0.2)

    print(f"\n=== {len(all_articles)} articles collected ===")
    if not all_articles:
        return

    # Save list
    with open("/tmp/duodao_articles.json", "w") as f:
        json.dump(all_articles, f, ensure_ascii=False)
    print("Saved to /tmp/duodao_articles.json")

    # Phase 2: Fetch details
    print("\n=== Phase 2: Fetching details ===")
    conn = sqlite3.connect(TEMP_DB, timeout=60)
    c = conn.cursor()
    c.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT, content TEXT, source_url TEXT UNIQUE,
        site_name TEXT, publish_date TEXT
    )""")
    conn.commit()

    inserted = 0
    skipped = 0
    empty = 0

    for i, art in enumerate(all_articles):
        c.execute("SELECT COUNT(*) FROM gov_raw WHERE source_url = ?", (art["url"],))
        if c.fetchone()[0] > 0:
            skipped += 1
            continue

        content = fetch_detail(art["url"])
        if content:
            try:
                c.execute(
                    "INSERT OR IGNORE INTO gov_raw (title, content, source_url, site_name, publish_date) VALUES (?, ?, ?, ?, ?)",
                    (art["title"], content, art["url"], SITE_NAME, art["date"])
                )
                conn.commit()
                inserted += 1
            except Exception as e:
                print(f"  DB error: {e}")
        else:
            empty += 1

        if i % 50 == 0 or i == len(all_articles) - 1:
            print(f"  [{i}/{len(all_articles)}] done={inserted} skip={skipped} empty={empty}")

        time.sleep(0.1)

    conn.close()
    print(f"\n=== Done ===")
    print(f"Total: {len(all_articles)}")
    print(f"Inserted: {inserted}")
    print(f"Skipped: {skipped}")
    print(f"Empty content: {empty}")


if __name__ == "__main__":
    main()