#!/usr/bin/env python3
"""
莒县人民政府-公告公示 (www.juxian.gov.cn)
====================================
CMS: 大汉版通 (Hanweb) - JPage 动态分页 (CSRF保护)
列表: 首页HTML内嵌XML datastore (45条), 分页需JPage AJAX+CSRF
详情: /art/2026/6/8/art_163432_10410495.html, <div id="zoom"> 含正文

用法:
    python3 crawl_juxian_gggs.py          # 全量(仅首页45条)
    python3 crawl_juxian_gggs.py --test   # 测试 5 条
"""

import re, sys, os, time
from datetime import datetime, timedelta, timezone
import requests, urllib3
import os
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME  = "莒县人民政府-公告公示"
LIST_URL   = "http://www.juxian.gov.cn/col/col163432/index.html?number=JU0023"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CUTOFF     = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS    = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch_list():
    try:
        r = requests.get(LIST_URL, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print("  ! list fail: " + str(e)[:60])
        return None

def parse_list(html):
    """Parse records from embedded XML datastore"""
    items = []
    # Extract all CDATA records
    for rec in re.finditer(r'<record><!\[CDATA\[(.*?)\]\]></record>', html, re.DOTALL):
        rec_html = rec.group(1)
        # URL + title
        a_m = re.search(r'<a[^>]*href="([^"]*)"[^>]*title="([^"]*)"', rec_html)
        if not a_m:
            a_m = re.search(r'<a[^>]*href="([^"]*)"[^>]*>([^<]+)</a>', rec_html)
        if not a_m:
            continue
        url = a_m.group(1)
        if not url.startswith("http"):
            url = "http://www.juxian.gov.cn" + url if url.startswith("/") else "http://www.juxian.gov.cn/" + url
        title = a_m.group(2).strip() if a_m.lastindex >= 2 else a_m.group(2).strip()
        # Date
        date_m = re.search(r'<span class="time">(\d{4}-\d{2}-\d{2})', rec_html)
        pub_date = date_m.group(1) if date_m else ""
        items.append({"title": title, "url": url, "pub_date": pub_date})
    return items

def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        html = r.text
        # Content is in <div id="zoom">
        m = re.search(r'id="zoom"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            content = m.group(1)
            # Clean scripts/styles
            content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
            content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
            # Remove meta markers
            content = re.sub(r'<!--.*?-->', '', content)
            content = re.sub(r'<meta[^>]*>', '', content)
            return content.strip()
        return ""
    except Exception as e:
        print("  ! detail fail: " + str(e)[:60])
        return ""

def to_db(items):
    if not items:
        return 0, 0
    import sqlite3
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, fail = 0, 0
    for it in items:
        try:
            db.execute(
                "INSERT OR REPLACE INTO gov_raw "
                "(site_name, title, page_url, content, publish_date, summary, tags) "
                "VALUES (?,?,?,?,?,?,?)",
                (
                    SITE_NAME,
                    (it.get("title") or "")[:500],
                    it.get("url", ""),
                    it.get("content", ""),
                    (it.get("pub_date") or "")[:10],
                    "",
                    "公告公示",
                )
            )
            if db.total_changes > 0:
                ok += 1
            else:
                fail += 1
        except Exception as e:
            fail += 1
    db.execute(
        "INSERT INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary "
        "FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,)
    )
    db.commit()
    db.close()
    return ok, fail

def crawl(test=False):
    html = fetch_list()
    if not html:
        return 0, 0
    items = parse_list(html)
    if not items:
        print("  ! no items found")
        return 0, 0
    
    # Filter by date
    filtered = [it for it in items if not it["pub_date"] or it["pub_date"] >= CUTOFF]
    
    f = filtered[0]["pub_date"] if filtered else "?"
    l = filtered[-1]["pub_date"] if filtered else "?"
    print(f"  List: {len(items)} items, {len(filtered)} within 3yr ({f} ~ {l})")
    
    if test:
        filtered = filtered[:5]
        print(f"  TEST mode: {len(filtered)} items")
    
    if not filtered:
        return 0, 0
    
    # Fetch details
    for i, item in enumerate(filtered):
        print(f"  [{i+1}/{len(filtered)}] {item['title'][:40]}... ", end="", flush=True)
        content = fetch_detail(item["url"])
        item["content"] = content
        print(f"{len(content)}B")
    
    ok, fail = to_db(filtered)
    return ok, fail

if __name__ == "__main__":
    test = "--test" in sys.argv
    mode = "TEST" if test else "FULL"
    print()
    print(f"[{SITE_NAME}] {mode}")
    t0 = time.time()
    ok, fail = crawl(test=test)
    print(f"  Time: {round(time.time()-t0, 1)}s")
    print(f"  New: {ok} Skip: {fail}")