#!/usr/bin/env python3
"""金溪县人民政府-调查征集 (jinxi.gov.cn col10626) 爬虫

CMS: 浙江政企CMS (ZJEG)
列表: /col/col10626/index.html  (XML datastore 嵌入页面)
详情: /art/YYYY/M/D/art_XXXXX_XXXXXXX.html
      title: <!--<$[标题]>begin-->TITLE<!--<$[标题]>end-->
      content: <!--ZJEG_RSS.content.begin-->...<!--ZJEG_RSS.content.end-->
"""

import os, sqlite3, re, time, requests, sys, json
from datetime import datetime, timezone, timedelta
from urllib3 import disable_warnings, exceptions
from urllib.parse import unquote
disable_warnings(exceptions.InsecureRequestWarning)

SITE_NAME = "金溪县-调查征集"
LIST_URL = "https://www.jinxi.gov.cn/col/col10626/index.html"
BASE = "https://www.jinxi.gov.cn"
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
MAX_WORKERS = 8

# Unit ID for 调查征集 (from initial page analysis)
UNIT_ID = "65057"
COLUMN_ID = "10560"
TOTAL_RECORDS = 2623


def fetch(url, max_retry=3):
    for att in range(max_retry):
        try:
            r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
            r.encoding = "utf-8"
            if r.status_code == 200:
                return r.text
        except Exception as e:
            if att < max_retry - 1:
                time.sleep(2)
    return None


def parse_records_from_page(html):
    """Parse records from the embedded XML datastore in the page"""
    records = []
    # Find ALL script type="text/xml" blocks with recordset
    for m in re.finditer(r'<script[^>]*type="text/xml"[^>]*>(.*?)</script>', html, re.S):
        script = m.group(1)
        if '<recordset>' not in script:
            continue
        for rec in re.finditer(r'<record>(.*?)</record>', script, re.S):
            rec_html = rec.group(1)
            # Extract title and URL
            a_m = re.search(r'href=[\'"]/(art/[^\'"]+)[\'"]', rec_html)
            title_m = re.search(r'<a[^>]*>(.*?)</a>', rec_html, re.S)
            if a_m and title_m:
                url = BASE + "/" + a_m.group(1)
                title = re.sub(r'<[^>]+>', '', title_m.group(1)).strip()
                # Extract date from URL: /art/YYYY/M/D/art_X_Y.html
                date_m = re.search(r'/art/(\d{4})/(\d+)/(\d+)/', rec_html)
                date_str = ""
                if date_m:
                    m_, d_ = int(date_m.group(2)), int(date_m.group(3))
                    date_str = f"{date_m.group(1)}-{m_:02d}-{d_:02d}"
                if title and url:
                    records.append({"title": title, "url": url, "pub_date": date_str})
    return records


def extract_nextpage_url(html):
    """Extract the next page URL from the XML datastore"""
    m = re.search(r'<nextgroup>(.*?)</nextgroup>', html, re.S)
    if m:
        href_m = re.search(r'href="([^"]+)"', m.group(1))
        if href_m:
            return unquote(href_m.group(1))
    return None


def fetch_detail(url):
    """Extract full title + content from detail page"""
    html = fetch(url)
    if not html:
        return "", ""

    # Title from comment markers
    title = ""
    m = re.search(r'<!--<\$\[(?:标题|title)\]>begin-->(.*?)<!--<\$\[(?:标题|title)\]>end-->', html)
    if m:
        title = m.group(1).strip()
    if not title:
        m = re.search(r'class="[^"]*con-title[^"]*"[^>]*>(.*?)</p>', html, re.S)
        if m:
            title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    if not title:
        m = re.search(r'<title>(.*?)</title>', html)
        if m:
            title = m.group(1).strip()
    title = re.sub(r'begin-->|end-->', '', title).strip()

    # Content from ZJEG markers
    content = ""
    m = re.search(r'<!--ZJEG_RSS\.content\.begin-->(.*?)<!--ZJEG_RSS\.content\.end-->', html, re.S)
    if m:
        content = m.group(1).strip()
    if not content:
        m = re.search(r'<div[^>]*id="zoom"[^>]*>(.*?)</div>', html, re.S)
        if m:
            content = m.group(1).strip()
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.S)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.S)

    # Date from detail page
    date_str = ""
    for d in re.finditer(r'(\d{4}[-/]\d{2}[-/]\d{2})', html):
        d_clean = d.group(1).replace("/", "-")
        ctx = html[max(0,d.start()-80):d.end()+20]
        if any(kw in ctx for kw in ['date', '时间', '发布', '日期', 'publish', '生成日期']):
            date_str = d_clean
            break
    if not date_str:
        for d in re.finditer(r'(\d{4}[-/]\d{2}[-/]\d{2})', html):
            ctx = html[max(0,d.start()-30):d.end()+10]
            if 'sp_time' in ctx or 'time' in ctx.lower():
                date_str = d.group(1).replace("/", "-")
                break
    if not date_str:
        for d in re.finditer(r'(\d{4}[-/]\d{2}[-/]\d{2})', html):
            date_str = d.group(1).replace("/", "-")
            break

    return title, content, date_str


def main():
    test = "--test" in sys.argv
    mode = "TEST" if test else "FULL"
    print(f"\n=== {SITE_NAME} ({mode}) ===", flush=True)

    # Fetch first page
    html = fetch(LIST_URL)
    if not html:
        print("  [FAIL] Cannot fetch list page")
        return

    # Parse records from first page
    all_items = parse_records_from_page(html)
    print(f"  Page 1: {len(all_items)} records", flush=True)

    # Paginate using nextgroup links
    page = 1
    next_url = extract_nextpage_url(html)
    while next_url:
        page += 1
        html = fetch(next_url)
        if not html:
            break
        records = parse_records_from_page(html)
        if not records:
            break
        all_items.extend(records)
        next_url = extract_nextpage_url(html)
        if page % 20 == 0:
            print(f"  Page {page}: {len(all_items)} total", flush=True)

    print(f"  Total: {len(all_items)} records, {page} pages", flush=True)

    # Filter by 3-year cutoff
    items = [it for it in all_items if not it["pub_date"] or it["pub_date"] >= CUTOFF]
    print(f"  Within 3yr: {len(items)} items", flush=True)

    if not items:
        print("  No items to crawl")
        return

    if test:
        items = items[:3]
        print(f"  TEST mode: {len(items)} items", flush=True)

    # Fetch details
    conn = None if test else sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor() if conn else None
    total_new = 0
    done = 0

    for i, item in enumerate(items):
        print(f"  [{i+1}/{len(items)}] {item['title'][:40]}... ", end="", flush=True)
        title, content, pub_date = fetch_detail(item["url"])
        if not title:
            title = item["title"]
        if not pub_date:
            pub_date = item["pub_date"]

        if test:
            print(f"title={title[:40]} content={len(content)}B date={pub_date}")
            continue

        try:
            cursor.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, tags) VALUES (?,?,?,?,?,?,?)",
                (SITE_NAME, title[:500], item["url"], content, pub_date, "", "调查征集"),
            )
            if cursor.rowcount > 0:
                total_new += 1
                print(f"{len(content)}B")
            else:
                print("skip")
        except Exception as e:
            print(f"DB error: {e}")

        done += 1
        if done % 50 == 0 and conn:
            conn.commit()

    if conn:
        conn.commit()
        conn.execute(
            "INSERT INTO gov_search(rowid, title, site_name, summary) "
            "SELECT r.rowid, r.title, r.site_name, r.summary "
            "FROM gov_raw r WHERE r.rowid NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (SITE_NAME,)
        )
        conn.commit()
        c = conn.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,))
        cnt = c.fetchone()[0]
        conn.close()
        print(f"\n=== 完成 ===", flush=True)
        print(f"  新增: {total_new}条", flush=True)
        print(f"  累计: {cnt}条", flush=True)


if __name__ == "__main__":
    main()