#!/usr/bin/env python3
import os
"""
crawl_hukou.py — 湖口县人民政府·生态环境局·环境影响评价
====================================================
TRS CMS，pagination: index_N.html (0-indexed, 0=N page base)

列表：<a class='xxgklist' href="..." title="Title">...</a><span>YYYY-MM-DD</span>
详情：<div id="content">正文 (TRS_EDITOR) | <meta name="PubDate" content="YYYY-MM-DD">

用法:
    python3 crawl_hukou.py             # 全量（9页+近3年）
    python3 crawl_hukou.py 1           # 增量（只爬首页）
"""

import re, sys, os, time, requests

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "湖口县行政许可"
BASE_URL = "https://www.hukou.gov.cn"
LIST_URL = f"{BASE_URL}/sthjj/fdzdgknr/hjjg/"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}

# 列表正则
LIST_PAT = re.compile(
    r"<a class='xxgklist' [^>]*href=\"([^\"]+)\"[^>]*title=\"([^\"]*)\"[^>]*>.*?</a>\s*<span>([^<]+)</span>",
    re.DOTALL,
)


def fetch(url):
    resp = requests.get(url, headers=HEADERS, timeout=30)
    resp.encoding = "utf-8"
    return resp.text


def get_list_items(url):
    """获取列表页"""
    try:
        html = fetch(url)
    except Exception as e:
        print(f"  ❌ 请求失败: {e}")
        return []
    items = []
    for m in LIST_PAT.finditer(html):
        href = m.group(1).strip()
        title = m.group(2).strip()
        pub_date = m.group(3).strip()
        if not href.startswith("http"):
            href = BASE_URL + href if href.startswith("/") else f"{BASE_URL}/{href}"
        items.append({"title": title, "url": href, "pub_date": pub_date})
    return items


def get_detail(item):
    """获取详情"""
    try:
        html = fetch(item["url"])
    except Exception as e:
        print(f"    ❌ 详情页请求失败: {e}")
        item["content"] = ""
        item["source"] = SITE_NAME
        return item

    # 正文 — <div id="content">
    m = re.search(r'<div[^>]*id="content"[^>]*>(.*?)</div>', html, re.DOTALL)
    content = m.group(1).strip() if m else ""
    content = re.sub(r'<script[^>]*>.*?</script>', "", content, flags=re.DOTALL)

    # 日期 — <meta name="PubDate" content="YYYY-MM-DD">
    m = re.search(r'PubDate"\s*content="(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        item["pub_date"] = m.group(1)

    # 标题 — <title>
    m = re.search(r"<title>(.*?)</title>", html)
    if m:
        t = m.group(1).replace(" - 湖口县人民政府网站", "").replace(" - 湖口县人民政府", "").replace(" - 湖口县", "").strip()
        if t:
            item["title"] = t

    item["content"] = content
    item["source"] = SITE_NAME
    return item


def main():
    incremental = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    mode = "增量" if incremental else "全量"
    print(f"🔍 {SITE_NAME} [{mode}]")

    t0 = time.time()
    all_items = []

    # 首页
    items = get_list_items(LIST_URL)
    if not items:
        print("  ⏭ 首页无数据")
        return
    print(f"  📄 第1页: {len(items)}条 ({items[-1]['pub_date']}~{items[0]['pub_date']})")
    all_items.extend(items)

    if not incremental:
        # 第2~9页 (index_1.html ~ index_8.html)
        for pg in range(1, 9):
            url = f"{LIST_URL}index_{pg}.html"
            items = get_list_items(url)
            if not items:
                break
            filtered = [it for it in items if not it["pub_date"] or it["pub_date"] >= "2023-06"]
            print(f"  📄 第{pg+1}页: {len(items)}条 ({items[-1]['pub_date']}~{items[0]['pub_date']}) → 近3年{len(filtered)}条")
            all_items.extend(filtered)
            if len(filtered) < len(items):
                break

    if not all_items:
        print("  ⏭ 无数据")
        return

    # 爬详情
    print(f"\n  爬取详情页 ({len(all_items)}条)...")
    for i, item in enumerate(all_items):
        get_detail(item)
        if (i + 1) % 10 == 0:
            print(f"    ... {i+1}/{len(all_items)}", end="\r")

    elapsed = time.time() - t0
    print(f"\n📊 共获取 {len(all_items)} 条，耗时 {elapsed:.0f}s")

    # 入库
    import sqlite3
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")

    ok, skip = 0, 0
    for it in all_items:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                (
                    SITE_NAME, it["url"], it["url"],
                    (it["title"] or "")[:500], it.get("pub_date", ""),
                    (it["title"] or "")[:500], it.get("content", ""),
                    "active", "环境影响评价", "",
                ),
            )
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except Exception as e:
            skip += 1

    db.commit()
    db.execute(
        "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,),
    )
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}")


if __name__ == "__main__":
    main()