#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
深泽县人民政府 - 公告公示 (shenze.gov.cn/columns/160b07f3-35d9-41b3-b820-92b1a0a2f7ca)
CMS: 河北 columns/UUID 模板
列表: /columns/{colid}/index.html  + ?page=N (pageSize=15, totalPages=264)
详情: div#conN (含表格, 需清字号条/打印)
用法: python3 crawl_shenze_gsgg.py [--pages=N] [--json out.jsonl]
"""
import re, sys, os, json, time, sqlite3, requests
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from urllib.parse import urljoin

SITE_NAME = "深泽县人民政府-公告公示"
BASE_URL = "https://www.shenze.gov.cn"
COL_ID = "160b07f3-35d9-41b3-b820-92b1a0a2f7ca"
TID = "88e4d675-d486-4df6-958d-93bd390727ed"
BLOCK_ID = "11a9fda4-be32-4ae3-bdd3-e86dd44a8217"
BLOCK_URL = f"{BASE_URL}/columns/{COL_ID}/templates/{TID}/blocks/{BLOCK_ID}"
LIST_URL = f"{BASE_URL}/columns/{COL_ID}/index.html"
DOMAIN = "www.shenze.gov.cn"
GROUP = "河北"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}

def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)

def clean_title(t):
    t = re.sub(r" | |\u200b|\ufeff", "", t or "")
    t = re.sub(r"\s+", " ", t)
    t = re.sub(r"\.\.\.|…$", "", t).strip()
    return t

def fetch(url, timeout=30):
    for attempt in range(4):
        try:
            r = requests.get(url, headers=HEADERS, timeout=timeout)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            if attempt == 3:
                log(f"fetch ERR {url[-50:]}: {e}")
                return ""
            time.sleep(2 * (attempt + 1))
    return ""

def fetch_list(page):
    # 石家庄站群分页: blocks AJAX 接口 (必须带 fix=0)
    url = f"{BLOCK_URL}?page={page}&fix=0"
    html = fetch(url)
    if not html or len(html) < 100:
        return []
    items = []
    # 列表项: <a href=".../2026MM/DD/uuid.html">标题</a> + 日期在相邻 <a>
    pat = re.compile(r'<a target="_blank" href="([^"]*/columns/' + re.escape(COL_ID) + r'/20\d{4}/\d{1,2}/[0-9a-f-]+\.html)"[^>]*>(.*?)</a>', re.S)
    for m in pat.finditer(html):
        href, inner = m.group(1), m.group(2)
        title = re.sub(r"<[^>]+>", "", inner)
        title = clean_title(title)
        if not title:
            tm = re.search(r'title="([^"]+)"', inner)
            title = clean_title(tm.group(1) if tm else "")
        if title and href.startswith("http"):
            items.append({"title": title, "url": href})
    return items

def extract_clean_text(content_html, page_url=None):
    if not content_html:
        return ""
    soup = BeautifulSoup(content_html, "html.parser")
    # 清理干扰: 字号条/打印/字体选择
    for tag in soup.find_all(attrs={"class": re.compile(r"font|print|fz|FZ")}):
        tag.decompose()
    for tag in soup.find_all(string=re.compile(r"【字体：|字号：|打印")):
        p = tag.find_parent(["p", "div", "span", "td"])
        if p:
            p.decompose()
    for a_tag in soup.find_all("a", href=True):
        href = a_tag.get("href", "")
        text = a_tag.get_text(strip=True) or "附件"
        full_url = urljoin(page_url or BASE_URL, href)
        a_tag.replace_with(f'<p><a href="{full_url}">{text}</a></p>')
    for tag in soup.find_all(['span', 'b', 'strong', 'font', 'em', 'i', 'u', 's']):
        tag.unwrap()
    parts = []
    for el in soup.find_all(['table', 'p']):
        if el.name == 'table':
            parts.append(str(el))
        elif el.name == 'p' and not el.find_parent('table'):
            t = el.get_text(separator='', strip=True)
            if t:
                parts.append(f'<p>{t}</p>')
    return '\n\n'.join(parts) if parts else content_html.strip()

def parse_detail(url):
    html = fetch(url)
    if not html:
        return {"title": "", "publish_date": "", "content": ""}
    soup = BeautifulSoup(html, "html.parser")
    result = {"title": "", "publish_date": "", "content": ""}
    ttag = soup.find("title")
    if ttag:
        result["title"] = clean_title(re.sub(r"_\w*$", "", ttag.get_text(strip=True)))
    m = re.search(r"发布时间：(\d{4}-\d{2}-\d{2})", html)
    if m:
        result["publish_date"] = m.group(1)
    conN = soup.find("div", id="conN")
    if conN:
        result["content"] = extract_clean_text(str(conN), url)
    if not result["content"]:
        # fallback: 找最大文本 div
        for div in soup.find_all("div"):
            txt = div.get_text(separator="", strip=True)
            if len(txt) > 300:
                result["content"] = extract_clean_text(str(div), url)
                break
    return result

def crawl(pages=5, json_out=None):
    log(f"Crawl pages={pages} cutoff={CUTOFF}")
    items_all = []
    seen = set()
    for p in range(1, pages + 1):
        items = fetch_list(p)
        new = 0
        for it in items:
            if it["url"] not in seen:
                seen.add(it["url"])
                items_all.append(it)
                new += 1
        log(f"P{p}: {len(items)} items ({new} new), total {len(items_all)}")
        if not items:
            break
        time.sleep(0.6)

    enriched = []
    for idx, it in enumerate(items_all, 1):
        d = parse_detail(it["url"])
        if d["publish_date"] and d["publish_date"] < CUTOFF:
            continue
        enriched.append({
            "title": d["title"] or it["title"],
            "page_url": it["url"],
            "publish_date": d["publish_date"],
            "content": d["content"],
            "site_name": SITE_NAME,
            "column": "公告公示",
        })
        if idx % 10 == 0:
            log(f"  {idx}/{len(items_all)}")
        time.sleep(0.3)

    if json_out:
        with open(json_out, "w", encoding="utf-8") as f:
            for item in enriched:
                f.write(json.dumps(item, ensure_ascii=False) + "\n")
        log(f"JSONL: {len(enriched)} items -> {json_out}")
        return

    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=10000")
    c = conn.cursor()
    stored = skipped = 0
    for item in enriched:
        try:
            c.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, content, summary, category, script_name)"
                " VALUES (?,?,?,?,?,?,?,?,?)",
                (item["site_name"], item["page_url"], item["page_url"],
                 item["title"], item["publish_date"], item["content"],
                 (item["content"] or "")[:500], item["column"], "crawl_shenze_gsgg.py"))
            if c.rowcount > 0:
                stored += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"DB error: {e}")
            skipped += 1
    conn.commit()
    conn.close()
    log(f"Result: {stored} new, {skipped} skipped")

if __name__ == "__main__":
    args = sys.argv[1:]
    pages = 5
    json_out = None
    if "--pages=" in " ".join(args):
        pages = int(re.search(r"--pages=(\d+)", " ".join(args)).group(1))
    if "--json" in args:
        json_out = args[args.index("--json") + 1]
    crawl(pages=pages, json_out=json_out)