#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""crawl_nc_tzgg.py - 南昌市人民政府-公示公告 (www.nc.gov.cn)
站点: 静态HTML, 无WAF
列表: /ncszf/tzgg/2021_nav_list.shtml (createPageHTML 分页, 共67页, 1000/page?)
  分页: /ncszf/tzgg/2021_nav_list_{N}.shtml
详情: /ncszf/tzgg/{YYYYMM}/{hash}.shtml
运行: python3 crawl_nc_tzgg.py [--pages=N] [--dryrun]
"""
import sys, os, re, json, time, requests, sqlite3
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import urllib3
urllib3.disable_warnings()

SITE_NAME = "南昌市-公示公告"
BASE = "https://www.nc.gov.cn"
MAX_PAGES_DEFAULT = 5
CUTOFF = (datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
OUT = "/tmp/nc_tzgg.jsonl"
DB_PATH = "/root/search.db"
SCRIPT_NAME = "crawl_nc_tzgg.py"

LIST_TPL = BASE + "/ncszf/tzgg/2021_nav_list{suffix}.shtml"


def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
    return MAX_PAGES_DEFAULT


def fetch(url, retries=2):
    for i in range(retries):
        try:
            r = requests.get(url, headers={"User-Agent": UA}, timeout=20, verify=False)
            r.encoding = "utf-8"
            return r.text
        except Exception:
            time.sleep(2)
    return ""


def list_items(page):
    if page == 1:
        url = LIST_TPL.format(suffix="")
    else:
        url = LIST_TPL.format(suffix=f"_{page}")
    html = fetch(url)
    if not html:
        return []
    items = []
    # 详情链接: /ncszf/tzgg/{YYYYMM}/{hash}.shtml
    for m in re.finditer(r'href="(/ncszf/tzgg/\d{6}/[0-9a-f]+\.shtml)"[^>]*>([^<]{8,100})', html):
        items.append({"title": m.group(2).strip(), "url": BASE + m.group(1)})
    return items


def parse_detail(url):
    html = fetch(url)
    if not html:
        return "", "", ""
    title = ""
    m = re.search(r"<title>([^<]*?)(?:\s*-\s*南昌市人民政府)?</title>", html, re.S)
    if m:
        title = re.sub(r"<[^>]+>", "", m.group(1)).strip()
    date_text = ""
    for pat in [r'(\d{4}-\d{2}-\d{2})\s*\d{2}:\d{2}', r"发布时间[：:]\s*(\d{4}-\d{2}-\d{2})"]:
        m = re.search(pat, html)
        if m:
            date_text = m.group(1)
            break
    content = ""
    soup = BeautifulSoup(html, "html.parser")
    zoom = soup.find("div", id=re.compile(r"zoom|content|article", re.I)) or soup.find("div", class_=re.compile(r"zoom|content|article|wzcon|text", re.I))
    if zoom:
        for el in zoom.find_all(class_=re.compile(r"share|qrcode|print|wza", re.I)):
            el.decompose()
        parts = []
        for el in zoom.find_all(["p", "pre", "table", "div"], recursive=True):
            if el.name in ("p", "pre") and el.find_parent("table"):
                continue
            if el.name == "table":
                parts.append(str(el))
                continue
            if el.name == "div":
                if el.find(["p", "pre", "table"], recursive=False):
                    continue
                txt = el.get_text("", strip=True)
                if txt:
                    parts.append(txt)
                continue
            txt = el.get_text("", strip=True)
            if txt and not re.match(r"^(责任编辑|初审|复审|终审|\[纠错\]|浏览次数|字号|分享)", txt):
                parts.append(txt)
        dedup = []
        for p in parts:
            if p and (not dedup or dedup[-1] != p):
                dedup.append(p)
        content = "\n\n".join(dedup)
    atts = []
    for a in soup.find_all("a", href=re.compile(r"\.(docx?|pdf|xlsx?|rar|zip|wps|et|dps)$", re.I)):
        href = a.get("href", "")
        name = a.get_text(strip=True) or href.split("/")[-1]
        if href.startswith("/"):
            href = BASE + href
        elif not href.startswith("http"):
            href = BASE + "/" + href
        atts.append({"name": name, "url": href})
    for a in atts:
        link = f'<p><a href="{a["url"]}" target="_blank">{a["name"]}</a></p>'
        if link not in content:
            content += "\n" + link
    if len(content.strip()) < 20:
        content = f'<p><a href="{url}" target="_blank">{title}</a></p>'
    return title, date_text, content


def main():
    max_pages = parse_pages_arg()
    dryrun = "--dryrun" in sys.argv
    print(f"[nc_tzgg] pages={max_pages} cutoff={CUTOFF}", flush=True)
    results = []
    for pn in range(1, max_pages + 1):
        items = list_items(pn)
        if not items:
            print(f"  第{pn}页 0 条, 停止", flush=True)
            break
        print(f"  第{pn}页 {len(items)} 条", flush=True)
        for it in items:
            try:
                title, date_text, content = parse_detail(it["url"])
                if date_text and date_text < CUTOFF:
                    continue
                results.append({"url": it["url"], "title": title or it["title"], "date": date_text, "content": content})
            except Exception as e:
                print(f"    ✗ {str(e)[:60]}", flush=True)
            time.sleep(0.4)
    print(f"[完成] {len(results)} 条", flush=True)
    if dryrun:
        for r in results[:3]:
            print(json.dumps(r, ensure_ascii=False)[:200])
        return
    with open(OUT, "w", encoding="utf-8") as f:
        for r in results:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    if os.path.exists(DB_PATH):
        conn = sqlite3.connect(DB_PATH, timeout=120)
        conn.execute("PRAGMA busy_timeout=120000")
        c = conn.cursor()
        added = 0
        for r in results:
            c.execute("SELECT id FROM gov_raw WHERE page_url=?", (r["url"],))
            if c.fetchone():
                continue
            c.execute(
                "INSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank, script_name) VALUES (?,?,?,?,?,?,?,?,?,?)",
                (r["title"], SITE_NAME, "江西", r["url"], r["date"], r["content"], "", "",
                 int(r["date"].replace("-", "")) if re.match(r"\d{4}-\d{2}-\d{2}", r["date"]) else 0, SCRIPT_NAME))
            if c.rowcount:
                added += 1
        conn.commit()
        conn.close()
        print(f"[入库] 新增 {added}", flush=True)


if __name__ == "__main__":
    main()