#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""crawl_hsq_tztg.py - 黄山市黄山区人民政府-公示公告
站点: https://www.hsq.gov.cn (帝联 NWAF: 412 + Environment Checking + ctct_bundle)
过盾: playwright + playwright-stealth (同 ahshx 方案)
列表: 静态HTML /dwzw/tztg/index.html → /dwzw/tztg/{id}.html
  分页: /content/column/6794244?pageIndex=N (Ls.pagination)
详情: div.wzcon.j-fontContent 正文 + meta ArticleTitle/PubDate + 附件下载
"""
import sys, os, re, json, time, random, sqlite3, html as html_lib
from datetime import datetime, timedelta
from playwright.sync_api import sync_playwright
from playwright_stealth import Stealth

SITE_NAME = "黄山市黄山区-公示公告"
BASE = "https://www.hsq.gov.cn"
LIST_URL = "https://www.hsq.gov.cn/dwzw/tztg/index.html"
COLUMN_ID = "6794244"
MAX_PAGES_DEFAULT = 5
CUTOFF = (datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
OUT = "/tmp/hsq_tztg.jsonl"
DB_PATH = "/root/search.db"
CHROMIUM = "/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome"
LOCK_FILE = "/tmp/hsq_tztg.lock"
SCRIPT_NAME = "crawl_hsq_tztg.py"


def _drop_container_parts(parts):
    """剔除「容器段」：find_all(['p','div']) 会同时收下容器 <div> 与它内部的 <p>，
    导致同一内容重复（容器那份常还被 get_text(strip=True) 拍平）。
    判据：某段被列表内另一段完全包含 → 它是容器段 → 剔除。
    保护：剔除后为空则原样返回。
    """
    if not parts:
        return parts
    ps = [p for p in parts if isinstance(p, str)]
    if len(ps) != len(parts):
        return parts
    keep = []
    for a in parts:
        if len(a) >= 40 and any(b is not a and b and b in a for b in parts):
            continue
        keep.append(a)
    return keep if keep else parts


def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return MAX_PAGES_DEFAULT


def clean_title(t):
    t = re.sub(r"^\[图\]\s*", "", t or "").strip()
    t = re.sub(r"^[•··\s]+", "", t).strip()
    return t.replace("\u200b", "").replace("\ufeff", "")


def parse_content(page):
    """从详情页提取 (title, date, content_html, attachments)"""
    html = page.content()
    title = ""
    m = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()
    if not title:
        m = re.search(r"<title>(.*?)(?:_黄山市黄山区人民政府)?</title>", html, re.S)
        if m:
            title = m.group(1).strip()
    date_text = ""
    m = re.search(r'<meta name="PubDate" content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        date_text = m.group(1)
    if not date_text:
        m = re.search(r"发布时间[：:]\s*(\d{4}-\d{2}-\d{2})", html)
        if m:
            date_text = m.group(1)

    # 正文容器
    content = ""
    try:
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(html, "html.parser")
        wz = soup.find("div", class_=re.compile(r"wzcon|j-fontContent"))
        if not wz:
            wz = soup.find("div", id=re.compile(r"zoom|content|article", re.I))
        if wz:
            # 清理无用的页脚工具条
            for el in wz.find_all(class_=re.compile(r"appendix|share|qrcode|scan|ewm|print|wza", re.I)):
                el.decompose()
            # 附件
            attachments = []
            for a in wz.find_all("a", href=re.compile(r"\.(docx?|pdf|xlsx?|rar|zip|wps|et|dps)$", re.I)):
                href = a.get("href", "")
                name = a.get_text(strip=True) or href.split("/")[-1]
                if href.startswith("/"):
                    href = BASE + href
                attachments.append({"name": name, "url": href})
            # 段落化: p/pre 级
            parts = []
            for el in wz.find_all(["p", "pre", "table", "div"], recursive=True):
                if el.name in ("p", "pre") and el.find_parent("table"):
                    continue
                if el.name == "table" and el.find_parent("table"):
                    continue
                if el.name == "table":
                    tbl = str(el)
                    tbl = re.sub(r'href=["\'](/[^"\']+)["\']', lambda mm: 'href="%s%s"' % (BASE, mm.group(1)), tbl)
                    parts.append(tbl)
                    continue
                if el.name == "div":
                    if el.find(["p", "pre", "table"], recursive=False):
                        continue
                    txt = el.get_text("", strip=True)
                    if txt:
                        parts.append(txt)
                    continue
                txt = el.get_text("", strip=True)
                if txt and not re.match(r"^(责任编辑|初审|复审|终审|\[纠错\]|浏览次数|字号|分享到|扫一扫)", txt):
                    parts.append(txt)
            dedup = []
            for p in parts:
                if p and (not dedup or dedup[-1] != p):
                    dedup.append(p)
            parts = dedup
            parts = _drop_container_parts(parts)
            content = "\n\n".join(parts)
            # 附件转内嵌段
            for a in attachments:
                link = f'<p><a href="{a["url"]}" target="_blank">{a["name"]}</a></p>'
                if link not in content:
                    content += "\n" + link
            if len(content.strip()) < 20:
                content = f'<p><a href="{page.url}" target="_blank">{title}</a></p>'
    except ImportError:
        pass
    return title, date_text, content


def main():
    max_pages = parse_pages_arg()
    dryrun = "--dryrun" in sys.argv
    jsonl_mode = "--jsonl" in sys.argv
    print(f"[hsq] 公示公告 pages={max_pages} cutoff={CUTOFF}", flush=True)

    with sync_playwright() as p:
        browser = p.chromium.launch(
            executable_path=CHROMIUM, headless=True,
            args=["--no-sandbox", "--disable-setuid-sandbox", "--disable-blink-features=AutomationControlled"])
        ctx = browser.new_context(user_agent=UA, viewport={"width": 1366, "height": 768})
        Stealth().apply_stealth_async(ctx)
        page = ctx.new_page()
        try:
            items = []
            for pn in range(1, max_pages + 1):
                if pn == 1:
                    url = LIST_URL
                else:
                    url = f"{BASE}/content/column/{COLUMN_ID}?pageIndex={pn}"
                print(f"  [列表] 第{pn}页 {url}", flush=True)
                page.goto(url, timeout=60000, wait_until="domcontentloaded")
                time.sleep(10 if pn == 1 else 5)  # 首页过盾等待
                # 提取列表项
                rows = page.eval_on_selector_all(
                    "a[href*='/dwzw/tztg/']",
                    "els => els.map(e => ({t: e.innerText.trim(), h: e.href})).filter(x => x.t.length > 5)")
                seen = set()
                page_items = []
                for r in rows:
                    m = re.search(r"/dwzw/tztg/(\d+)\.html", r["h"])
                    if not m or r["h"] in seen:
                        continue
                    seen.add(r["h"])
                    page_items.append({"title": clean_title(r["t"]), "url": r["h"], "id": m.group(1)})
                if not page_items:
                    print(f"    第{pn}页 0 条, 停止", flush=True)
                    break
                print(f"    第{pn}页 {len(page_items)} 条", flush=True)
                items.extend(page_items)
                # 判断是否到 CUTOFF (列表页不含日期, 详情页取)
                time.sleep(random.uniform(2, 4))

            print(f"[列表] 共 {len(items)} 条", flush=True)

            # 抓详情
            results = []
            for idx, it in enumerate(items):
                time.sleep(random.uniform(3, 6))
                try:
                    page.goto(it["url"], timeout=45000, wait_until="domcontentloaded")
                    time.sleep(2)
                    title, date_text, content = parse_content(page)
                    if not date_text:
                        date_text = ""
                    if date_text and date_text < CUTOFF:
                        continue
                    results.append({
                        "url": it["url"],
                        "title": title or it["title"],
                        "date": date_text,
                        "content": content,
                    })
                    print(f"  [{idx+1}/{len(items)}] {results[-1]['title'][:40]} | {date_text}", flush=True)
                except Exception as e:
                    print(f"  [{idx+1}/{len(items)}] 失败: {str(e)[:60]}", flush=True)

            print(f"[完成] {len(results)} 条", flush=True)
            if dryrun:
                for r in results:
                    print(json.dumps(r, ensure_ascii=False))
                return
            if jsonl_mode:
                with open(OUT, "a", encoding="utf-8") as f:
                    for r in results:
                        f.write(json.dumps(r, ensure_ascii=False) + "\n")
                print(f"[jsonl] {len(results)} 条 -> {OUT}", flush=True)
                return
            # 入库
            conn = sqlite3.connect(DB_PATH, timeout=60)
            conn.execute("PRAGMA busy_timeout=300000")
            c = conn.cursor()
            new_count = 0
            for r in results:
                c.execute("SELECT id FROM gov_raw WHERE page_url=?", (r["url"],))
                if c.fetchone():
                    continue
                c.execute(
                    "INSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank, script_name) VALUES (?,?,?,?,?,?,?,?,?,?)",
                    (r["title"], SITE_NAME, "安徽", r["url"], r["date"], r["content"], "", "",
                     int(r["date"].replace("-", "")) if re.match(r"\d{4}-\d{2}-\d{2}", r["date"]) else 0,
                     SCRIPT_NAME))
                if c.rowcount:
                    new_count += 1
            conn.commit()
            conn.close()
            print(f"[入库] 新增 {new_count}", flush=True)
        finally:
            browser.close()


if __name__ == "__main__":
    main()