#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""简阳市人民政府 - 公示公告/环境保护 栏目爬虫
站点: https://www.scjy.gov.cn (四川成都, /jianyang/ 站群)
栏目:
  --col=gsgg  公示公告  c123218  es-search lmCode=d622afddb27b43809f0686e7bacb0847 (832条/42页)
  --col=hb    环境保护  c123242  es-search lmCode=2bfac8b06c194e76adfff651b2df5fa9 (94条/5页)
WAF: 瑞数4代 JS 挑战 (412, $_ts) → Playwright 过挑战 + 页面内 fetch
列表: GET /es-search/search/{lmCode}?_template=zhaofa/jian_list&_isAgg=1&_pageSize=20&page=N
      返回 HTML: div.gail_content_list > a.gail_list_doctitle[title][href] + div.gail_list_doctime
      尾部 ejscreatePageHTML("page_div", 总页数, 当前页, 20, 总条数, callback)
详情: fetch → meta ArticleTitle(完整标题)/PubDate + div.main_bd 正文 (图片型公告 img 绝对化; 附件相对路径绝对化)
入库: 自动检测 /root/search.db 存在则 INSERT OR IGNORE 直插 gov_raw (FTS 触发器自动同步 gov_search)
运行: python3 crawl_jianyang.py [--col=gsgg|hb] [--pages=N] [--no-db]
"""
import json, os, re, sys, time, sqlite3
from datetime import datetime, timedelta
from playwright.sync_api import sync_playwright

COLS = {
    "gsgg": {
        "site_name": "简阳市-公示公告",
        "lm_code": "d622afddb27b43809f0686e7bacb0847",
        "category": "公示公告",
        "seed": "https://www.scjy.gov.cn/jianyang/c123218/list_more.shtml",
    },
    "hb": {
        "site_name": "简阳市-环境保护",
        "lm_code": "2bfac8b06c194e76adfff651b2df5fa9",
        "category": "环境保护",
        "seed": "https://www.scjy.gov.cn/jianyang/c123242/zdxxgk_list.shtml",
    },
}
BASE = "https://www.scjy.gov.cn"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
DB_PATH = "/root/search.db"

def clean_title(t):
    t = re.sub(r"^[•··.\s]+", "", t or "").strip()
    t = t.replace("\u200b", "").replace("\ufeff", "")
    t = re.sub(r"\.{3,}$", "", t).strip()
    return t

def clean_content(html):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup.find_all(lambda t: t.name and ":" in t.name):
        tag.decompose()
    for tag in soup.find_all(["span", "font"]):
        tag.unwrap()
    # 只提升含嵌套 p 的外层 p (如 <p 外壳><div><p>段1</p><p>段2</p></div></p>),
    # 保留内层真段落 p — 勿 unwrap 内层 p 否则段落被拍平成一坨 (永丰修复)
    while True:
        nested = [p for p in soup.find_all("p") if p.find("p")]
        if not nested:
            break
        nested[0].unwrap()
    for p in soup.find_all("p"):
        if p.get("style"):
            p.attrs = {"align": p.get("align")} if p.get("align") else {}
    return str(soup)

def absolutize_url(href, base_url):
    if not href or href.startswith("javascript"):
        return href
    href = href.strip()
    if href.startswith("http"):
        return href
    if href.startswith("/"):
        return BASE + href
    return base_url.rsplit("/", 1)[0] + "/" + href

def extract_attachments(html_frag, detail_url):
    atts = []
    for m in re.finditer(r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', html_frag, re.I | re.S):
        href, name_html = m.group(1).strip(), m.group(2)
        if not href or href.startswith("javascript"):
            continue
        if re.search(r'\.(pdf|docx?|xlsx?|zip|rar|7z)$', href, re.I):
            name = re.sub(r"<[^>]+>", "", name_html).strip()[:100]
            atts.append({"name": name or href.split("/")[-1], "url": absolutize_url(href, detail_url)})
    seen = set()
    uniq = []
    for a in atts:
        if a["url"] not in seen:
            seen.add(a["url"])
            uniq.append(a)
    return uniq

def fetch_html(page, url):
    for _ in range(2):
        try:
            return page.evaluate("""async (u) => { const r = await fetch(u); return await r.text(); }""", url)
        except Exception:
            time.sleep(2)
    return None

def push_to_db(results, site_name, category):
    if not results:
        return 0, 0
    if not os.path.exists(DB_PATH):
        print(f"  未发现 {DB_PATH}, 跳过入库 (仅写 JSONL)", flush=True)
        return 0, 0
    conn = sqlite3.connect(DB_PATH, timeout=290)
    conn.execute("PRAGMA busy_timeout=290000")
    added = skipped = 0
    cur = conn.cursor()
    for r in results:
        try:
            cur.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments, group_name, industry)
                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""",
                (site_name, r["url"], r["url"], r["title"], r["pub_date"], r["content"],
                 (r["content"] or "")[:500], category, "crawl_jianyang.py", r["attachments"],
                 r.get("group_name", "四川"), r.get("industry", "other"))
            )
            if cur.rowcount > 0:
                added += 1
            else:
                skipped += 1
        except sqlite3.Error as e:
            print(f"  DB err: {e}", flush=True)
            skipped += 1
        conn.commit()
    conn.close()
    return added, skipped

def main():
    col = "gsgg"
    for a in sys.argv:
        if a.startswith("--col="):
            col = a.split("=", 1)[1]
    if col not in COLS:
        print(f"未知栏目 {col}, 可选: {list(COLS)}", flush=True)
        return
    cfg = COLS[col]
    site_name = cfg["site_name"]
    no_db = "--no-db" in sys.argv
    max_pages = None
    for a in sys.argv:
        if a.startswith("--pages="):
            max_pages = int(a.split("=", 1)[1])
    print(f"站点: {site_name} CUTOFF: {CUTOFF}", flush=True)

    results = []
    stats = {"list": 0, "detail_ok": 0, "empty": 0, "cutoff": 0, "pages": 0, "total": 0}
    cutoff_hit = False
    with sync_playwright() as p:
        browser = p.chromium.launch(
            headless=True,
            args=["--no-sandbox", "--disable-blink-features=AutomationControlled", "--disable-dev-shm-usage"])
        ctx = browser.new_context(user_agent=UA, locale="zh-CN", viewport={"width": 1366, "height": 900}, ignore_https_errors=True)
        page = ctx.new_page()

        try:
            page.goto(cfg["seed"], timeout=60000, wait_until="commit")
        except Exception:
            pass
        passed = False
        for i in range(20):
            page.wait_for_timeout(3000)
            try:
                html = page.content()
            except Exception:
                continue
            if "$_ts" not in html and len(html) > 5000:
                print(f"瑞数挑战通过 {i*3}s", flush=True)
                passed = True
                break
        if not passed:
            print("挑战失败!", flush=True)
            browser.close()
            return

        n = 1
        while True:
            if max_pages and n > max_pages:
                break
            api_url = f"{BASE}/es-search/search/{cfg['lm_code']}?_template=zhaofa/jian_list&_isAgg=1&_pageSize=20&page={n}"
            txt = fetch_html(page, api_url)
            if not txt:
                print(f"  第{n}页获取失败", flush=True)
                break
            # 总页数/总条数
            m = re.search(r'ejscreatePageHTML\("page_div",\s*(\d+)\s*,\s*\d+\s*,\s*20\s*,\s*(\d+)\s*,', txt)
            if m and stats["total"] == 0:
                stats["pages"] = int(m.group(1))
                stats["total"] = int(m.group(2))
                print(f"共 {stats['total']} 条 / {stats['pages']} 页", flush=True)
            items = re.findall(
                r'<a href="([^"]+)"[^>]*class="gail_list_doctitle"[^>]*title="([^"]+)">.*?</a>\s*<div class="gail_list_doctime">([^<]+)<',
                txt, re.S)
            if not items:
                print(f"  第{n}页无条目, 停止", flush=True)
                break
            print(f"  第{n}页: {len(items)} 条", flush=True)
            for href, t_title, t_date in items:
                stats["list"] += 1
                list_title = clean_title(t_title)
                # 列表给 http:// 详情 URL → https 页面内 fetch 混合内容拦截 → 换 https
                detail_url = href.replace("http://", "https://")
                pub_date = (t_date or "")[:10]
                if pub_date and pub_date < CUTOFF:
                    stats["cutoff"] += 1
                    cutoff_hit = True
                    continue
                detail_html = fetch_html(page, detail_url)
                if not detail_html:
                    print(f"    详情失败: {list_title[:30]}", flush=True)
                    continue
                stats["detail_ok"] += 1
                m2 = re.search(r'<meta[^>]+name="ArticleTitle"[^>]+content="([^"]+)"', detail_html)
                title = clean_title(m2.group(1)) if m2 else list_title
                m2 = re.search(r'<meta[^>]+name="PubDate"[^>]+content="([^"]+)"', detail_html)
                pub_date = (m2.group(1) if m2 else pub_date)[:10]
                if pub_date and pub_date < CUTOFF:
                    stats["cutoff"] += 1
                    cutoff_hit = True
                    continue
                m2 = re.search(r'<div[^>]*class="main_bd"[^>]*>(.*?)</div>\s*<div[^>]*class="xgfj"', detail_html, re.S)
                content_html = m2.group(1) if m2 else ""
                content = clean_content(content_html) if content_html else ""
                text = re.sub(r"<[^>]+>", "", content).strip()
                if len(text) < 10 and "<img" not in content:
                    print(f"    空正文跳过: {title[:30]}", flush=True)
                    stats["empty"] += 1
                    continue
                # img/链接 绝对化 (相对路径按详情页目录)
                content = re.sub(
                    r'(<img[^>]+src=")([^"]+)(")',
                    lambda mm: mm.group(1) + absolutize_url(mm.group(2), detail_url) + mm.group(3), content)
                atts = extract_attachments(content_html, detail_url)
                rec = {
                    "site_name": site_name,
                    "title": title,
                    "pub_date": pub_date,
                    "content": content,
                    "source_url": detail_url,
                    "url": detail_url,
                    "group_name": "四川",
                    "industry": "other",
                    "attachments": json.dumps(atts, ensure_ascii=False) if atts else "",
                }
                results.append(rec)
            if cutoff_hit:
                break
            if n >= stats["pages"]:
                break
            n += 1
        browser.close()

    out = f"/tmp/jianyang_{col}.jsonl"
    with open(out, "w", encoding="utf-8") as f:
        for r in results:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    print(f"已写 {len(results)} 条到 {out}", flush=True)
    if not no_db:
        added, skipped = push_to_db(results, site_name, cfg["category"])
        print(f"=== 入库: 新增{added} 跳过{skipped} ===", flush=True)
    print(f"=== 完成: 列表{stats['list']} 详情OK{stats['detail_ok']} 空{stats['empty']} CUTOFF截{stats['cutoff']} ===", flush=True)

if __name__ == "__main__":
    main()