#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""crawl_wenjiang_sthj.py - 成都市温江区-生态环境
站点: https://www.wenjiang.gov.cn (瑞数4代: 412 + $_ts; 证书过期须 ignore_https_errors)
列表: /gkml/sthj/detail-list/column-index-1.shtml?page=N (静态 ul.list>li.myflex)
  onclick="window.open('/gkml/sthj/{id}.shtml')" 提取详情链接
  字段: .cont 标题 / .unit 部门 / .date 日期
分页: column-index-{N}.shtml (路径式, 同 longquanyi)
"""
import sys, os, re, json, time, random, sqlite3, html as html_lib
from datetime import datetime, timedelta
from playwright.sync_api import sync_playwright

SITE_NAME = "成都市温江区-生态环境"
BASE = "https://www.wenjiang.gov.cn"
LIST_TPL = "https://www.wenjiang.gov.cn/gkml/sthj/detail-list/column-index-{pn}.shtml"
MAX_PAGES_DEFAULT = 5
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
# 生态环境栏目 (环评审批决定/受理公示/行政处罚等)
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
OUT = "/tmp/wenjiang_sthj.jsonl"
DB_PATH = "/root/search.db"
CHROMIUM = "/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome"
SCRIPT_NAME = "crawl_wenjiang_sthj.py"


def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return MAX_PAGES_DEFAULT


def clean_title(t):
    return re.sub(r"^[•··\s]+", "", t or "").strip().replace("\u200b", "").replace("\ufeff", "")


def extract_items(page):
    """从 li.myflex 提取 (title, url, date)"""
    items = []
    lis = page.query_selector_all("ul.list > li.myflex, li.myflex")
    for li in lis:
        onclick = li.get_attribute("onclick") or ""
        m = re.search(r"window\.open\('([^']+)'", onclick)
        if not m:
            m = re.search(r"window\.open\(\"([^\"]+)\"", onclick)
        if not m:
            continue
        href = m.group(1)
        if href.startswith("/"):
            href = BASE + href
        elif not href.startswith("http"):
            href = BASE + "/" + href
        cont = li.query_selector(".cont")
        date_el = li.query_selector(".date")
        title = cont.inner_text().strip() if cont else ""
        date = date_el.inner_text().strip() if date_el else ""
        if title:
            items.append({"title": clean_title(title), "url": href, "date": date})
    return items


def extract_block(el):
    """递归提取 HTML 块 (保持顺序): p/pre 保留内部HTML, table 保留, div 递归"""
    if el.name == "table":
        tbl = str(el)
        tbl = re.sub(r'href=["\'](/[^"\']+)["\']', lambda mm: 'href="%s%s"' % (BASE, mm.group(1)), tbl)
        return [tbl]
    if el.name in ("p", "pre"):
        if el.find_parent("table"):
            return []
        txt = el.get_text(" ", strip=True)
        if not txt or re.match(r"^(责任编辑|初审|复审|终审|\[纠错\]|浏览次数|字号|分享到|扫一扫|相关附件)", txt):
            return []
        return [re.sub(r"\s+", " ", str(el)).strip()]
    if el.name == "div":
        blocks = []
        for c in el.find_all(["p", "pre", "table", "div"], recursive=False):
            blocks.extend(extract_block(c))
        if not blocks:
            txt = el.get_text(" ", strip=True)
            if txt:
                blocks.append(f"<p>{txt}</p>")
        return blocks
    return []


def parse_detail(page):
    """详情页: title/date/content (附件型: 正文=附件PDF链接)"""
    html = page.content()
    title = ""
    m = re.search(r"<h1[^>]*>(.*?)</h1>", html, re.S)
    if m:
        title = re.sub(r"<[^>]+>", "", m.group(1)).strip()
    if not title:
        m = re.search(r"<title>(.*?)(?:\s*-\s*(?:重点项目清单|信息公开目录|成都市龙泉驿区人民政府).*)?</title>", html, re.S)
        if m:
            title = m.group(1).strip()
    date_text = ""
    m = re.search(r'<meta name="PubDate" content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        date_text = m.group(1)
    if not date_text:
        m = re.search(r"发布日期[：:]\s*(\d{4}-\d{2}-\d{2})", html)
        if m:
            date_text = m.group(1)
    content = ""
    try:
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(html, "html.parser")
        wz = soup.find("div", class_=re.compile(r"^content$|article|detail|wzcon", re.I))
        if not wz:
            wz = soup.find("div", class_=re.compile(r"content|article|detail|wzcon", re.I))
        if not wz:
            wz = soup.find("div", id=re.compile(r"zoom|content|article|detail", re.I))
        if wz:
            for el in wz.find_all(class_=re.compile(r"share|qrcode|print|wza", re.I)):
                el.decompose()
            parts = []
            for el in wz.find_all(["p", "pre", "table", "div"], recursive=False):
                parts.extend(extract_block(el))
            dedup = []
            for p in parts:
                if p and (not dedup or dedup[-1] != p):
                    dedup.append(p)
            content = "\n\n".join(dedup)
        # 附件独立提取 (全页 uploadfiles 链接, 不依赖 wz 容器)
        attachments = []
        for a in soup.find_all("a", href=re.compile(r"uploadfiles/.*\.(docx?|pdf|xlsx?|rar|zip|wps|et|dps)(\?|$)", re.I)):
            href = a.get("href", "")
            name = re.sub(r"【.*?】", "", a.get_text(strip=True)).lstrip("• ") or href.split("/")[-1]
            if href.startswith("/"):
                href = BASE + href
            elif not href.startswith("http"):
                href = BASE + "/" + href
            attachments.append({"name": name, "url": href})
        for a in attachments:
            link = f'<p><a href="{a["url"]}" target="_blank">{a["name"]}</a></p>'
            if link not in content:
                content += "\n" + link
        if len(content.strip()) < 20:
            content = f'<p><a href="{page.url}" target="_blank">{title}</a></p>'
    except ImportError:
        pass
    return title, date_text, content


def main():
    max_pages = parse_pages_arg()
    dryrun = "--dryrun" in sys.argv
    jsonl_mode = "--jsonl" in sys.argv
    print(f"[longquanyi] 重点项目清单 pages={max_pages} cutoff={CUTOFF}", flush=True)

    with sync_playwright() as p:
        browser = p.chromium.launch(
            executable_path=CHROMIUM, headless=True,
            args=["--no-sandbox", "--disable-setuid-sandbox", "--disable-blink-features=AutomationControlled"])
        ctx = browser.new_context(user_agent=UA, viewport={"width": 1366, "height": 768}, ignore_https_errors=True)
        page = ctx.new_page()
        try:
            items = []
            for pn in range(1, max_pages + 1):
                url = LIST_TPL.format(pn=pn)
                print(f"  [列表] 第{pn}页", flush=True)
                page.goto(url, timeout=60000, wait_until="domcontentloaded")
                time.sleep(8 if pn == 1 else 4)  # 瑞数挑战
                page_items = extract_items(page)
                if not page_items:
                    print(f"    第{pn}页 0 条, 停止", flush=True)
                    break
                print(f"    第{pn}页 {len(page_items)} 条", flush=True)
                items.extend(page_items)
                time.sleep(random.uniform(2, 4))

            print(f"[列表] 共 {len(items)} 条", flush=True)
            # 去重
            seen = set()
            items = [x for x in items if not (x["url"] in seen or seen.add(x["url"]))]

            results = []
            for idx, it in enumerate(items):
                time.sleep(random.uniform(3, 6))
                try:
                    page.goto(it["url"], timeout=45000, wait_until="domcontentloaded")
                    # 瑞数挑战时序不定: 轮询等正文容器出现 (最多15s), 失败兜底 sleep
                    try:
                        page.wait_for_selector("div.content, div.article, div.detail, div.wzcon", timeout=15000)
                    except Exception:
                        pass
                    time.sleep(2)
                    title, date_text, content = parse_detail(page)
                    date_text = date_text or it["date"]
                    if date_text and date_text < CUTOFF:
                        continue
                    results.append({
                        "url": it["url"],
                        "title": title or it["title"],
                        "date": date_text,
                        "content": content,
                    })
                    print(f"  [{idx+1}/{len(items)}] {results[-1]['title'][:40]} | {date_text}", flush=True)
                except Exception as e:
                    print(f"  [{idx+1}/{len(items)}] 失败: {str(e)[:60]}", flush=True)

            print(f"[完成] {len(results)} 条", flush=True)
            if dryrun:
                for r in results:
                    print(json.dumps(r, ensure_ascii=False))
                return
            if jsonl_mode:
                with open(OUT, "a", encoding="utf-8") as f:
                    for r in results:
                        f.write(json.dumps(r, ensure_ascii=False) + "\n")
                print(f"[jsonl] {len(results)} 条 -> {OUT}", flush=True)
                return
            conn = sqlite3.connect(DB_PATH, timeout=60)
            conn.execute("PRAGMA busy_timeout=300000")
            c = conn.cursor()
            new_count = 0
            for r in results:
                c.execute("SELECT id FROM gov_raw WHERE page_url=?", (r["url"],))
                if c.fetchone():
                    continue
                c.execute(
                    "INSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank, script_name) VALUES (?,?,?,?,?,?,?,?,?,?)",
                    (r["title"], SITE_NAME, "四川", r["url"], r["date"], r["content"], "", "",
                     int(r["date"].replace("-", "")) if re.match(r"\d{4}-\d{2}-\d{2}", r["date"]) else 0,
                     SCRIPT_NAME))
                if c.rowcount:
                    new_count += 1
            conn.commit()
            conn.close()
            print(f"[入库] 新增 {new_count}", flush=True)
        finally:
            browser.close()


if __name__ == "__main__":
    main()