#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""祁门县人民政府 - 环评公示栏目爬虫
站点: https://www.ahqimen.gov.cn
栏目: /zwgk/public/column/6616643?type=4&catId=6719037&action=list&nav=3 (建设项目环境影响评价)
WAF: 创宇盾 CT2-WAAP (知道创宇), 412 JS 挑战 → Playwright 过挑战后 fetch
列表: GET /huangshanzwgk/zwgk/site/label/8888?labelName=publicInfoList&siteId=6793339&organId=6616643&pageSize=10&pageIndex=N&isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&result=&isJson=true&isSetValue=true&catId=6719037
  返回 JSON: {data:[{title,link,publishDate,contentId}], pageCount, total}
详情: fetch link → h1.wztit 标题 + div.wzcon.j-fontContent 正文 + meta ArticleTitle/PubDate/ContentSource
入库: 自动检测 /root/search.db 存在则 INSERT OR IGNORE 直插 gov_raw (FTS 触发器自动同步 gov_search)
运行: python3 crawl_qimen_hpgs.py [--pages=N] [--no-db] [--sync-only]
"""
import json, os, re, sys, time, sqlite3
from datetime import datetime, timedelta
from playwright.sync_api import sync_playwright

SITE_NAME = "祁门县-环评公示"
WARM = "https://www.ahqimen.gov.cn/"
API = "https://www.ahqimen.gov.cn/huangshanzwgk/zwgk/site/label/8888"
ORGAN_ID = "6616643"
CAT_ID = "6719037"
SITE_ID = "6793339"
PAGE_SIZE = 10
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
IP = "219.152.188.33"
HOST = "www.ahqimen.gov.cn"
OUT = "/tmp/qimen_hpgs.jsonl"
DB_PATH = "/root/search.db"
CATEGORY = "环评公示"

def parse_pages_arg():
    """支持 --pages=N 和纯数字"""
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return 0

def clean_title(t):
    t = re.sub(r"^[•··\s]+", "", t or "").strip()
    t = t.replace("\u200b", "").replace("\ufeff", "")
    return t

def clean_content(html):
    """正文清洗: span/font unwrap + 嵌套p扁平化 + o:p清理 + 保留表格"""
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")
    # 移除 Word 命名空间标签
    for tag in soup.find_all(lambda t: t.name and ":" in t.name):
        tag.decompose()
    # 首个 div (wzcon 容器标签) 解构: 取内部内容
    if soup.div is not None and soup.div.name == "div":
        soup.div.unwrap()
    # span/font unwrap
    for tag in soup.find_all(["span", "font"]):
        tag.unwrap()
    # 嵌套 p 扁平化
    # 只提升含嵌套 p 的外层 p (如 <p 外壳><div><p>段1</p><p>段2</p></div></p>),
    # 保留内层真段落 p — 勿 unwrap 内层 p 否则段落被拍平成一坨 (永丰修复)
    while True:
        nested = [p for p in soup.find_all("p") if p.find("p")]
        if not nested:
            break
        nested[0].unwrap()
    # 清 p style
    for p in soup.find_all("p"):
        if p.get("style"):
            p.attrs = {"align": p.get("align")} if p.get("align") else {}
    return str(soup)

def extract_attachments(detail_html, detail_url):
    """正文内附件链接: <a href="*.pdf/doc/xls...">"""
    atts = []
    for m in re.finditer(r'<a[^>]+href="([^"]+)"[^>]*>([^<]{1,100})</a>', detail_html, re.I):
        href, name = m.group(1).strip(), m.group(2).strip()
        if not href or href.startswith("javascript"):
            continue
        if re.search(r'\.(pdf|docx?|xlsx?|zip|rar|7z|png|jpe?g|gif)$', href, re.I):
            if href.startswith("/"):
                href = "https://www.ahqimen.gov.cn" + href
            elif not href.startswith("http"):
                href = detail_url.rsplit("/", 1)[0] + "/" + href
            atts.append({"name": name or href.split("/")[-1], "url": href})
    # 去重
    seen = set()
    uniq = []
    for a in atts:
        if a["url"] not in seen:
            seen.add(a["url"])
            uniq.append(a)
    return uniq

EIA_FILTER = re.compile(r'环境影响|环评|公示|征求意见|受理|批复|社会稳定风险|环境报告|环境分析|环境评价')

def is_eia_title(title):
    return bool(EIA_FILTER.search(title or ""))

def push_to_db(results):
    """INSERT OR IGNORE 直插 gov_raw; FTS 触发器 trg_gov_raw_fts_ins 自动同步 gov_search"""
    if not results:
        return 0, 0
    if not os.path.exists(DB_PATH):
        print(f"  未发现 {DB_PATH}, 跳过入库 (仅写 JSONL)", flush=True)
        return 0, 0
    conn = sqlite3.connect(DB_PATH, timeout=290)
    conn.execute("PRAGMA busy_timeout=290000")
    added = skipped = 0
    cur = conn.cursor()
    for r in results:
        try:
            cur.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments, group_name, industry)
                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""",
                (SITE_NAME, r["url"], r["url"], r["title"], r["pub_date"], r["content"],
                 (r["content"] or "")[:500], CATEGORY, "crawl_qimen_hpgs.py", r["attachments"],
                 r.get("group_name", "安徽"), r.get("industry", "other"))
            )
            if cur.rowcount > 0:
                added += 1
            else:
                skipped += 1
        except sqlite3.Error as e:
            print(f"  DB err: {e}", flush=True)
            skipped += 1
        conn.commit()
    conn.close()
    return added, skipped

def main():
    max_pages = parse_pages_arg()
    no_db = "--no-db" in sys.argv
    if "--sync-only" in sys.argv:
        print("sync-only 模式: 本脚本抓取+入库一体, 无需单独同步", flush=True)
        return
    print(f"站点: {SITE_NAME} 最大页: {max_pages or '全量'} CUTOFF: {CUTOFF}", flush=True)

    results = []
    stats = {"list": 0, "eia": 0, "detail_ok": 0, "empty": 0}
    with sync_playwright() as p:
        browser = p.chromium.launch(
            headless=True,
            args=[
                "--no-sandbox",
                "--disable-blink-features=AutomationControlled",
                "--disable-dev-shm-usage",
                f"--host-resolver-rules=MAP {HOST} {IP}",
            ])
        ctx = browser.new_context(user_agent=UA, locale="zh-CN", viewport={"width": 1366, "height": 900})
        page = ctx.new_page()

        # 过创宇盾挑战
        try:
            page.goto(WARM, timeout=60000, wait_until="commit")
        except Exception:
            pass
        passed = False
        for i in range(20):
            page.wait_for_timeout(3000)
            try:
                html = page.content()
            except Exception:
                continue
            if "Environment Checking" not in html and len(html) > 5000:
                print(f"创宇盾挑战通过 {i*3}s", flush=True)
                passed = True
                break
        if not passed:
            print("挑战失败!", flush=True)
            browser.close()
            return

        # 分页抓列表
        page_idx = 1
        total_fetched = 0
        cutoff_hit = False
        while True:
            if (max_pages and page_idx > max_pages) or cutoff_hit:
                break
            qs = "&".join([
                "labelName=publicInfoList", f"siteId={SITE_ID}", f"organId={ORGAN_ID}",
                f"pageSize={PAGE_SIZE}", f"pageIndex={page_idx}", "isDate=true",
                "dateFormat=yyyy-MM-dd", "length=50", "type=4", "action=list",
                "result=", "isJson=true", "isSetValue=true", f"catId={CAT_ID}"])
            try:
                body = page.evaluate("""async (u) => {
                    const r = await fetch(u, {credentials: 'include'});
                    return await r.text();
                }""", f"{API}?{qs}")
                d = json.loads(body)
            except Exception as e:
                print(f"  页{page_idx} 列表失败: {str(e)[:80]} 重试...", flush=True)
                time.sleep(3)
                try:
                    body = page.evaluate("""async (u) => {
                        const r = await fetch(u, {credentials: 'include'});
                        return await r.text();
                    }""", f"{API}?{qs}")
                    d = json.loads(body)
                except Exception as e2:
                    print(f"  页{page_idx} 重试失败: {str(e2)[:80]}", flush=True)
                    break

            items = d.get("data", [])
            page_count = d.get("pageCount", 0)
            if not items:
                break
            print(f"  页{page_idx}/{page_count} 列表 {len(items)} 条", flush=True)

            for item in items:
                link = item.get("link", "")
                title = clean_title(item.get("title", ""))
                pub_date = (item.get("publishDate") or "")[:10]
                stats["list"] += 1
                # 只保留环评相关标题
                if not is_eia_title(title):
                    continue
                stats["eia"] += 1
                if pub_date and pub_date < CUTOFF:
                    print(f"    日期 {pub_date} < CUTOFF 停止", flush=True)
                    cutoff_hit = True
                    break
                # 详情
                try:
                    detail_html = page.evaluate("""async (u) => {
                        const r = await fetch(u, {credentials: 'include'});
                        return await r.text();
                    }""", link)
                except Exception:
                    time.sleep(2)
                    try:
                        detail_html = page.evaluate("""async (u) => {
                            const r = await fetch(u, {credentials: 'include'});
                            return await r.text();
                        }""", link)
                    except Exception:
                        print(f"    详情失败: {title[:30]}", flush=True)
                        continue
                stats["detail_ok"] += 1
                # 正文容器
                wz = detail_html.find('wzcon j-fontContent')
                if wz < 0:
                    wz = detail_html.find('class="wzcon')
                if wz >= 0:
                    # 回退到 <div 标签开始
                    tag_start = detail_html.rfind('<div', 0, wz)
                    if tag_start >= 0:
                        wz = tag_start
                    # 取容器内容到下一个 </div> 匹配
                    chunk = detail_html[wz:]
                    depth = 0
                    end = 0
                    for mm in re.finditer(r'<div[^>]*>|</div>', chunk):
                        if mm.group(0).startswith('</'):
                            depth -= 1
                            if depth == 0:
                                end = mm.end()
                                break
                        else:
                            depth += 1
                    content_html = chunk[:end] if end else chunk[:8000]
                else:
                    # 兜底: meta 之后的内容
                    content_html = ""
                content = clean_content(content_html) if content_html else ""
                # 正文纯文本长度
                text = re.sub(r"<[^>]+>", "", content).strip()
                if len(text) < 10 and "<img" not in content:
                    print(f"    空正文跳过: {title[:30]}", flush=True)
                    stats["empty"] += 1
                    continue
                # 图片型公告: img src 绝对化
                if "<img" in content:
                    content = re.sub(
                        r'(<img[^>]+src=")([^"]+)(")',
                        lambda m: m.group(1) + (m.group(2) if m.group(2).startswith("http") else ("https://www.ahqimen.gov.cn" + m.group(2) if m.group(2).startswith("/") else link.rsplit("/", 1)[0] + "/" + m.group(2))) + m.group(3),
                        content)
                atts = extract_attachments(detail_html, link)
                rec = {
                    "site_name": SITE_NAME,
                    "title": title,
                    "pub_date": pub_date,
                    "content": content,
                    "source_url": link,
                    "url": link,
                    "group_name": "安徽",
                    "industry": "other",
                    "attachments": json.dumps(atts, ensure_ascii=False) if atts else "",
                }
                results.append(rec)
                total_fetched += 1

            page_idx += 1
            if page_idx > page_count:
                break
            time.sleep(1)

        browser.close()

    save_results(results, stats)
    if not no_db:
        added, skipped = push_to_db(results)
        print(f"=== 入库: 新增{added} 跳过{skipped} ===", flush=True)
    print(f"=== 完成: 列表{stats['list']} 环评{stats['eia']} 详情OK{stats['detail_ok']} 空{stats['empty']} ===", flush=True)

def save_results(results, stats=None):
    with open(OUT, "w", encoding="utf-8") as f:
        for r in results:
            f.write(json.dumps(r, ensure_ascii=False) + "\n")
    print(f"已写 {len(results)} 条到 {OUT}", flush=True)

if __name__ == "__main__":
    main()