#!/usr/bin/env python3
"""Crawler for 清水河县-部门文件 (qingshuihe.gov.cn zfxxgk zfwj)
   https://www.qingshuihe.gov.cn/zfxxgkzl/fdzdgknr/?gk=3 部门文件栏目
   瑞数4代 WAF: playwright 渲染 (page.content() 拿渲染后 DOM)
   列表: zfwj/index.html ~ index_333.html (15条/页, 334页, 5000条)
   详情: div#Zoom 正文, <title> 标题, 文号, 日期, 附件 downloadPdf
"""
import asyncio, re, sys, sqlite3, json, os, time
from playwright.async_api import async_playwright

HOST = "www.qingshuihe.gov.cn"
IP = "139.9.247.160"
BASE = f"http://{HOST}"
LIST_BASE = f"{BASE}/zfxxgkzl/fdzdgknr/zfwj"
MAIN_URL = f"{BASE}/zfxxgkzl/fdzdgknr/?gk=3"
DB = "/root/search.db"
SITE_NAME = "清水河县-部门文件"
SCRIPT_NAME = "crawl_qingshuihe_bmwj.py"
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
CUTOFF = "2023-08-18"  # 与全库一致

def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)

def parse_args():
    pages = 1
    incremental = False
    output = None  # JSONL 输出路径（本地跑，服务器导入）
    start = 1  # 起始页（并发分片用）
    for i, a in enumerate(sys.argv):
        if a == "--pages" and i + 1 < len(sys.argv):
            pages = int(sys.argv[i + 1])
        elif a == "--incremental":
            incremental = True
        elif a == "--output" and i + 1 < len(sys.argv):
            output = sys.argv[i + 1]
        elif a == "--start" and i + 1 < len(sys.argv):
            start = int(sys.argv[i + 1])
        elif a.startswith("--output="):
            output = a.split("=", 1)[1]
        elif a.startswith("--pages="):
            pages = int(a.split("=", 1)[1])
        elif a.startswith("--start="):
            start = int(a.split("=", 1)[1])
    return pages, incremental, output, start

def extract_list_items(html):
    """从列表页提取 (url, 序号)。标题从详情页取（列表页 GBK 乱码）"""
    items = []
    seen = set()
    for m in re.finditer(r'<a[^>]*href="(http://www\.qingshuihe\.gov\.cn/zfxxgkzl/fdzdgknr/zfwj/[^"]+\.html)"[^>]*>', html):
        url = m.group(1)
        if url not in seen:
            seen.add(url)
            items.append(url)
    return items

def extract_detail(html):
    """提取详情: title, content, date, 文号, attachments, has_table"""
    result = {"title": "", "content": "", "date": "", "doc_no": "", "attachments": [], "has_table": False}
    # 标题: <title>xxx_ 清水河县人民政府</title>
    m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
    if m:
        t = re.sub(r'[\s_]*清水河县人民政府[\s_]*$', '', m.group(1)).strip()
        t = re.sub(r'_\s*$', '', t).strip()
        if t:
            result["title"] = t
    # 附件: 整页扫描 downloadPdf + 正文内附件（附件区可能在正文容器外）
    for am in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>([^<]{2,80})</a>', html):
        href, txt = am.group(1), am.group(2).strip()
        if any(k in txt for k in ["附件", "下载", "文件"]) or re.search(r'\.(docx?|xlsx?|pdf|zip|rar|wps|et)$', href, re.I) or "downloadPdf" in href or "download" in href.lower():
            full = href if href.startswith("http") else (BASE + href if href.startswith("/") else BASE + "/" + href)
            full = full.replace("&amp;", "&")
            if (txt, full) not in [(a["name"], a["url"]) for a in result["attachments"]]:
                result["attachments"].append({"name": txt, "url": full})
    # 正文容器
    for pattern in [r'<div[^>]*id="Zoom"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="trs_editor_view"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="TRS_Editor"[^>]*>(.*?)</div>']:
        m = re.search(pattern, html, re.DOTALL)
        if m:
            inner = m.group(1)
            result["has_table"] = bool(re.search(r'<table', inner))
            # 清理: 去掉 script/style
            inner = re.sub(r'<script[^>]*>.*?</script>', '', inner, flags=re.DOTALL)
            inner = re.sub(r'<style[^>]*>.*?</style>', '', inner, flags=re.DOTALL)
            result["content"] = inner.strip()
            break
    # 日期: 发布日期
    m = re.search(r'发布[日日期期]*[：:]\s*([\d]{4}[-/][\d]{1,2}[-/][\d]{1,2})', html)
    if not m:
        m = re.search(r'([\d]{4}-[\d]{2}-[\d]{2})', html)
    if m:
        d = m.group(1).replace("/", "-")
        result["date"] = d
    # 文号
    m = re.search(r'([\u4e00-\u9fa5]{2,10}〔[\d]{4}〕[\d]+号)', html)
    if m:
        result["doc_no"] = m.group(1)
    return result

async def nav(page, url, wait_ms=4000, max_retry=3):
    """导航并等待渲染（瑞数挑战由 JS 渲染，必须 page.content()）"""
    for attempt in range(max_retry):
        try:
            await page.goto(url, wait_until="domcontentloaded", timeout=35000)
        except Exception as e:
            log(f"nav error {url.split('/')[-1][:40]}: {str(e)[:60]}")
            await page.wait_for_timeout(3000)
            continue
        await page.wait_for_timeout(wait_ms)
        html = await page.content()
        if len(html) > 20000:
            return html
        await page.wait_for_timeout(3000)
        html = await page.content()
        if len(html) > 20000:
            return html
        await page.wait_for_timeout(3000)
    return ""

async def crawl(pages: int, incremental: bool, output: str = None, start: int = 1):
    conn = None
    if not output:
        conn = sqlite3.connect(DB, timeout=30)
        cur = conn.cursor()
    out_f = None
    if output:
        out_f = open(output, "w", encoding="utf-8")
    new_count = 0
    skip_count = 0
    fail_count = 0
    t0 = time.time()

    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            args=["--disable-blink-features=AutomationControlled",
                  f"--host-resolver-rules=MAP {HOST} {IP}"]
        )
        ctx = await browser.new_context(user_agent=UA, viewport={"width": 1366, "height": 900}, locale="zh-CN")
        page = await ctx.new_page()

        # 1. 主页面过瑞数（重试直到成功）
        main_html = ""
        for attempt in range(6):
            await page.goto(MAIN_URL, wait_until="domcontentloaded", timeout=45000)
            await page.wait_for_timeout(6000)
            main_html = await page.content()
            if len(main_html) > 20000:
                log(f"瑞数通过 (attempt {attempt+1})")
                break
            log(f"瑞数挑战重试 {attempt+1}/6...")
        if len(main_html) <= 20000:
            log("主页面挑战失败，退出")
            return

        # 2. 列表页（start 偏移）
        for pg in range(start, start + pages):
            if pg == 1:
                list_url = f"{LIST_BASE}/index.html"
            else:
                list_url = f"{LIST_BASE}/index_{pg-1}.html"
            html = await nav(page, list_url)
            if not html:
                log(f"页{pg} 挑战失败，跳过")
                fail_count += 1
                continue
            urls = extract_list_items(html)
            if not urls:
                log(f"页{pg} 无条目（可能到尾页），停止")
                break
            log(f"页{pg}: {len(urls)} 条")
            for url in urls:
                # 去重检查
                if conn:
                    cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
                    if cur.fetchone():
                        skip_count += 1
                        continue
                # 详情页
                dhtml = await nav(page, url)
                if not dhtml:
                    log(f"  详情失败: {url.split('/')[-1]}")
                    fail_count += 1
                    continue
                det = extract_detail(dhtml)
                if not det["title"] or not det["content"]:
                    # 微信外链等无详情: 标题+URL
                    if det["title"]:
                        det["content"] = f'<p><a href="{url}">{det["title"]}</a></p>'
                    else:
                        log(f"  无标题无正文: {url.split('/')[-1]}")
                        fail_count += 1
                        continue
                content_html = det["content"]
                rec = {
                    "site_name": SITE_NAME, "source_url": list_url, "page_url": url,
                    "title": det["title"], "publish_date": det["date"] or "1900-01-01",
                    "summary": det["content"][:200].replace("<", "").replace(">", ""),
                    "status": "published", "category": "部门文件", "content": content_html,
                    "tags": det["doc_no"], "attachments": det["attachments"],
                    "group_name": "内蒙古", "has_table": 1 if det["has_table"] else 0,
                    "script_name": SCRIPT_NAME,
                }
                if out_f:
                    out_f.write(json.dumps(rec, ensure_ascii=False) + "\n")
                    out_f.flush()
                    new_count += 1
                else:
                    cur.execute("""
                        INSERT OR IGNORE INTO gov_raw
                        (site_name, source_url, page_url, title, publish_date, date_rank, summary, status,
                         category, visits, content, tags, industry, attachments, group_name, has_table, script_name)
                        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                    """, (
                        SITE_NAME, list_url, url, det["title"], det["date"] or "1900-01-01",
                        det["date"] or "1900-01-01", det["content"][:200].replace("<", "").replace(">", ""),
                        "published", "部门文件", 0, content_html,
                        det["doc_no"], "", json.dumps(det["attachments"], ensure_ascii=False),
                        "内蒙古", 1 if det["has_table"] else 0, SCRIPT_NAME
                    ))
                    if cur.rowcount > 0:
                        new_count += 1
                    else:
                        skip_count += 1
                    conn.commit()
                if new_count % 10 == 0 and new_count > 0:
                    log(f"  进度: 新增{new_count} 跳过{skip_count} 失败{fail_count} ({time.time()-t0:.0f}s)")

        await browser.close()

    if conn:
        conn.close()
    if out_f:
        out_f.close()
    log(f"完成: 新增{new_count} 跳过{skip_count} 失败{fail_count} 用时{time.time()-t0:.0f}s")

if __name__ == "__main__":
    pages, incremental, output, start = parse_args()
    log(f"pages={pages} start={start} incremental={incremental} output={output}")
    asyncio.run(crawl(pages, incremental, output, start))