#!/usr/bin/env python3
"""Crawler for 清水河县人民政府 - 通知公告 (qingshuihe.gov.cn zwgk tzgg)
   瑞数4代 WAF: playwright 渲染 (page.content() 拿渲染后 DOM)
   列表: zwgk/tzgg/index.html ~ index_7.html (15条/页, 8页 ~120条)
   详情: /zwgk/tzgg/YYYYMM/t*.html, div#Zoom 正文, <title> 标题, 发布日期
   参考: crawl_qingshuihe_bmwj.py (同站瑞数4代攻破模板)
"""
import asyncio, re, sys, sqlite3, json, os, time
from playwright.async_api import async_playwright

HOST = "www.qingshuihe.gov.cn"
IP = "139.9.247.160"
BASE = f"http://{HOST}"
LIST_BASE = f"{BASE}/zwgk/tzgg"
MAIN_URL = f"{BASE}/"
DB = "/root/search.db"
SITE_NAME = "清水河县人民政府-通知公告"
SCRIPT_NAME = "crawl_qingshuihe_tzgg.py"
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
CUTOFF = "2023-08-18"  # 与全库一致


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def parse_args():
    pages = 3
    incremental = True
    output = None
    start = 1
    for i, a in enumerate(sys.argv):
        if a == "--pages" and i + 1 < len(sys.argv):
            pages = int(sys.argv[i + 1])
        elif a == "--incremental":
            incremental = True
        elif a == "--output" and i + 1 < len(sys.argv):
            output = sys.argv[i + 1]
        elif a == "--start" and i + 1 < len(sys.argv):
            start = int(sys.argv[i + 1])
        elif a.startswith("--output="):
            output = a.split("=", 1)[1]
        elif a.startswith("--pages="):
            pages = int(a.split("=", 1)[1])
        elif a.startswith("--start="):
            start = int(a.split("=", 1)[1])
    return pages, incremental, output, start


def extract_list_items(html):
    """从列表页提取 (url, 标题)。只匹配 tzgg 详情页链接"""
    items = []
    seen = set()
    for m in re.finditer(r'<a[^>]*href="(http://www\.qingshuihe\.gov\.cn/zwgk/tzgg/\d{6}/t\d+_\d+\.html)"[^>]*>', html):
        url = m.group(1)
        if url not in seen:
            seen.add(url)
            items.append(url)
    return items


def extract_detail(html):
    """提取详情: title, content, date, attachments, has_table"""
    result = {"title": "", "content": "", "date": "", "attachments": [], "has_table": False}
    # 标题: <title>xxx_ 清水河县人民政府</title>
    m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
    if m:
        t = re.sub(r'[\s_]*清水河县人民政府[\s_]*$', '', m.group(1)).strip()
        t = re.sub(r'_*\s*$', '', t).strip()
        if t:
            result["title"] = t
    # 附件: 整页扫描
    for am in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>([^<]{2,80})</a>', html):
        href, txt = am.group(1), am.group(2).strip()
        if any(k in txt for k in ["附件", "下载", "文件"]) or re.search(r'\.(docx?|xlsx?|pdf|zip|rar|wps|et)$', href, re.I) or "download" in href.lower():
            full = href if href.startswith("http") else (BASE + href if href.startswith("/") else BASE + "/" + href)
            full = full.replace("&amp;", "&")
            if (txt, full) not in [(a["name"], a["url"]) for a in result["attachments"]]:
                result["attachments"].append({"name": txt, "url": full})
    # 正文容器
    for pattern in [r'<div[^>]*id="Zoom"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="trs_editor_view"[^>]*>(.*?)</div>',
                    r'<div[^>]*class="TRS_Editor"[^>]*>(.*?)</div>']:
        m = re.search(pattern, html, re.DOTALL)
        if m:
            inner = m.group(1)
            result["has_table"] = bool(re.search(r'<table', inner))
            inner = re.sub(r'<script[^>]*>.*?</script>', '', inner, flags=re.DOTALL)
            inner = re.sub(r'<style[^>]*>.*?</style>', '', inner, flags=re.DOTALL)
            result["content"] = inner.strip()
            break
    # 日期: 发布日期
    m = re.search(r'发布[日日期期]*[：:]\s*([\d]{4}[-/][\d]{1,2}[-/][\d]{1,2})', html)
    if not m:
        m = re.search(r'([\d]{4}-[\d]{2}-[\d]{2})', html)
    if m:
        result["date"] = m.group(1).replace("/", "-")
    return result


async def nav(page, url, wait_ms=4000, max_retry=3):
    for attempt in range(max_retry):
        try:
            await page.goto(url, wait_until="domcontentloaded", timeout=35000)
        except Exception as e:
            log(f"nav error {url.split('/')[-1][:40]}: {str(e)[:60]}")
            await page.wait_for_timeout(3000)
            continue
        await page.wait_for_timeout(wait_ms)
        html = await page.content()
        if len(html) > 20000:
            return html
        await page.wait_for_timeout(3000)
        html = await page.content()
        if len(html) > 20000:
            return html
        await page.wait_for_timeout(3000)
    return ""


async def crawl(pages: int, incremental: bool, output: str = None, start: int = 1):
    conn = None
    if not output:
        conn = sqlite3.connect(DB, timeout=30)
        cur = conn.cursor()
    out_f = None
    if output:
        out_f = open(output, "w", encoding="utf-8")
    new_count = 0
    skip_count = 0
    fail_count = 0
    t0 = time.time()

    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            args=["--disable-blink-features=AutomationControlled",
                  f"--host-resolver-rules=MAP {HOST} {IP}"]
        )
        ctx = await browser.new_context(user_agent=UA, viewport={"width": 1366, "height": 900}, locale="zh-CN")
        page = await ctx.new_page()

        # 1. 主页面过瑞数
        main_html = ""
        for attempt in range(6):
            await page.goto(MAIN_URL, wait_until="domcontentloaded", timeout=45000)
            await page.wait_for_timeout(6000)
            main_html = await page.content()
            if len(main_html) > 20000:
                log(f"瑞数通过 (attempt {attempt+1})")
                break
            log(f"瑞数挑战重试 {attempt+1}/6...")
        if len(main_html) <= 20000:
            log("主页面挑战失败，退出")
            return

        # 2. 列表页
        for pg in range(start, start + pages):
            if pg == 1:
                list_url = f"{LIST_BASE}/index.html"
            else:
                list_url = f"{LIST_BASE}/index_{pg-1}.html"
            html = await nav(page, list_url)
            if not html:
                log(f"页{pg} 挑战失败，跳过")
                fail_count += 1
                continue
            urls = extract_list_items(html)
            if not urls:
                log(f"页{pg} 无条目（可能到尾页），停止")
                break
            log(f"页{pg}: {len(urls)} 条")
            for url in urls:
                if conn:
                    cur.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
                    if cur.fetchone():
                        skip_count += 1
                        continue
                dhtml = await nav(page, url)
                if not dhtml:
                    log(f"  详情失败: {url.split('/')[-1]}")
                    fail_count += 1
                    continue
                det = extract_detail(dhtml)
                if not det["title"]:
                    log(f"  无标题: {url.split('/')[-1]}")
                    fail_count += 1
                    continue
                if not det["content"]:
                    det["content"] = f'<p><a href="{url}">{det["title"]}</a></p>'
                content_html = det["content"]
                rec = {
                    "site_name": SITE_NAME, "source_url": list_url, "page_url": url,
                    "title": det["title"], "publish_date": det["date"] or "1900-01-01",
                    "summary": det["content"][:200].replace("<", "").replace(">", ""),
                    "status": "published", "category": "通知公告", "content": content_html,
                    "tags": "", "attachments": det["attachments"],
                    "group_name": "内蒙古", "has_table": 1 if det["has_table"] else 0,
                    "script_name": SCRIPT_NAME,
                }
                if out_f:
                    out_f.write(json.dumps(rec, ensure_ascii=False) + "\n")
                    out_f.flush()
                    new_count += 1
                else:
                    cur.execute("""
                        INSERT OR IGNORE INTO gov_raw
                        (site_name, source_url, page_url, title, publish_date, date_rank, summary, status,
                         category, visits, content, tags, industry, attachments, group_name, has_table, script_name)
                        VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
                    """, (
                        SITE_NAME, list_url, url, det["title"], det["date"] or "1900-01-01",
                        det["date"] or "1900-01-01", det["content"][:200].replace("<", "").replace(">", ""),
                        "published", "通知公告", 0, content_html,
                        "", "", json.dumps(det["attachments"], ensure_ascii=False),
                        "内蒙古", 1 if det["has_table"] else 0, SCRIPT_NAME
                    ))
                    if cur.rowcount > 0:
                        new_count += 1
                    else:
                        skip_count += 1
                    conn.commit()
                if new_count % 10 == 0 and new_count > 0:
                    log(f"  进度: 新增{new_count} 跳过{skip_count} 失败{fail_count} ({time.time()-t0:.0f}s)")

        await browser.close()

    if conn:
        conn.close()
    if out_f:
        out_f.close()
    log(f"完成: 新增{new_count} 跳过{skip_count} 失败{fail_count} 用时{time.time()-t0:.0f}s")


if __name__ == "__main__":
    pages, incremental, output, start = parse_args()
    log(f"pages={pages} start={start} incremental={incremental} output={output}")
    asyncio.run(crawl(pages, incremental, output, start))