#!/usr/bin/env python3
import os
"""
crawl_tangyin.py — 汤阴县人民政府·行政许可（生态环境）
=====================================================
WAF: wzws-waf-cgi JS挑战 — 需用 Playwright (--disable-blink-features)
静态HTML分页 index_{n}.html
详情正文 <div class="content">
列表标题取自 <a title="...">

用法:
    python3 crawl_tangyin.py             # 全量（5页+近3年）
    python3 crawl_tangyin.py 1           # 增量（只爬首页）
"""

import re, sys, os, time, json

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "汤阴县行政许可"
BASE_URL = "https://www.tangyin.gov.cn"
LIST_URL = f"{BASE_URL}/xxgk/zdlygk/sthj/xzxk/"


def norm_date(d):
    if not d:
        return ""
    m = re.match(r"(\d{4})\.(\d{1,2})\.(\d{1,2})", str(d))
    if m:
        return f"{int(m.group(1)):04d}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    m = re.match(r"(\d{4})-(\d{1,2})-(\d{1,2})", str(d))
    if m:
        return f"{int(m.group(1)):04d}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    return str(d)[:10]


def main():
    incremental = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    mode = "增量" if incremental else "全量"
    print(f"🔍 {SITE_NAME} [{mode}]", flush=True)

    # 检测是否需 Playwright（有WAF的服务器环境）
    need_playwright = True

    t0 = time.time()
    all_items = []
    max_pages = 1 if incremental else 5

    if need_playwright:
        print("  启动 Playwright ...")
        from playwright.sync_api import sync_playwright
        with sync_playwright() as p:
            browser = p.chromium.launch(
                headless=True,
                args=["--disable-blink-features=AutomationControlled"],
            )
            ctx = browser.new_context()
            page = ctx.new_page()
            page.set_extra_http_headers({"Accept-Language": "zh-CN,zh;q=0.9"})

            for pg in range(1, max_pages + 1):
                url = LIST_URL if pg == 1 else f"{LIST_URL}index_{pg}.html"
                print(f"  📄 第{pg}页: {url}")
                try:
                    page.goto(url, timeout=30000)
                    time.sleep(3)  # 等WAF挑战通过
                    html = page.content()
                except Exception as e:
                    print(f"    ❌ 请求失败: {e}")
                    break

                items = []
                pattern = re.compile(
                    r'<li\s+class="cl">.*?<span\s+class="date">([^<]+)</span>\s*<a\s+href="([^"]+)"[^>]*title="([^"]*)"[^>]*>.*?</a>\s*</li>',
                    re.DOTALL,
                )
                for m in pattern.finditer(html):
                    date_str = m.group(1).strip()
                    link = m.group(2).strip()
                    title = (m.group(3) or "").strip()
                    if not link.startswith("http"):
                        link = BASE_URL + link if link.startswith("/") else f"{BASE_URL}/{link}"
                    pub_date = norm_date(date_str)
                    if pub_date and pub_date < "2023-06":
                        continue
                    items.append({"title": title, "url": link, "pub_date": pub_date, "content": None, "source": SITE_NAME})

                print(f"    → {len(items)}条")
                all_items.extend(items)

            # 爬详情 — 用同一个 page 实例
            print(f"\n  爬取详情页 ({len(all_items)}条)...")
            for i, it in enumerate(all_items):
                try:
                    page.goto(it["url"], timeout=30000)
                    time.sleep(2)  # 等WAF验证
                    html = page.content()
                except Exception as e:
                    print(f"\n    ❌ 详情页请求失败: {it['url'][:60]} ({e})")
                    it["content"] = ""
                    continue

                m = re.search(r'class="content"[^>]*>(.*?)</div>\s*<!--\s*foot', html, re.DOTALL)
                content = m.group(1).strip() if m else ""
                content = re.sub(r'\sdata-mce-[a-zA-Z-]+="[^"]*"', "", content)
                it["content"] = content

                # 使用统一 site_name，不从页面提取
                it["source"] = SITE_NAME

                if (i + 1) % 5 == 0:
                    print(f"    ... {i+1}/{len(all_items)}", end="\r")

            browser.close()

    elapsed = time.time() - t0
    print(f"\n📊 共获取 {len(all_items)} 条，耗时 {elapsed:.0f}s")

    # 入库
    import sqlite3
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")

    ok, skip = 0, 0
    for it in all_items:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                (
                    it.get("source", SITE_NAME)[:200], it["url"], it["url"],
                    (it["title"] or "")[:500], it.get("pub_date", ""),
                    (it["title"] or "")[:500], it.get("content", ""),
                    "active", "行政许可", "",
                ),
            )
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except Exception as e:
            skip += 1

    db.commit()
    db.execute(
        "INSERT INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,),
    )
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}")


if __name__ == "__main__":
    main()