#!/usr/bin/env python3
"""
crawl_gdqy_hpgs.py - 清远市生态环境局-建设项目环境影响评价信息
站点: www.gdqy.gov.cn (瑞数4代 + 帝联NWAF ctct_bundle 双WAF, 用户称"天翼云人机验证")
过盾: playwright + playwright-stealth (ahshx 同款方案)
列表: 静态分页 index_N.html (服务端渲染, 页面内 fetch 200)
  容器 ul.infoList > li > h4 > a(标题) + span.time(日期), 472条/20页/24页
详情: content/post_XXX.html (页面内 fetch 200, goto 会超时!)
  meta ArticleTitle/PubDate/ContentSource, 正文 div.article-content article-content-body
限速: 详情间 random 3-8s 随机抖动 (用户要求), 单浏览器会话控制请求量
"""
import sys, os, re, asyncio, json, random
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE = "https://www.gdqy.gov.cn"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"

# 栏目: dir 相对路径 + site_name
COL_MAP = {
    "zdly":        {"dir": "/xxgk/zzjg/zfjg/qyssthjj/xxgk/zdlyxxgkzl", "name": "清远市生态环境局-重点领域信息公开"},
    "jsxmhjyxpjxx": {"dir": "/xxgk/zzjg/zfjg/qyssthjj/xxgk/zdlyxxgkzl/jsxmhjyxpjxx", "name": "清远市生态环境局-建设项目环评信息"},
    "hjbh":        {"dir": "/xxgk/zzjg/zfjg/qyssthjj/xxgk/zdlyxxgkzl/hjbh", "name": "清远市生态环境局-环境保护信息公开"},
    "ggfwsx":      {"dir": "/xxgk/zzjg/zfjg/qyssthjj/xxgk/zdlyxxgkzl/ggfwsx", "name": "清远市生态环境局-公共服务事项"},
    "szhjxx":      {"dir": "/xxgk/zzjg/zfjg/qyssthjj/xxgk/zdlyxxgkzl/szhjxx", "name": "清远市生态环境局-水质环境信息"},
}
LIST_DIR = COL_MAP["jsxmhjyxpjxx"]["dir"]  # 默认(兼容旧配置)
SITE_NAME = COL_MAP["jsxmhjyxpjxx"]["name"]


async def run_async(col_key="jsxmhjyxpjxx", pages=5):
    col = COL_MAP[col_key]
    list_dir = col["dir"]
    site_name = col["name"]
    from playwright.async_api import async_playwright
    from playwright_stealth import Stealth
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            executable_path='/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome',
            args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled']
        )
        ctx = await browser.new_context(user_agent=UA, locale='zh-CN',
                                        viewport={'width': 1440, 'height': 900}, timezone_id='Asia/Shanghai')
        stealth = Stealth()
        await stealth.apply_stealth_async(ctx)
        page = await ctx.new_page()

        # 过盾: 访问列表页
        try:
            resp = await page.goto(BASE + list_dir + '/', wait_until='domcontentloaded', timeout=45000)
            await page.wait_for_timeout(15000)
            html = await page.content()
            m = re.search(r'<title>([^<]*)</title>', html)
            print(f"  [LOAD] status={resp.status if resp else '?'} title={m.group(1) if m else '?'} len={len(html)}")
            if m and '请稍候' in m.group(1):
                print("  [WAF] 挑战未过")
                await browser.close()
                return 0
        except Exception as e:
            print(f"  [LOAD ERR] {str(e)[:120]}")
            await browser.close()
            return 0

        # 列表: 页面内 fetch index_N.html
        items_all = []
        seen = set()
        for pg in range(1, pages + 1):
            if pg == 1:
                list_html = html
            else:
                try:
                    list_html = await page.evaluate(f"""async () => {{
                        const r = await fetch('{BASE}{list_dir}/index_{pg}.html', {{headers: {{'X-Requested-With': 'XMLHttpRequest'}}}});
                        return await r.text();
                    }}""")
                except Exception as e:
                    print(f"  [LIST ERR p{pg}] {str(e)[:100]}")
                    break
            # ul.infoList > li > h4 > a + span.time
            rows = re.findall(r'<li>\s*<h4><a href="([^"]*post_\d+\.html)"[^>]*>([\s\S]{0,200}?)</a><span class="time">(\d{4}-\d{2}-\d{2})</span>', list_html)
            if not rows:
                # 兜底宽松匹配
                rows = re.findall(r'<a href="([^"]*post_\d+\.html)"[^>]*>([\s\S]{0,200}?)</a><span[^>]*>(\d{4}-\d{2}-\d{2})</span>', list_html)
            new_rows = 0
            for u, t, d in rows:
                title = re.sub(r'<[^>]+>', '', t).strip()
                url = u if u.startswith('http') else BASE + u
                if url not in seen:
                    seen.add(url)
                    items_all.append({"url": url, "title": title, "date": d})
                    new_rows += 1
            print(f"  第{pg}页: 解析 {len(rows)} 条, 新增 {new_rows} (累计 {len(items_all)})")
            if len(rows) < 20:
                print("  [END] 列表结束")
                break

        # 详情页 (3-8s 随机抖动)
        records = []
        for idx, it in enumerate(items_all):
            print(f"  [{idx+1}/{len(items_all)}] {it['title'][:38]}...")
            try:
                res = await page.evaluate(f"""async () => {{
                    const r = await fetch('{it['url']}', {{headers: {{'X-Requested-With': 'XMLHttpRequest'}}}});
                    return {{status: r.status, html: await r.text()}};
                }}""")
                dh = res['html']
                if res['status'] != 200 or '请稍候' in dh[:2000]:
                    print(f"    [FETCH WARN] status={res['status']}")
                    continue
            except Exception as e:
                print(f"    [DETAIL ERR] {str(e)[:100]}")
                continue
            # meta
            title = it['title']
            m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', dh)
            if m and m.group(1).strip():
                title = m.group(1).strip()
            pub_date = it['date']
            m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', dh)
            if m:
                dm = re.search(r'\d{4}-\d{2}-\d{2}', m.group(1))
                if dm:
                    pub_date = dm.group()
            # 正文 div.article-content
            content = ""
            m = re.search(r'<div[^>]*class="article-content[^"]*"[^>]*>([\s\S]*?)</div>\s*(?:<div[^>]*class="article-share"|</div>\s*</div>|<!--)', dh)
            if m:
                content = m.group(1).strip()
            if not content:
                m = re.search(r'<div[^>]*id="zoomcon"[^>]*>([\s\S]*?)</div>', dh)
                if m:
                    content = m.group(1).strip()
            if not content.strip():
                print("    [SKIP] 正文空")
                continue
            content = re.sub(r'href="/', 'href="' + BASE + '/', content)
            content = re.sub(r'src="/', 'src="' + BASE + '/', content)
            records.append({
                "title": title,
                "url": it['url'],
                "pub_date": pub_date,
                "site_name": site_name,
                "content": content,
                "summary": "",
            })
            # 3-8s 随机抖动 (用户要求)
            if idx < len(items_all) - 1:
                wait = random.uniform(3.0, 8.0)
                await asyncio.sleep(wait)
        await browser.close()

    print(f"  共 {len(records)} 条有效")
    if records:
        push_to_searchdb(records, "gdqy_hpgs")
    return len(records)


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--col", default="jsxmhjyxpjxx", help="栏目键: " + ",".join(COL_MAP.keys()))
    parser.add_argument("--pages", type=int, default=5)
    parser.add_argument("--all", action="store_true")
    args = parser.parse_args()
    if args.col not in COL_MAP:
        print(f"[ERR] 未知栏目 {args.col}, 可选: {list(COL_MAP.keys())}")
        return 1
    col = COL_MAP[args.col]
    pages = 24 if args.all else args.pages
    print(f"  {col['name']} pages={pages}")
    cnt = asyncio.run(run_async(col_key=args.col, pages=pages))
    print(f"Done: {cnt} records")
    return 0


if __name__ == "__main__":
    sys.exit(main())