#!/usr/bin/env python3
"""
crawl_gddrc_jnjc.py - 广东省能源局-固定资产投资项目节能审查结果 (drc.gd.gov.cn/gdsnyj)
站点: TLS 指纹拦截 (curl Empty reply), playwright 真实浏览器可过, 偶发 goto 超时需重试
列表: GET /gdsnyj/gkmlpt/api/all/3869?page={N}&sid=257  (JSON)
  {classify:{post_count}, articles:[{id, title, date, document_number, publisher, url}]}
  栏目 3869 = 固定资产投资项目节能审查结果, total=900
详情: /gdsnyj/gkmlpt/content/{a}/{b}/post_{id}.html (API url 字段)
  meta ArticleTitle/PubDate/ContentSource, 正文 div.article-content
内容: 广东省能源局节能报告审查意见 (粤能许可〔2026〕XX号, 含项目投资/用能工艺)
"""
import sys, os, re, asyncio, json, random, time
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE = "https://drc.gd.gov.cn"
COL_ID = "3869"
SITE_ID = "257"
SITE_NAME = "广东省能源局-固定资产投资项目节能审查结果"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"


async def goto_with_retry(page, url, timeout=30000, retries=2):
    for attempt in range(retries):
        try:
            # asyncio.wait_for 硬超时: 某些 TLS 拦截场景 goto 挂起不抛异常
            resp = await asyncio.wait_for(
                page.goto(url, wait_until='domcontentloaded', timeout=timeout),
                timeout=timeout / 1000 + 15
            )
            await page.wait_for_timeout(5000)
            return resp
        except Exception:
            if attempt < retries - 1:
                await page.wait_for_timeout(10000)
            else:
                raise


async def run_async(pages=5):
    from playwright.async_api import async_playwright
    from playwright_stealth import Stealth
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            executable_path='/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome',
            args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled']
        )
        ctx = await browser.new_context(user_agent=UA, locale='zh-CN',
                                        viewport={'width': 1440, 'height': 900}, timezone_id='Asia/Shanghai')
        stealth = Stealth()
        await stealth.apply_stealth_async(ctx)
        page = await ctx.new_page()

        # 过盾: 栏目首页
        print("  [NAV] 打开栏目首页(过盾)…", flush=True)
        try:
            resp = await goto_with_retry(page, f"{BASE}/gdsnyj/gkmlpt/index#{COL_ID}")
            print(f"  [LOAD] status={resp.status if resp else '?'} title={(await page.title())[:50]}", flush=True)
        except Exception as e:
            print(f"  [LOAD ERR] {str(e)[:120]}")
            await browser.close()
            return 0

        # API 拉列表
        items_all = []
        seen = set()
        total = 0
        for pg in range(1, pages + 1):
            api = f"{BASE}/gdsnyj/gkmlpt/api/all/{COL_ID}?page={pg}&sid={SITE_ID}"
            data = await page.evaluate(f"""async () => {{
                try {{
                    const r = await fetch('{api}');
                    return await r.json();
                }} catch(e) {{ return {{error: e.message}}; }}
            }}""")
            if 'error' in data:
                print(f"  [API ERR] {data['error']}")
                break
            total = (data.get('classify') or {}).get('post_count', 0)
            arts = data.get('articles', []) or []
            if not arts:
                break
            print(f"  第{pg}页: {len(arts)} 条 (total={total})")
            for it in arts:
                url = it.get('url', '')
                if url and url not in seen:
                    seen.add(url)
                    title = it.get('title', '').strip()
                    items_all.append({
                        'url': url,
                        'title': title,
                        'date': str(it.get('date', '')),
                        'doc_no': it.get('document_number', ''),
                        'publisher': it.get('publisher', ''),
                    })
            if pg * len(arts) >= total:
                break

        print(f"  [LIST] total={total}, 去重后 {len(items_all)} 条", flush=True)

        # 增量过滤: 已入库 URL 不再重抓详情（本站详情抓取极慢）
        try:
            import sqlite3 as _sq
            _db = _sq.connect(os.getenv("SEARCH_DB", "/root/search.db"), timeout=60)
            _ex = set(r[0] for r in _db.execute(
                "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchall())
            _db.close()
        except Exception as e:
            print(f"  [WARN] 读取已入库URL失败: {e}", flush=True)
            _ex = set()
        _before = len(items_all)
        items_all = [it for it in items_all if it['url'] not in _ex]
        print(f"  [DB] 已入库 {len(_ex)} 条，本次待抓详情 {len(items_all)}/{_before}", flush=True)

        # 详情页
        records = []
        for idx, it in enumerate(items_all):
            print(f"  [{idx+1}/{len(items_all)}] {it['title'][:40]}...")
            try:
                resp = await goto_with_retry(page, it['url'], timeout=50000, retries=2)
                dh = await page.content()
            except Exception as e:
                print(f"    [DETAIL ERR] {str(e)[:100]} 降级为标题记录")
                records.append({
                    "site_name": SITE_NAME, "title": it['title'], "pub_date": it['date'],
                    "content": f"<p>{it['title']}</p>",
                    "source_url": it['url'], "url": it['url'],
                })
                continue
            title = it['title']
            m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', dh)
            if m and m.group(1).strip():
                title = m.group(1).strip()
            pub_date = it['date']
            m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', dh)
            if m:
                dm = re.search(r'\d{4}-\d{2}-\d{2}', m.group(1))
                if dm:
                    pub_date = dm.group()
            content = ""
            m = re.search(r'<div class="article-content"[^>]*>([\s\S]*?)</div>', dh)
            if m:
                content = m.group(1).strip()
            if not content.strip():
                print("    [SKIP] 正文空")
                continue
            content = re.sub(r'href="/', 'href="' + BASE + '/', content)
            content = re.sub(r'src="/', 'src="' + BASE + '/', content)
            records.append({
                "title": title,
                "url": it['url'],
                "pub_date": pub_date,
                "site_name": SITE_NAME,
                "content": content,
                "summary": "",
            })
            if idx < len(items_all) - 1:
                await asyncio.sleep(random.uniform(1.5, 4.0))
        await browser.close()

    print(f"  共 {len(records)} 条有效")
    if records:
        push_to_searchdb(records, "gddrc_jnjc")
    return len(records)


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=5)
    args = parser.parse_args()
    n = asyncio.run(run_async(pages=args.pages))
    print(f"Done: {n} records")
    return 0


if __name__ == "__main__":
    sys.exit(main())