#!/usr/bin/env python3
"""
crawl_rongcheng_gggs.py - 荣成市人民政府-公告公示 (col61226, number=RCA21)
站点: www.rongcheng.gov.cn (TRS 系, 无 WAF, curl 可直连列表页但 search.jsp 需浏览器上下文)
列表: 页面 div56364 由 loadDynamic('/module/xxgk/search.jsp?infotypeId=RCA21...') 渲染
  翻页: 调用页面全局 funGoPage('/module/xxgk/search.jsp', N) (AJAX, 不刷新页面)
  curl 直连 search.jsp 全部被拒("请传入正确参数") → 必须 playwright 页面内操作
  每页 20 条, 共 7727 条/387 页
详情: /art/YYYY/M/D/art_61226_XXXX.html
  meta ArticleTitle/PubDate, 正文 <meta name="ContentStart"> 到 <meta name="ContentEnd">, 容器 div.xqwz
"""
import sys, os, re, asyncio
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE = "http://www.rongcheng.gov.cn"
SITE_NAME = "荣成市-公告公示"
LIST_URL = f"{BASE}/col/col61226/index.html?number=RCA21"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"


async def run_async(pages=5):
    from playwright.async_api import async_playwright
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            executable_path='/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome',
            args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled']
        )
        ctx = await browser.new_context(user_agent=UA, locale='zh-CN',
                                        viewport={'width': 1440, 'height': 900}, timezone_id='Asia/Shanghai')
        page = await ctx.new_page()

        # 打开列表页, 等首屏渲染
        try:
            resp = await page.goto(LIST_URL, wait_until='domcontentloaded', timeout=40000)
            await page.wait_for_timeout(8000)
            print(f"  [LOAD] status={resp.status if resp else '?'}")
        except Exception as e:
            print(f"  [LOAD ERR] {str(e)[:120]}")
            await browser.close()
            return 0

        # 收集列表 (翻页)
        items_all = []
        seen = set()
        for pg in range(1, pages + 1):
            if pg > 1:
                try:
                    await page.evaluate(f"funGoPage('{BASE}/module/xxgk/search.jsp', {pg})")
                    await page.wait_for_timeout(6000)
                except Exception as e:
                    print(f"  [PAGE ERR p{pg}] {str(e)[:100]}")
                    break
            rows = await page.evaluate("""() => {
                const div = document.getElementById('div56364');
                if (!div) return [];
                const out = [];
                const seenHref = new Set();
                for (const tr of div.querySelectorAll('tr')) {
                    const a = tr.querySelector('a[href*="/art/"]');
                    if (!a) continue;
                    const href = a.getAttribute('href');
                    if (!href || seenHref.has(href)) continue;
                    seenHref.add(href);
                    const txt = tr.textContent.replace(/\\s+/g, ' ').trim();
                    const dm = txt.match(/(\\d{4})-(\\d{2})-(\\d{2})/);
                    out.push({
                        href: href,
                        title: a.textContent.trim(),
                        date: dm ? dm[0] : ''
                    });
                }
                return out;
            }""")
            new_rows = 0
            for r in rows:
                url = r['href'] if r['href'].startswith('http') else BASE + r['href']
                if url not in seen:
                    seen.add(url)
                    items_all.append({'url': url, 'title': r['title'], 'date': r['date']})
                    new_rows += 1
            print(f"  第{pg}页: 读取 {len(rows)} 行, 新增 {new_rows} (累计 {len(items_all)})")
            if len(rows) < 20:
                print("  [END] 列表结束")
                break

        # 详情页
        records = []
        for idx, it in enumerate(items_all):
            print(f"  [{idx+1}/{len(items_all)}] {it['title'][:38]}...")
            try:
                resp = await page.goto(it['url'], wait_until='domcontentloaded', timeout=30000)
                await page.wait_for_timeout(2500)
                html = await page.content()
            except Exception as e:
                print(f"    [DETAIL ERR] {str(e)[:100]}")
                continue
            # meta 标题
            title = it['title']
            m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
            if m and m.group(1).strip():
                title = m.group(1).strip()
            # 日期
            pub_date = it['date']
            m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', html)
            if m:
                dm = re.search(r'\d{4}-\d{2}-\d{2}', m.group(1))
                if dm:
                    pub_date = dm.group()
            # 正文 ContentStart -> ContentEnd
            content = ""
            si = html.find('<meta name="ContentStart">')
            ei = html.find('<meta name="ContentEnd">')
            if si > 0 and ei > si:
                content = html[si + len('<meta name="ContentStart">'):ei].strip()
            if not content:
                # 兜底: xqwz 容器
                m = re.search(r'<div class="xqwz">([\s\S]*?)(?:<div class="xqfx"|</body>)', html)
                if m:
                    seg = m.group(1)
                    si2 = seg.find('<!--<$[信息内容]>begin-->')
                    if si2 > 0:
                        content = seg[si2:].strip()
            if not content.strip():
                print("    [SKIP] 正文空")
                continue
            # 链接绝对化
            content = re.sub(r'href="/', 'href="' + BASE + '/', content)
            content = re.sub(r'src="/', 'src="' + BASE + '/', content)
            records.append({
                "title": title,
                "url": it['url'],
                "pub_date": pub_date,
                "site_name": SITE_NAME,
                "content": content,
                "summary": "",
            })
        await browser.close()

    print(f"  共 {len(records)} 条有效")
    if records:
        push_to_searchdb(records, "rongcheng_gggs")
    return len(records)


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=5)
    parser.add_argument("--all", action="store_true")
    args = parser.parse_args()
    pages = 50 if args.all else args.pages
    print(f"  {SITE_NAME} pages={pages}")
    cnt = asyncio.run(run_async(pages=pages))
    print(f"Done: {cnt} records")
    return 0


if __name__ == "__main__":
    sys.exit(main())