#!/usr/bin/env python3
"""
crawl_ahtxq_tzgg.py - 屯溪区-通知公告 (www.ahtxq.gov.cn)
站点: 帝联NWAF + 瑞数4代双WAF (与 ahshx/黄山 同款)
过盾: playwright + playwright-stealth
列表: 首页 /zxzx/tzgg/index.html (10条/页) + 翻页 API GET /content/column/6793870?pageIndex=N
  返回 HTML 片段 (li 列表), 页面内 fetch 即可
  列表项: <li><span class="right date">YYYY-MM-DD</span><a href="/zxzx/tzgg/{id}.html">标题</a></li>
详情: /zxzx/tzgg/{id}.html, meta ArticleTitle/PubDate/ContentSource, 正文 div.wzcon.j-fontContent
  内容含: 突出生态环境问题整改验收销号公示 / 接访安排 / 帮扶名单公示 等
"""
import sys, os, re, asyncio, json, fcntl, time
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE = "https://www.ahtxq.gov.cn"
COLUMN_API = "https://www.ahtxq.gov.cn/content/column/6793870"
SITE_NAME = "屯溪区-通知公告"
LOCK_FILE = "/tmp/ahtxq_tzgg.lock"

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"

def acquire_lock(max_wait=1800):
    fd = open(LOCK_FILE, 'w')
    try:
        fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
        return fd
    except BlockingIOError:
        print(f"  [LOCK] 另一 ahtxq 实例运行中, 等待释放 (最多 {max_wait}s)...")
        deadline = time.time() + max_wait
        while time.time() < deadline:
            try:
                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
                return fd
            except BlockingIOError:
                time.sleep(10)
        print("  [LOCK] 等待超时, 放弃")
        return None


async def run_async(pages=5):
    from playwright.async_api import async_playwright
    from playwright_stealth import Stealth
    async with async_playwright() as p:
        browser = await p.chromium.launch(
            headless=True,
            executable_path='/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome',
            args=['--no-sandbox', '--disable-setuid-sandbox', '--disable-blink-features=AutomationControlled']
        )
        ctx = await browser.new_context(user_agent=UA, locale='zh-CN',
                                        viewport={'width': 1440, 'height': 900}, timezone_id='Asia/Shanghai')
        stealth = Stealth()
        await stealth.apply_stealth_async(ctx)
        page = await ctx.new_page()

        # 过盾: 访问列表页
        try:
            resp = await page.goto(BASE + "/zxzx/tzgg/index.html", wait_until='domcontentloaded', timeout=35000)
            await page.wait_for_timeout(8000)
            html = await page.content()
            m = re.search(r'<title>([^<]*)</title>', html)
            print(f"  [LOAD] status={resp.status if resp else '?'} title={m.group(1) if m else '?'} len={len(html)}")
            if m and ('NWAF' in m.group(1) or 'Environment' in m.group(1)):
                print("  [WAF] 仍被拦截")
                await browser.close()
                return 0
        except Exception as e:
            print(f"  [LOAD ERR] {str(e)[:120]}")
            await browser.close()
            return 0

        # 列表: 第1页 DOM 提取 + 翻页 API
        items_all = []
        seen = set()

        def extract_from_html(h):
            out = []
            for mm in re.finditer(r'<li[^>]*>\s*<span[^>]*>([^<]*)</span>\s*<a[^>]*href="([^"]*)"[^>]*title="([^"]*)"', h):
                date, href, title = mm.group(1).strip(), mm.group(2).strip(), mm.group(3).strip()
                if href.startswith('/'):
                    href = BASE + href
                out.append({'url': href, 'title': title, 'date': date})
            return out

        # 第 1 页 (DOM)
        html = await page.content()
        items = extract_from_html(html)
        if not items:
            # 回退: 从 DOM 提取
            items = await page.evaluate("""() => {
                const out = [];
                document.querySelectorAll('ul li').forEach(li => {
                    const a = li.querySelector('a');
                    if (a && /\\d{6,}\\.html/.test(a.href)) {
                        const sp = li.querySelector('.date');
                        out.push({url: a.href, title: (a.getAttribute('title')||a.innerText||'').trim(), date: sp ? sp.innerText.trim() : ''});
                    }
                });
                return out;
            }""")
        print(f"  第1页: {len(items)} 条")
        for it in items:
            if it['url'] and it['url'] not in seen:
                seen.add(it['url'])
                items_all.append(it)

        # 翻页
        for pg in range(2, pages + 1):
            api_url = f"{COLUMN_API}?pageIndex={pg}"
            txt = await page.evaluate(f"fetch('{api_url}').then(r=>r.text()).catch(e=>'ERR:'+e.message)")
            if isinstance(txt, str) and txt.startswith('ERR'):
                print(f"  [p{pg}] fetch ERR: {txt[:80]}")
                break
            items = extract_from_html(txt)
            if not items:
                print(f"  [p{pg}] 无更多")
                break
            print(f"  第{pg}页: {len(items)} 条")
            for it in items:
                if it['url'] and it['url'] not in seen:
                    seen.add(it['url'])
                    items_all.append(it)
            await page.wait_for_timeout(1200)

        print(f"  [LIST] 去重后 {len(items_all)} 条")

        # 详情页
        records = []
        for idx, it in enumerate(items_all):
            print(f"  [{idx+1}/{len(items_all)}] {it['title'][:40]}...")
            try:
                resp = await page.goto(it['url'], wait_until='domcontentloaded', timeout=30000)
                await page.wait_for_timeout(2500)
                html = await page.content()
            except Exception as e:
                print(f"    [DETAIL ERR] {str(e)[:100]}")
                continue
            title = it['title']
            m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
            if m and m.group(1).strip():
                title = m.group(1).strip()
            pub_date = it['date']
            m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', html)
            if m:
                dm = re.search(r'\d{4}-\d{2}-\d{2}', m.group(1))
                if dm:
                    pub_date = dm.group()
            content = ""
            m = re.search(r'class="wzcon\s+j-fontContent[^"]*"[^>]*>([\s\S]*?)</div>\s*</div>', html)
            if m:
                content = m.group(1).strip()
            if not content:
                idx2 = html.find('class="wzcon j-fontContent"')
                if idx2 > 0:
                    start = html.find('>', idx2) + 1
                    content = html[start:start+20000]
            if not content.strip():
                print("    [SKIP] 正文空")
                continue
            content = re.sub(r'href="/', 'href="' + BASE + '/', content)
            content = re.sub(r'src="/', 'src="' + BASE + '/', content)
            records.append({
                "title": title,
                "url": it['url'],
                "pub_date": pub_date,
                "site_name": SITE_NAME,
                "content": content,
                "summary": "",
            })
        await browser.close()

    print(f"  共 {len(records)} 条有效")
    if records:
        push_to_searchdb(records, "ahtxq_tzgg")
    return len(records)


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=5)
    args = parser.parse_args()
    lock = acquire_lock()
    if not lock:
        return
    try:
        n = asyncio.run(run_async(pages=args.pages))
        print(f"Done: {n} records")
    finally:
        os.close(lock.fileno())


if __name__ == "__main__":
    main()