#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Retry tzgg.html with WAF-challenge tolerance"""
import asyncio, json
from playwright.async_api import async_playwright

async def main():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True, args=["--disable-blink-features=AutomationControlled", "--no-sandbox"])
        ctx = await browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
            viewport={"width": 1400, "height": 900},
        )
        page = await ctx.new_page()
        reqs = []
        async def on_request(req):
            if "truecms" in req.url or "jpage" in req.url or "messageController" in req.url:
                reqs.append({"REQ": req.method, "url": req.url[:220], "post": (req.post_data or "")[:400]})
        async def on_response(resp):
            if "truecms" in resp.url or "jpage" in resp.url:
                reqs.append({"RESP": resp.status, "url": resp.url[:220]})
        page.on("request", on_request)
        page.on("response", on_response)

        for attempt in range(3):
            try:
                print(f"attempt {attempt}: goto commit...")
                await page.goto("https://sthjj.nantong.gov.cn/ntshbj/tzgg/tzgg.html", wait_until="commit", timeout=45000)
                break
            except Exception as e:
                print(f"  goto err: {str(e)[:150]}")
                await page.wait_for_timeout(5000)
        await page.wait_for_timeout(15000)
        try:
            html = await page.content()
            print("html len:", len(html))
            items = await page.query_selector_all("#msgList li, .list-ul li")
            print("list items in DOM:", len(items))
            with open("/root/gov_crawler/nt_tzgg_rendered.html", "w", encoding="utf-8") as f:
                f.write(html)
        except Exception as e:
            print("content err:", e)
        print("\n=== requests ===")
        for r in reqs:
            print(json.dumps(r, ensure_ascii=False))
        await browser.close()

asyncio.run(main())