#!/usr/bin/env python3
"""Try to bypass CloudWAF with Playwright stealth"""
from playwright.sync_api import sync_playwright
import time

with sync_playwright() as p:
    browser = p.chromium.launch(
        headless=True,
        args=['--no-sandbox', '--disable-blink-features=AutomationControlled']
    )
    context = browser.new_context(
        user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
        viewport={'width': 1920, 'height': 1080},
        locale='zh-CN',
    )
    page = context.new_page()
    
    # First visit homepage then navigate
    try:
        page.goto('https://njna.nanjing.gov.cn/', wait_until='domcontentloaded', timeout=15000)
        print('Homepage loaded, waiting...')
        time.sleep(2)
        # Now navigate to the 环境保护 page
        page.goto('https://njna.nanjing.gov.cn/njsjbxqglwyh/214/275/index_17586.html', 
                   wait_until='domcontentloaded', timeout=30000)
        time.sleep(3)
        html = page.content()
        
        # Check if blocked
        blocked_markers = ['知道创宇', 'jiasule', 'cloudwaf']
        blocked = any(m in html for m in blocked_markers)
        
        if blocked or '403' in html[:500]:
            print('BLOCKED - length:', len(html))
        else:
            print('SUCCESS! Length:', len(html))
            # Save the HTML
            with open('/root/gov_crawler/njna_page.html', 'w') as f:
                f.write(html)
            print('Saved to njna_page.html')
            # Extract list items
            from bs4 import BeautifulSoup
            soup = BeautifulSoup(html, 'html.parser')
            items = soup.select('ul#result li')
            print('List items:', len(items))
            for item in items[:5]:
                a = item.select_one('a')
                date_span = item.select_one('span.d2')
                if a:
                    print(' ', a.get('title', '')[:50], date_span.text.strip() if date_span else '')
    except Exception as e:
        print('Error:', e)
    
    browser.close()