#!/usr/bin/env python3
"""
宿松县 - 县经开区 > 回应关切 (susong.gov.cn)
CMS: 龙讯 (Lonsun), AJAX 分页
AJAX API: /site/label/8888?labelName=publicInfoList&...&pageIndex=N
Detail: /public/2000003991/{contentId}.html
  Content container: div.ls-content
  Attachments: PDF links inside content
Protection: Cloudflare JS Challenge (needs Playwright)
"""
import sys
import re
import json
import os
import sqlite3
import time
from datetime import datetime

from playwright.sync_api import sync_playwright

# === CONFIG ===
SITE_NAME = "宿松县经开区-回应关切"
BASE_URL = "https://www.susong.gov.cn"
LIST_URL = f"{BASE_URL}/public/column/2000003991?type=4&catId=41334066&action=list&nav=3"
DB_PATH = "/root/search.db"
CUTOFF_DATE = "2023-01-01"
PAGES_DEFAULT = 1  # Only 1 page (17 items total)
ITEMS_PER_PAGE = 20


def solve_cloudflare():
    """Use Playwright to solve Cloudflare JS challenge and return context/page"""
    p = sync_playwright().__enter__()
    browser = p.chromium.launch(
        headless=True,
        args=[
            "--disable-blink-features=AutomationControlled",
            "--no-sandbox",
            "--disable-dev-shm-usage",
        ]
    )
    context = browser.new_context(
        user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
        viewport={"width": 1920, "height": 1080},
        locale="zh-CN",
        timezone_id="Asia/Shanghai",
    )
    context.add_init_script("""
        Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
        Object.defineProperty(navigator, 'languages', { get: () => ['zh-CN', 'zh', 'en'] });
    """)
    page = context.new_page()
    
    print(f"[INFO] Solving Cloudflare challenge for {LIST_URL}")
    page.goto(LIST_URL, timeout=60000, wait_until="domcontentloaded")
    page.wait_for_timeout(8000)
    
    # Verify page loaded
    title = page.title()
    if "521" in title or not title:
        print(f"[ERROR] Cloudflare challenge not bypassed, title: {title}")
        page.screenshot(path="/tmp/susong_cf_error.png")
        browser.close()
        p.__exit__(None, None, None)
        return None, None, None
    
    print(f"[OK] Page loaded: {title}")
    return p, browser, context, page


def fetch_list_api(page_obj, page_index=1):
    """Fetch list via AJAX API using Playwright"""
    url = (f"{BASE_URL}/site/label/8888"
           f"?_={time.time()}"
           f"&labelName=publicInfoList"
           f"&siteId=2000005581"
           f"&organId=2000003991"
           f"&pageSize={ITEMS_PER_PAGE}"
           f"&pageIndex={page_index}"
           f"&isDate=true&dateFormat=yyyy-MM-dd&length=50"
           f"&type=4&action=list&isDriving=true&result=&isJson=true"
           f"&keyWords=&isSetValue=true&catIds=&catId=41334066")
    
    try:
        result = page_obj.evaluate(f"""async () => {{
            const resp = await fetch('{url}');
            return await resp.text();
        }}""")
        data = json.loads(result)
        items = data.get("data", [])
        total = data.get("total", 0)
        page_count = data.get("pageCount", 1)
        print(f"  [API] pageIndex={page_index}, {len(items)} items, total={total}, pages={page_count}")
        return items, total, page_count
    except Exception as e:
        print(f"  [API ERROR] page {page_index}: {e}")
        return [], 0, 0


def fetch_detail(page_obj, content_id):
    """Fetch detail page using in-page fetch (faster than goto)"""
    url = f"{BASE_URL}/public/2000003991/{content_id}.html"
    
    try:
        result = page_obj.evaluate(f"""async () => {{
            try {{
                const resp = await fetch('{url}');
                const html = await resp.text();
                
                // Parse HTML to extract content
                const parser = new DOMParser();
                const doc = parser.parseFromString(html, 'text/html');
                
                // Title
                const pageTitle = (doc.title || '').replace('_信息公开_宿松县人民政府', '').trim();
                
                // Content
                const contentEl = doc.querySelector('div.ls-content');
                if (!contentEl) return JSON.stringify({{title: pageTitle, content: '', attachments: []}});
                
                // Attachments
                const attachmentLinks = [];
                const anchors = contentEl.querySelectorAll('a[href$=".pdf"]');
                anchors.forEach(a => {{
                    const href = a.href || a.getAttribute('href') || '';
                    const text = (a.innerText || '').trim();
                    const fullHref = href.startsWith('http') ? href : 'https://www.susong.gov.cn' + href;
                    if (fullHref && !attachmentLinks.some(x => x.href === fullHref)) {{
                        attachmentLinks.push({{href: fullHref, text: text || fullHref.split('/').pop()}});
                    }}
                }});
                
                const textContent = contentEl.innerText || '';
                
                return JSON.stringify({{
                    title: pageTitle,
                    content: textContent,
                    attachments: attachmentLinks
                }});
            }} catch(e) {{
                return JSON.stringify({{title: '', content: '', attachments: [], error: e.toString()}});
            }}
        }}""")
        
        data = json.loads(result)
        return data
    except Exception as e:
        print(f"  [DETAIL ERROR] {url}: {e}")
        return {"title": "", "content": "", "attachments": []}


def insert_to_db(items):
    """Insert items into gov_raw"""
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    inserted = 0
    skipped = 0
    
    for item in items:
        title = item.get("title", "")
        url = item.get("url", "")
        date = item.get("date", "")
        content = item.get("content", "")
        attachments = item.get("attachments", "")
        
        if not content and not title:
            skipped += 1
            continue
        
        try:
            c.execute("""
                INSERT OR IGNORE INTO gov_raw
                (title, page_url, site_name, publish_date, content, summary, attachments, source_url, date_rank)
                VALUES (?, ?, ?, ?, ?, '', ?, ?, CAST(strftime('%s', ?) AS INTEGER))
            """, (
                title, url, SITE_NAME, date, content, attachments, url, date,
            ))
            if c.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            print(f"  [DB ERROR] {title[:30]}: {e}")
            skipped += 1
    
    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    global PAGES_DEFAULT
    max_pages = PAGES_DEFAULT
    for arg in sys.argv[1:]:
        if arg.isdigit():
            max_pages = int(arg)
    
    print(f"[INFO] {SITE_NAME} - 爬虫, max_pages={max_pages}")
    
    p_obj, browser, context, page = None, None, None, None
    
    try:
        # Step 1: Solve Cloudflare
        pw = sync_playwright().__enter__()
        browser = pw.chromium.launch(
            headless=True,
            args=["--disable-blink-features=AutomationControlled", "--no-sandbox", "--disable-dev-shm-usage"]
        )
        context = browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            viewport={"width": 1920, "height": 1080},
            locale="zh-CN", timezone_id="Asia/Shanghai",
        )
        context.add_init_script("""
            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
            Object.defineProperty(navigator, 'languages', { get: () => ['zh-CN', 'zh', 'en'] });
        """)
        page = context.new_page()
        
        print(f"[INFO] Solving Cloudflare challenge...")
        page.goto(LIST_URL, timeout=60000, wait_until="domcontentloaded")
        page.wait_for_timeout(8000)
        
        title = page.title()
        if "521" in title or not title:
            print(f"[ERROR] Cloudflare bypass failed, title: {title}")
            return
        print(f"[OK] Cloudflare bypassed: {title}")
        
        # Step 2: Fetch list from API
        all_items = []
        total_new = 0
        total_old = 0
        
        for page_index in range(1, max_pages + 1):
            items, total, page_count = fetch_list_api(page, page_index)
            
            if not items:
                print(f"  [STOP] No items on page {page_index}")
                break
            
            for item in items:
                content_id = item.get("contentId")
                title = item.get("title", "")
                link = item.get("link", "")
                pub_date = item.get("publishDate", "")[:10]
                
                if pub_date < CUTOFF_DATE:
                    print(f"  [STOP] Date {pub_date} < {CUTOFF_DATE}")
                    break
                
                print(f"  [{page_index}] {pub_date} {title[:50]}...")
                
                # Fetch detail
                detail = fetch_detail(page, content_id)
                
                # Build attachments string
                attach_str = ""
                for att in detail.get("attachments", []):
                    if attach_str:
                        attach_str += "\n"
                    attach_str += f"[{att['text']}]({att['href']})"
                
                # Clean content
                content_text = detail.get("content", "")
                
                all_items.append({
                    "title": title,
                    "url": link,
                    "date": pub_date,
                    "content": content_text,
                    "attachments": attach_str,
                })
            
            # Batch insert
            if all_items:
                new, old = insert_to_db(all_items)
                total_new += new
                total_old += old
                print(f"  [DB] Page {page_index}: +{new} new, {old} existing")
                all_items = []
        
        if all_items:
            new, old = insert_to_db(all_items)
            total_new += new
            total_old += old
        
        print(f"\n[DONE] 新增: {total_new}, 跳过: {total_old}")
        
        if total_new > 0:
            conn = sqlite3.connect(DB_PATH, timeout=30)
            conn.execute("SELECT 1 /* noop: gov_search 由触发器维护, 无需 rebuild */")
            conn.commit()
            conn.close()
            print("[FTS] Rebuilt")
        
    except Exception as e:
        print(f"[ERROR] {e}")
        import traceback
        traceback.print_exc()
    
    finally:
        if browser:
            browser.close()
        try:
            pw.stop()
        except:
            pass


if __name__ == "__main__":
    main()