#!/usr/bin/env python3
"""金溪县人民政府-环境保护-法定主动公开内容 (jinxi.gov.cn) 爬虫

Tab: 法定主动公开内容 (section-tab #2)
列表: POST /module/xxgk/search.jsp (Playwright 辅助)
详情: /art/YYYY/M/D/art_21169_XXXXXXX.html
      ZJEG 格式: <!--<$[标题]>begin-->TITLE<!--<$[标题]>end-->
                 <!--ZJEG_RSS.content.begin-->CONTENT<!--ZJEG_RSS.content.end-->
"""

import os, sqlite3, re, time, requests, sys, json
from datetime import datetime, timezone, timedelta
from urllib3 import disable_warnings, exceptions
disable_warnings(exceptions.InsecureRequestWarning)

SITE_NAME = "金溪县-环境保护-法定主动"
BASE = "https://www.jinxi.gov.cn"
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url, max_retry=3):
    for att in range(max_retry):
        try:
            r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
            r.encoding = "utf-8"
            if r.status_code == 200:
                return r.text
        except Exception as e:
            if att < max_retry - 1:
                time.sleep(2)
    return None

def get_all_urls_via_playwright(daily=False):
    try:
        import asyncio
        from playwright.async_api import async_playwright
        
        async def _run():
            async with async_playwright() as p:
                browser = await p.chromium.launch(headless=True)
                page = await browser.new_page()
                await page.goto(
                    "https://www.jinxi.gov.cn/col/col21169/index.html?number=W00004W00004W00010",
                    wait_until="domcontentloaded", timeout=25000
                )
                await asyncio.sleep(2)
                tabs = await page.query_selector_all(".section-tab")
                await tabs[2].click()
                await asyncio.sleep(2)
                all_items = []
                # Daily: only page 1 (newest items). Full: crawl all pages
                max_pages = 1 if daily else 6
                for pg in range(1, max_pages + 1):
                    links = await page.eval_on_selector_all(
                        "a[href*='art_21169']",
                        "els => els.map(el => ({href: el.href, text: el.textContent.trim()}))"
                    )
                    for l in links:
                        all_items.append({"title": l["text"], "url": l["href"]})
                    if daily:
                        break  # daily mode: page 1 only
                    next_btn = await page.query_selector("a:has-text('下一页')")
                    if next_btn:
                        cls = await next_btn.get_attribute("class") or ""
                        if "disabled" in cls:
                            break
                        await next_btn.click()
                        await asyncio.sleep(2)
                    else:
                        break
                await browser.close()
                return all_items
        
        return asyncio.run(_run())
    except Exception as e:
        print(f"  [ERROR] Playwright: {e}")
        return []

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return "", "", ""
    
    title = ""
    m = re.search(r'<!--<\$\[(?:标题|title)\]>begin-->(.*?)<!--<\$\[(?:标题|title)\]>end-->', html)
    if m:
        title = m.group(1).strip()
    if not title:
        m = re.search(r'<title>(.*?)</title>', html)
        if m:
            title = m.group(1).strip()
    if not title:
        m = re.search(r'<h1[^>]*>(.*?)</h1>', html)
        if m:
            title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    
    content = ""
    m = re.search(r'<!--ZJEG_RSS\.content\.begin-->(.*?)<!--ZJEG_RSS\.content\.end-->', html, re.S)
    if m:
        content = m.group(1).strip()
    if not content:
        m = re.search(r'<div[^>]*id="zoom"[^>]*>(.*?)</div>', html, re.S)
        if m:
            content = m.group(1).strip()
    if not content:
        m = re.search(r'<div[^>]*class="content"[^>]*>(.*?)</div>', html, re.S)
        if m:
            content = m.group(1).strip()
    
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.S)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.S)
    
    date_str = ""
    for d in re.finditer(r'(\d{4}-\d{1,2}-\d{1,2})', html):
        ctx = html[max(0,d.start()-50):d.end()+20]
        if any(kw in ctx for kw in ['date', '时间', '发布', '日期', 'publish', 'Date']):
            date_str = d.group(1)
            break
    if not date_str:
        for d in re.finditer(r'(\d{4}-\d{1,2}-\d{1,2})', html):
            date_str = d.group(1)
            break
    
    return title, content, date_str

def main():
    test = "--test" in sys.argv
    daily = "--daily" in sys.argv
    mode = "DAILY" if daily else ("TEST" if test else "FULL")
    print(f"\n=== {SITE_NAME} ({mode}) ===", flush=True)
    
    print(f"  [Playwright] Fetching {'page 1 only' if daily else 'all pages'}...", flush=True)
    items = get_all_urls_via_playwright(daily=daily)
    print(f"  Found {len(items)} items", flush=True)
    
    if not items:
        print("  No items to crawl")
        return
    
    if test:
        items = items[:3]
    
    conn = None if test else sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor() if conn else None
    total_new = 0
    done = 0
    
    for i, item in enumerate(items):
        print(f"  [{i+1}/{len(items)}] {item['title'][:40]}... ", end="", flush=True)
        title, content, pub_date = fetch_detail(item["url"])
        if not title:
            title = item["title"]
        title = re.sub(r'begin-->|end-->', '', title).strip()
        
        if test:
            print(f"title={title[:50]} content={len(content)}B date={pub_date}")
            continue
        
        try:
            cursor.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, tags) VALUES (?,?,?,?,?,?,?)",
                (SITE_NAME, title[:500], item["url"], content, pub_date, "", "环境保护-法定主动"),
            )
            if cursor.rowcount > 0:
                total_new += 1
                print(f"{len(content)}B")
            else:
                print("skip")
        except Exception as e:
            print(f"DB error: {e}")
        
        done += 1
        if done % 20 == 0 and conn:
            conn.commit()
    
    if conn:
        conn.commit()
        conn.execute(
            "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
            "SELECT r.rowid, r.title, r.site_name, r.summary "
            "FROM gov_raw r WHERE r.rowid NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (SITE_NAME,)
        )
        conn.commit()
        c = conn.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,))
        cnt = c.fetchone()[0]
        conn.close()
        print(f"\n=== 完成 ===", flush=True)
        print(f"  新增: {total_new}条", flush=True)
        print(f"  累计: {cnt}条", flush=True)

if __name__ == "__main__":
    main()