#!/usr/bin/env python3
"""徐州贾汪区-环境保护 - Epoint WebBuilder 5.0 API"""
import os, sys, re, json, requests
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "徐州贾汪区-环境保护"
BASE = "https://www.xzjw.gov.cn"
API_URL = f"{BASE}/EWB-FRONT/rest/lightfrontaction/getgovinfolist"
PAGE_SIZE = 20
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

HEADERS = {"User-Agent": "Mozilla/5.0", "Content-Type": "application/json"}
SITE_GUID = "25392afd-419a-43b4-ab9a-e5a27a105dd5"
PARAMS = {"deptcode": "001001", "categorynum": "003013006", "siteGuid": SITE_GUID}

import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

def extract_content(url):
    """Fetch detail page and extract content from p#ivs_content"""
    try:
        r = requests.get(url, headers=HEADERS, verify=False, timeout=15)
        if r.status_code != 200: return ""
        html = r.text
        m = re.search(r'<p[^>]*id="ivs_content"[^>]*>(.*?)</p>\s*</div>', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
            return content
        m = re.search(r'class="mian-cont"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
        if m:
            return m.group(1).strip()
        return ""
    except:
        return ""

def crawl(test=False, incremental=True):
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    
    existing = set()
    if incremental:
        existing = set(r[0] for r in conn.execute(
            "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchall())
    
    total_new = 0
    page = 0
    total_api = 0
    
    while True:
        data = {**PARAMS, "pageIndex": page, "pageSize": PAGE_SIZE}
        try:
            r = requests.post(API_URL, json=data, headers=HEADERS, verify=False, timeout=15)
            if r.status_code != 200:
                print(f"  API error: {r.status_code}")
                break
            result = r.json()
            items = result.get("custom", {}).get("data", [])
            total_api = result.get("custom", {}).get("total", 0)
            
            if not items:
                break
            
            batch = []
            for item in items:
                title = (item.get("realtitle") or item.get("title", "")).strip()
                date = item.get("infodate", "")
                infourl = item.get("infourl", "")
                
                if not title or not infourl:
                    continue
                if date < CUTOFF:
                    continue
                
                page_url = BASE + infourl
                if page_url in existing:
                    continue
                
                content = extract_content(page_url)
                
                batch.append((SITE_NAME, page_url, page_url, title, date,
                             content or title, content or title, date, '003013006'))
            
            if batch:
                conn.executemany(
                    """INSERT OR IGNORE INTO gov_raw
                    (site_name, source_url, page_url, title, publish_date, content, summary, date_rank, category)
                    VALUES (?,?,?,?,?,?,?,?,?)""", batch)
                conn.commit()
                total_new += len(batch)
            
            print(f"  第{page+1}页: {len(items)}条, 新增{len(batch)}条")
            
            if test or len(items) < PAGE_SIZE:
                break
            page += 1
            
        except Exception as e:
            print(f"  Error: {e}")
            break
    
    if total_new > 0:
        conn.execute(
            "INSERT OR REPLACE INTO gov_search(rowid,title,site_name,summary) "
            "SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r "
            "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (SITE_NAME,))
        conn.commit()
    
    total_db = conn.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchone()[0]
    conn.close()
    
    print(f"\n[{SITE_NAME}]")
    print(f"  总API数: {total_api}")
    print(f"  本次新增: {total_new}")
    print(f"  DB总数: {total_db}")
    return total_new

if __name__ == "__main__":
    test_mode = "--test" in sys.argv
    inc = "--full" not in sys.argv
    crawl(test=test_mode, incremental=inc)