#!/usr/bin/env python3
"""祥云县-生态环境 爬虫 (JPAAS CMS /queryList API)"""
import re, sys, sqlite3, json
import requests
import os

SITE_NAME = "祥云县-生态环境"
BASE = "https://www.xiangyun.gov.cn"
API_URL = f"{BASE}/queryList"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Content-Type": "application/json;charset=UTF-8",
    "Referer": f"{BASE}/xyxrmzf/c106069/pc/list.html",
}

API_BODY = {
    "tenantId": "1961003460807458816",
    "channelCode": ["c106069"],
    "webSiteCode": ["xyxrmzf"],
    "pageSize": 15,
    "current": 1,
}

def fetch_page(session, page):
    body = dict(API_BODY)
    body["current"] = page
    try:
        r = session.post(API_URL, json=body, headers=HEADERS, timeout=30)
        return r.json() if r.status_code == 200 else None
    except:
        return None

def parse_list(api_data):
    items = []
    results = api_data.get("data", {}).get("results", [])
    for entry in results:
        src = entry.get("source", {})
        title = (src.get("title") or "").strip()
        pub_date = (src.get("pubDate") or "")[:10]
        urls_raw = src.get("urls", "{}")
        try:
            urls = json.loads(urls_raw) if isinstance(urls_raw, str) else urls_raw
        except:
            urls = {}
        url = urls.get("pc", "")
        if url and not url.startswith("http"):
            url = BASE + url
        content = src.get("content", {})
        html_content = content.get("content", "") if isinstance(content, dict) else ""
        # Clean style/class attrs, strip script/style tags
        html_content = re.sub(r'\s*style="[^"]*"', "", html_content)
        html_content = re.sub(r'\s*class="[^"]*"', "", html_content)
        html_content = re.sub(r'<script[^>]*>.*?</script>', "", html_content, flags=re.DOTALL | re.I)
        html_content = re.sub(r'<style[^>]*>.*?</style>', "", html_content, flags=re.DOTALL | re.I)
        if title and url:
            items.append({
                "title": title,
                "url": url,
                "date": pub_date,
                "content": html_content,
            })
    return items

def run(max_pages=None):
    if max_pages is None:
        max_pages = MAX_PAGES
    session = requests.Session()
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    known = set(r[0] for r in db.execute(
        "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchall())
    db.close()

    all_items = []
    for page in range(1, max_pages + 1):
        print(f"  Page {page}...", end=" ", flush=True)
        api_data = fetch_page(session, page)
        if not api_data:
            print("FAIL")
            break
        items = parse_list(api_data)
        new_items = [it for it in items if it["url"] not in known]
        total = api_data.get("data", {}).get("total", 0)
        print(f"OK {len(items)} items (new: {len(new_items)}, total: {total})")
        all_items.extend(new_items)
        if len(items) < 10 or page * 15 >= total:
            break

    if not all_items:
        print("  no new data")
        return

    print(f"  Processing {len(all_items)} items...")
    results = []
    for i, item in enumerate(all_items):
        summary = re.sub(r'<[^>]+>', ' ', item.get("content", "")).strip()
        summary = re.sub(r'\s+', ' ', summary)[:500]
        results.append({
            "site_name": SITE_NAME,
            "source_url": item["url"][:500],
            "page_url": item["url"],
            "title": item["title"][:500],
            "publish_date": item["date"][:10] if item["date"] else "",
            "summary": item["title"][:500],
            "content": item.get("content", ""),
            "status": "active",
            "category": "",
            "tags": "",
        })
        if (i + 1) % 50 == 0:
            print(f"    [{i+1}/{len(all_items)}]")

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in results:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name,source_url,page_url,title,publish_date,summary,content,status,category,tags) VALUES (?,?,?,?,?,?,?,?,?,?)",
                (item["site_name"][:200], item["source_url"], item["page_url"], item["title"],
                 item["publish_date"], item["summary"], item["content"], item["status"],
                 item["category"], item["tags"]),
            )
            if db.total_changes > 0: ok += 1
            else: skip += 1
        except: skip += 1
    db.commit()
    db.execute(
        "INSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,),
    )
    db.commit()
    db.close()
    print(f"  Saved: new={ok}, skip={skip}, FTS synced")

if __name__ == "__main__":
    max_p = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else None
    run(max_p)