#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
crawl_ynws_gsgg.py - 文山州-公示公告
CMS: 数融CMS (UCAP), API分页
API: POST /queryList {channelCode: ["gsgg"]}
     20条/页, 545条, ~28页
正文直接包含在API返回中
"""
import requests
import re
import json
import sys
import os
import time
from datetime import datetime

DB_PATH = "/root/search.db"
API_URL = "https://www.ynws.gov.cn/queryList"
SITE_NAME = "文山州-公示公告"
GROUP = "云南"
INDUSTRY = "环境公示"
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36",
    "Content-Type": "application/json",
}

import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SESSION = requests.Session()


def fetch_page(page=1):
    try:
        r = SESSION.post(API_URL, json={
            "current": page,
            "pageSize": 20,
            "webSiteCode": ["wszzf"],
            "channelCode": ["gsgg"],
        }, headers=HEADERS, timeout=60, verify=False)
        return r.json() if r.status_code == 200 else None
    except Exception as e:
        print(f"  ERR: {e}")
        return None


def norm_date(d):
    if not d:
        return ""
    m = re.search(r"(\d{4})-(\d{1,2})-(\d{1,2})", str(d))
    if m:
        y, mo, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
        return f"{y:04d}-{mo:02d}-{day:02d}" if 1990 <= y <= 2099 else ""
    return ""


def clean_html(html):
    if not html:
        return ""
    cleaned = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.I)
    cleaned = re.sub(r"<style[^>]*>.*?</style>", "", cleaned, flags=re.DOTALL | re.I)
    cleaned = re.sub(r"<p>\s*<br\s*/?>\s*</p>", "", cleaned)
    if len(cleaned) > 30000:
        cleaned = cleaned[:30000] + "..."
    return cleaned.strip()


def save_to_db(items_data):
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=10)
    c = conn.cursor()
    inserted = 0
    fts_batch = []
    for title, date_str, content, summary, page_url in items_data:
        try:
            c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
            if c.fetchone():
                continue
            c.execute(
                """INSERT INTO gov_raw 
                   (page_url, title, site_name, publish_date, content, summary,
                    source_url, category, industry, group_name, script_name)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
                (page_url, title, SITE_NAME, date_str, content, summary,
                 page_url, "公示公告", INDUSTRY, GROUP, "crawl_ynws_gsgg.py"),
            )
            new_id = c.lastrowid
            if new_id:
                fts_batch.append((new_id, title, SITE_NAME, summary or ""))
            inserted += 1
        except Exception as e:
            print(f"  [DB ERROR] {title[:30]}: {e}")
    conn.commit()
    if fts_batch:
        for rid, title, site, summary in fts_batch:
            try:
                c.execute(
                    "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
                    (rid, title, site, summary),
                )
            except Exception as e:
                print(f"  [FTS ERROR] id={rid}: {e}")
        conn.commit()
        print(f"  FTS同步: {len(fts_batch)}条")
    conn.close()
    return inserted


def main():
    import argparse
    parser = argparse.ArgumentParser(description=f"{SITE_NAME}爬虫")
    parser.add_argument("--pages", type=int, default=MAX_PAGES, help="爬取页数")
    args = parser.parse_args()

    max_pages = args.pages
    print(f"[{datetime.now().strftime('%H:%M:%S')}] {SITE_NAME}...", flush=True)

    all_batch = []
    for pg in range(1, max_pages + 1):
        data = fetch_page(pg)
        if not data or "data" not in data:
            print(f"  第{pg}页: 无返回", flush=True)
            break
        results = data["data"].get("results", [])
        if not results:
            print(f"  第{pg}页: 0条", flush=True)
            break

        page_ok = 0
        for item in results:
            src = item.get("source", {})
            title = (src.get("showTitle") or src.get("title") or "").strip()
            title = re.sub(r"<[^>]+>", "", title).strip()
            if not title:
                continue
            pub_date = norm_date(src.get("pubDate", ""))
            urls_str = src.get("urls", "{}")
            try:
                urls = json.loads(urls_str) if isinstance(urls_str, str) else urls_str
                page_url = urls.get("pc", "")
                if page_url and not page_url.startswith("http"):
                    page_url = "https://www.ynws.gov.cn" + page_url
            except:
                page_url = ""
            page_url = page_url.replace("\\/", "/")
            if not page_url:
                continue

            cobj = src.get("content", {})
            content = cobj.get("content", "") if isinstance(cobj, dict) else (cobj if isinstance(cobj, str) else "")
            content = clean_html(content)
            summary = re.sub(r"<[^>]+>", " ", content).strip()
            summary = re.sub(r"\s+", " ", summary)[:300]

            all_batch.append((title, pub_date, content, summary, page_url))
            page_ok += 1

        print(f"  第{pg}页: {page_ok}条", flush=True)

    if not all_batch:
        print("无数据")
        return

    saved = save_to_db(all_batch)
    print(f"[{datetime.now().strftime('%H:%M:%S')}] 入库{saved}/{len(all_batch)}条", flush=True)


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"耗时: {time.time()-t0:.1f}s")