#!/usr/bin/env python3
"""
crawl_yzq.py — 袁州区人民政府-公告公示
CMS: 数融 CMS (同宜春/铜鼓)
列表: POST /queryList JSON API (channelId, pageSize, current)
详情: 正文已包含在列表API返回的 content.content 字段，无需额外请求
WAF: 无

运行: python3 crawl_yzq.py [--full]
  --full: 全量 (6页 ~105条近3年)
  不加: 仅第1页 (日跑增量)
"""

import sys, json, sqlite3, os, re
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "袁州区人民政府-公告公示"
SOURCE = "袁州区人民政府"

API_URL = "https://www.yzq.gov.cn/queryList"
CHANNEL_ID = "1996809602120392704"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Content-Type": "application/json;charset=UTF-8",
    "Referer": "https://www.yzq.gov.cn/yzqrmzf/gggs/pc/list.html"
}

MAX_PAGES = 6
NUM_THREADS = 8
THREE_YEARS_AGO = datetime.now() - timedelta(days=3 * 365)
THREE_YEARS_STR = THREE_YEARS_AGO.strftime("%Y-%m-%d")


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def fetch_page(page_no):
    import urllib.request
    payload = json.dumps({
        "channelId": CHANNEL_ID,
        "pageSize": 20,
        "current": page_no
    }).encode("utf-8")
    req = urllib.request.Request(API_URL, data=payload, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=20)
        result = json.loads(resp.read().decode())
        return result.get("data", {}).get("results", [])
    except Exception as e:
        log(f"第{page_no}页请求失败: {e}")
        return []


def parse_items(items):
    articles = []
    for item in items:
        src = item.get("source", {})
        art_id = item.get("id", "")
        if not art_id:
            continue

        title = (src.get("title") or "").strip()
        pub_date = (src.get("pubDate") or "")[:10]

        if not title or not pub_date:
            continue

        # 3年过滤
        if pub_date < THREE_YEARS_STR:
            continue

        # 正文
        content = src.get("content", {}).get("content", "")
        if content:
            content = re.sub(r'\s+', ' ', content).strip()
        else:
            content = ""

        # 构造唯一URL
        page_url = f"https://www.yzq.gov.cn/yzqrmzf/gggs/pc/content/{art_id}/content_{art_id}.html"

        articles.append((page_url, title, pub_date, content))
    return articles


def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0

    for url, title, date, content in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, SOURCE, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[:50]}... {e}")

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    is_full = "--full" in sys.argv
    log(f"开始{'全量' if is_full else '增量'}爬取...")

    pages_to_crawl = range(1, MAX_PAGES + 1) if is_full else [1]

    all_articles = []
    for page_no in pages_to_crawl:
        items = fetch_page(page_no)
        if not items:
            log(f"第{page_no}页无数据，结束")
            break
        articles = parse_items(items)
        log(f"第{page_no}页: {len(items)}条（近3年: {len(articles)}条）")
        all_articles.extend(articles)

        # 如果该页全部超出3年范围，停止
        if len(articles) == 0:
            break

    log(f"共获取 {len(all_articles)} 条数据（近3年）")

    if not all_articles:
        log("没有数据，退出")
        return

    inserted, skipped = save_to_db(all_articles)
    log(f"入库完成: 新增 {inserted}, 跳过 {skipped}")


if __name__ == "__main__":
    main()