#!/usr/bin/env python3
"""宜丰县人民政府 - 环评信息 爬虫
AJAX API: POST /searchManuscript, 15条/页, 817条约55页
列表API直接返回title、pubDate、contentSource、全文HTML content
"""

import os
import json
import sys
import urllib.request
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "jxyf.gov.cn-环评信息"
API_URL = "http://www.jxyf.gov.cn/searchManuscript"
PAGE_SIZE = 15
CHANNEL_ID = "2014224355402833920"

HEADERS = {
    "Content-Type": "application/json;charset=UTF-8",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}

CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
print(f"[info] 日期过滤: >= {CUTOFF_DATE}")


def fetch_api_page(page_no):
    """调用API获取列表数据"""
    payload = json.dumps({
        "current": page_no,
        "pageSize": PAGE_SIZE,
        "channelTreeIds": [CHANNEL_ID],
        "title": "",
        "Contenthtml": ""
    }).encode()
    try:
        req = urllib.request.Request(API_URL, data=payload, headers=HEADERS)
        resp = urllib.request.urlopen(req, timeout=30)
        return json.loads(resp.read())
    except Exception as e:
        print(f"[error] API第{page_no}页请求失败: {e}")
        return None


def main():
    import sqlite3

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    c.execute("PRAGMA journal_mode=WAL")

    # Get first page to determine total
    data = fetch_api_page(1)
    if not data or not data.get("data"):
        print("[error] 无法获取数据")
        sys.exit(1)

    total = data["data"].get("total", 0)
    total_pages = (total + PAGE_SIZE - 1) // PAGE_SIZE
    print(f"[info] 总条数: {total}, 总页数: {total_pages}, 每页: {PAGE_SIZE}")

    new_count = 0
    old_count = 0
    dup_count = 0
    err_count = 0

    for page in range(1, total_pages + 1):
        data = fetch_api_page(page)
        if not data or not data.get("data"):
            print(f"[warn] 第{page}页API失败，跳过")
            continue

        results = data["data"].get("results", [])
        page_new = 0

        for item in results:
            title = item.get("title", "") or item.get("showTitle", "")
            date_str = item.get("pubDate", "")[:10]
            source = item.get("contentSource", "")

            # Build page URL from urls.pc (JSON string)
            urls_str = item.get("urls", "")
            page_url = ""
            if isinstance(urls_str, str) and urls_str.startswith("{"):
                try:
                    urls_obj = json.loads(urls_str)
                    if "pc" in urls_obj:
                        page_url = f"http://www.jxyf.gov.cn{urls_obj['pc']}"
                except json.JSONDecodeError:
                    pass
            elif isinstance(urls_str, dict):
                if "pc" in urls_str:
                    page_url = f"http://www.jxyf.gov.cn{urls_str['pc']}"

            if not page_url:
                err_count += 1
                continue

            # Date filter
            if date_str and date_str < CUTOFF_DATE:
                old_count += 1
                continue

            # Check duplicate
            c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=?", (page_url,))
            if c.fetchone()[0] > 0:
                dup_count += 1
                continue

            # Extract HTML content from response (no additional request needed!)
            content_dict = item.get("content", {})
            if isinstance(content_dict, dict):
                content = content_dict.get("content", "").strip()
            else:
                content = ""

            if not content:
                content = "[无正文] 附件型条目，请查看原文链接"

            try:
                c.execute("""INSERT OR IGNORE INTO gov_raw
                    (page_url, title, content, site_name, publish_date, summary, category)
                    VALUES (?, ?, ?, ?, ?, ?, ?)""",
                    (page_url, title, content, SITE_NAME, date_str, title, "环评信息"))
                if c.rowcount > 0:
                    page_new += 1
            except Exception as e:
                print(f"  [error] 入库失败: {e}")
                err_count += 1

        conn.commit()
        new_count += page_new
        print(f"[page {page}/{total_pages}] +{page_new} new, cumulative: {new_count}")

    conn.close()
    print(f"\n[完成] {SITE_NAME}: 新增 {new_count} 条, 旧跳过 {old_count}, 重复 {dup_count}, 错误 {err_count}")


if __name__ == "__main__":
    main()