#!/usr/bin/env python3
"""永昌县人民政府 - 重大项目 爬虫
JPAAS CMS, API分页, 20条/页, 233条共12页
列表: <li class="cf"><a href="...">title</a><span class="fr">date</span>
详情: <div class="ty-content"> 正文 (含HTML p标签)
"""

import os
import sys
import re
import json
import urllib.request
import urllib.parse
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "yongchang.gov.cn-重大项目"
BASE_URL = "https://www.yongchang.gov.cn"

API_URL = "/api-gateway/jpaas-publish-server/front/page/build/unit"
PAGE_SIZE = 20

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
print(f"[info] 日期过滤: >= {CUTOFF_DATE}")


def fetch_api_page(page_no):
    """调用JPAAS API获取列表数据"""
    params = {
        "parseType": "bulidstatic",
        "webId": "b9dd4637b92748389b02b23ceafbcba4",
        "tplSetId": "885c17997d1547f58f853025486081ce",
        "pageType": "column",
        "tagId": "国务院和国务院办公厅文件_list",
        "editType": "null",
        "pageId": "74363f48bb4b4acaafa9c38e6f92c56a",
        "paramJson": json.dumps({"pageNo": page_no, "pageSize": PAGE_SIZE}, ensure_ascii=False)
    }
    url = f"{BASE_URL}{API_URL}?{urllib.parse.urlencode(params)}"
    try:
        req = urllib.request.Request(url, headers=HEADERS)
        resp = urllib.request.urlopen(req, timeout=30)
        return json.loads(resp.read())
    except Exception as e:
        print(f"[error] API第{page_no}页请求失败: {e}")
        return None


def parse_list_items(html):
    """解析列表HTML，提取标题、URL、日期"""
    items = re.findall(r'<li class="cf">(.*?)</li>', html, re.DOTALL)
    results = []

    for item in items:
        url_m = re.search(r'href="([^"]+)"', item)
        date_m = re.search(r'<span class="fr">([^<]+)</span>', item)
        # Title is inside <a> tag, after <i></i>
        title_m = re.search(r'<a[^>]*><i></i>([^<]+)</a>', item)
        if url_m and title_m:
            results.append({
                "title": title_m.group(1).strip(),
                "url": url_m.group(1),
                "date": date_m.group(1).strip() if date_m else ""
            })
    return results


def fetch_detail_content(url):
    """从详情页提取正文HTML"""
    try:
        req = urllib.request.Request(url, headers=HEADERS)
        resp = urllib.request.urlopen(req, timeout=30)
        html = resp.read().decode("utf-8")
    except Exception as e:
        print(f"  [warn] 详情页请求失败: {url[-60:]}, {e}")
        return ""

    # Main content: <div class="ty-content"> ... </div>
    m = re.search(r'<div class="ty-content">([\s\S]*?)</div>\s*<div class="ty-close"', html)
    if m:
        content = m.group(1).strip()
        # Clean up excessive whitespace
        content = re.sub(r'\n\s*\n', '\n', content)
        return content

    return ""


def main():
    import sqlite3

    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    c.execute("PRAGMA journal_mode=WAL")

    data = fetch_api_page(1)
    if not data or not data.get("success"):
        print("[error] 无法获取数据，退出")
        sys.exit(1)

    html = data.get("data", {}).get("html", "")
    # Extract total count from pagination div
    count_m = re.search(r'count="(\d+)"', html)
    total = int(count_m.group(1)) if count_m else 0
    total_pages = (total + PAGE_SIZE - 1) // PAGE_SIZE
    print(f"[info] 总条数: {total}, 总页数: {total_pages}, 每页: {PAGE_SIZE}")

    new_count = 0
    old_count = 0
    dup_count = 0
    err_count = 0

    for page in range(1, total_pages + 1):
        data = fetch_api_page(page)
        if not data or not data.get("success"):
            print(f"[warn] 第{page}页API失败，跳过")
            continue

        html = data.get("data", {}).get("html", "")
        items = parse_list_items(html)
        page_new = 0

        for item in items:
            title = item["title"]
            url = item["url"]
            date = item["date"]

            # URL may be relative
            if url.startswith("/"):
                url = BASE_URL + url

            # Date filter
            if date and date < CUTOFF_DATE:
                old_count += 1
                continue

            # Check duplicate
            c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=?", (url,))
            if c.fetchone()[0] > 0:
                dup_count += 1
                continue

            content = fetch_detail_content(url)
            if not content:
                content = f"[无正文] 附件型或PDF型条目，请查看原文链接"

            try:
                c.execute("""INSERT OR IGNORE INTO gov_raw
                    (page_url, title, content, site_name, publish_date, summary, category)
                    VALUES (?, ?, ?, ?, ?, ?, ?)""",
                    (url, title, content, SITE_NAME, date, title, "重大项目"))
                if c.rowcount > 0:
                    page_new += 1
            except Exception as e:
                print(f"  [error] 入库失败: {e}")
                err_count += 1

        conn.commit()
        new_count += page_new
        print(f"[page {page}/{total_pages}] +{page_new} new, cumulative: {new_count}")

    conn.close()
    print(f"\n[完成] {SITE_NAME}: 新增 {new_count} 条, 旧跳过 {old_count}, 重复 {dup_count}, 错误 {err_count}")


if __name__ == "__main__":
    main()