#!/usr/bin/env python3
"""永昌县人民政府 - 公示公告 爬虫
JPAAS CMS, API分页, 15条/页, ~274页共~4100条
列表: <li class="cf"><a class="fl" href="..."><i></i>title</a><span class="fr">date</span>
详情: <div class="ty-content"> 正文 (含HTML p标签)
"""
import os
import sys
import re
import json
import time
import urllib.request
import urllib.parse
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "yongchang.gov.cn-公示公告"
BASE_URL = "https://www.yongchang.gov.cn"

API_URL = "/api-gateway/jpaas-publish-server/front/page/build/unit"
PAGE_SIZE = 15
CUTOFF_DATE = "2023-06-16"

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "b9dd4637b92748389b02b23ceafbcba4",
    "tplSetId": "885c17997d1547f58f853025486081ce",
    "pageType": "column",
    "editType": "null",
    "pageId": "59a1a9ac060643cc8870be5e3035e04a",
}


def fetch_api_page(page_no, retries=5):
    """调用JPAAS API获取列表数据（含重试）"""
    params = dict(API_PARAMS)
    params["paramJson"] = json.dumps({"pageNo": page_no, "pageSize": PAGE_SIZE}, ensure_ascii=False)
    url = f"{BASE_URL}{API_URL}?{urllib.parse.urlencode(params)}"
    for attempt in range(retries):
        try:
            req = urllib.request.Request(url, headers=HEADERS)
            resp = urllib.request.urlopen(req, timeout=30)
            return json.loads(resp.read())
        except Exception as e:
            if attempt < retries - 1:
                wait = 3 * (attempt + 1)
                print(f"  [warn] API第{page_no}页失败(第{attempt+1}次): {str(e)[:60]}, {wait}s后重试...")
                time.sleep(wait)
            else:
                print(f"  [error] API第{page_no}页请求失败({retries}次): {e}")
                return None
    return None


def parse_list_items(html):
    """解析列表HTML，提取标题、URL、日期"""
    items = re.findall(r'<li class="cf">(.*?)</li>', html, re.DOTALL)
    results = []
    for item in items:
        url_m = re.search(r'href="([^"]+)"', item)
        date_m = re.search(r'<span class="fr">([^<]+)</span>', item)
        title_m = re.search(r'<a[^>]*><i></i>([^<]+)</a>', item)
        if url_m and title_m:
            results.append({
                "title": title_m.group(1).strip(),
                "url": url_m.group(1),
                "date": date_m.group(1).strip() if date_m else ""
            })
    return results


def fetch_detail_content(url, retries=3):
    """从详情页提取正文HTML（含重试）"""
    for attempt in range(retries):
        try:
            if not url.startswith("http"):
                url = BASE_URL + url
            req = urllib.request.Request(url, headers=HEADERS)
            resp = urllib.request.urlopen(req, timeout=30)
            html = resp.read().decode("utf-8", errors="replace")
            break
        except Exception as e:
            if attempt < retries - 1:
                print(f"  [warn] 详情页失败(第{attempt+1}次): {str(e)[:60]}, 重试...")
                time.sleep(3)
            else:
                print(f"  [warn] 详情页请求失败({retries}次): {url[-60:]}, {e}")
                return ""
    else:
        return ""

    m = re.search(r'<div class="ty-content">([\s\S]*?)</div>\s*<div class="ty-close"', html)
    if m:
        content = m.group(1).strip()
        content = re.sub(r'\n\s*\n', '\n', content)
        content = re.sub(r'href="(?!http|/)', 'href="' + BASE_URL + '/', content)
        # 图片内嵌：下载并转为base64
        content = re.sub(r'src="(?!http|/|data:)', 'src="' + BASE_URL + '/', content)
        content = inline_images(content, referer=url)
        return content
    return ""


def inline_images(html_content, referer=None):
    """将img标签的src转为绝对URL（去掉base64内嵌）"""
    if "<img" not in html_content:
        return html_content  # 无图片，快速返回

    def _resolve_url(img_url):
        """将相对路径解析为绝对URL"""
        if img_url.startswith("data:"):
            return img_url, None
        full_url = img_url
        if not full_url.startswith("http"):
            full_url = (BASE_URL if full_url.startswith("/") else BASE_URL + "/") + full_url
        return img_url, full_url


def crawl(incremental_days=None):
    import sqlite3

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    c.execute("PRAGMA journal_mode=WAL")

    c.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT,
            page_url TEXT UNIQUE,
            content TEXT,
            publish_date TEXT,
            site_name TEXT DEFAULT '',
            crawl_time TEXT DEFAULT (datetime('now', '+8 hours')),
            similar TEXT
        )
    """)
    conn.commit()

    existing_urls = set()
    for row in c.execute("SELECT page_url FROM gov_raw WHERE site_name = ?", (SITE_NAME,)):
        existing_urls.add(row[0])

    if incremental_days:
        inc_cutoff = (datetime.now() - timedelta(days=incremental_days)).strftime("%Y-%m-%d")
        print(f"[info] 增量模式: 最近 {incremental_days} 天 >= {inc_cutoff}")
    else:
        inc_cutoff = CUTOFF_DATE
        print(f"[info] 全量模式: 截止 {CUTOFF_DATE}")

    data = fetch_api_page(1)
    if not data or not data.get("success"):
        print("[error] 无法获取数据，退出")
        conn.close()
        return

    html = data.get("data", {}).get("html", "")
    items = parse_list_items(html)
    if not items:
        print("[warn] 第1页无数据，退出")
        conn.close()
        return

    total_added = 0
    total_skipped = 0
    cutoff_reached = False
    page = 1

    while True:
        if cutoff_reached:
            break
        print(f"[info] 第{page}页 ({len(items)}条)...")

        for item in items:
            pub = item["date"][:10]
            if pub and pub < CUTOFF_DATE and not incremental_days:
                cutoff_reached = True
                print(f"  [info] 截止日期 {CUTOFF_DATE} (当前 {pub})，跳过后续")
                break
            if incremental_days and pub and pub < inc_cutoff:
                cutoff_reached = True
                print(f"  [info] 增量截止 {inc_cutoff} (当前 {pub})，跳过后续")
                break

            url = item["url"]
            if not url.startswith("http"):
                url = BASE_URL + url

            if url in existing_urls:
                total_skipped += 1
                continue

            content = fetch_detail_content(url)
            if not content:
                print(f"  [warn] 空正文: {item['title'][:40]}")
                total_skipped += 1
                continue

            try:
                c.execute(
                    "INSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, site_name) VALUES (?, ?, ?, ?, ?)",
                    (item["title"], url, content, pub, SITE_NAME),
                )
                if c.rowcount > 0:
                    total_added += 1
                    existing_urls.add(url)
                conn.commit()
            except Exception as e:
                print(f"  [error] DB写入失败: {e}")
                conn.rollback()

        if cutoff_reached:
            break

        page += 1
        data = fetch_api_page(page)
        if not data or not data.get("success"):
            print(f"[info] 第{page}页无响应，结束")
            break
        html = data.get("data", {}).get("html", "")
        items = parse_list_items(html)
        if not items:
            print(f"[info] 第{page}页无数据，结束")
            break

    conn.close()
    print(f"[done] 完成: 新增 {total_added} 条, 跳过 {total_skipped} 条")


if __name__ == "__main__":
    args = sys.argv[1:]
    if args and args[0].isdigit():
        crawl(incremental_days=int(args[0]))
    else:
        crawl()