#!/usr/bin/env python3
"""爬取 sthjj.kaifeng.gov.cn - 批复公告"""
import requests
from bs4 import BeautifulSoup
import sqlite3
import os
import re
import json
import sys
from datetime import datetime

BASE_URL = "http://sthjj.kaifeng.gov.cn"
SITE_NAME = "sthjj.kaifeng.gov.cn-批复公告"
CUTOFF_DATE = "2023-06-16"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
CHANNEL_CODE = "cpfgg"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": "http://sthjj.kaifeng.gov.cn/",
}

def fetch_article_list():
    """从 articleList.json 获取文章列表"""
    url = f"{BASE_URL}/kfssthjj/{CHANNEL_CODE}/{CHANNEL_CODE}-articleList.json"
    resp = requests.get(url, headers=HEADERS, timeout=30)
    resp.encoding = "utf-8"
    data = json.loads(resp.text)
    items = []
    for item in data:
        title = item.get("showTitle") or item.get("title", "")
        # 清理可能的HTML标签
        title = re.sub(r'<[^>]+>', '', title).strip()
        
        pub_date = item.get("pubDate", "")[:10]  # "2026-06-22 11:06" -> "2026-06-22"
        
        urls = item.get("urls", "{}")
        try:
            url_data = json.loads(urls)
            detail_url = url_data.get("pc", "")
        except:
            detail_url = ""
        
        if detail_url and not detail_url.startswith("http"):
            detail_url = BASE_URL + detail_url
        
        items.append({
            "title": title,
            "url": detail_url,
            "publish_date": pub_date,
        })
    return items

def get_detail(url):
    """获取详情页的正文HTML + 附件"""
    resp = requests.get(url, headers=HEADERS, timeout=30)
    resp.encoding = "utf-8"
    soup = BeautifulSoup(resp.text, "html.parser")
    
    content_parts = []
    
    # 正文
    div = soup.select_one("div.article-content.downmainbox")
    if div:
        # 附件链接补全
        for a in div.find_all("a"):
            href = a.get("href", "")
            if href and not href.startswith("http"):
                a["href"] = BASE_URL + href
        content_parts.append(str(div))
    
    # 备选：show_con 样式
    if not content_parts:
        div = soup.select_one("div.show_con")
        if div:
            content_parts.append(str(div))
    
    if content_parts:
        return "".join(content_parts)
    return ""

def crawl():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    
    c.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT,
            page_url TEXT UNIQUE,
            content TEXT,
            publish_date TEXT,
            site_name TEXT DEFAULT '',
            crawl_time TEXT DEFAULT (datetime('now', '+8 hours')),
            similar TEXT
        )
    """)
    conn.commit()
    
    print("获取文章列表...")
    items = fetch_article_list()
    print(f"共 {len(items)} 条")
    
    total_added = 0
    total_skipped = 0
    
    for item in items:
        pub = item["publish_date"]
        if pub and pub < CUTOFF_DATE:
            print(f"  截止日期 {CUTOFF_DATE} (当前 {pub})，跳过")
            break
        
        if not item["url"]:
            print(f"    跳过（无URL）: {item['title'][:30]}")
            total_skipped += 1
            continue
        
        c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
        if c.fetchone():
            total_skipped += 1
            continue
        
        content = get_detail(item["url"])
        if not content:
            print(f"    警告: 空正文 {item['title'][:30]}")
            total_skipped += 1
            continue
        
        c.execute(
            "INSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, site_name) VALUES (?, ?, ?, ?, ?)",
            (item["title"], item["url"], content, pub, SITE_NAME)
        )
        if c.rowcount > 0:
            total_added += 1
        conn.commit()
    
    conn.close()
    print(f"完成: 新增 {total_added} 条, 跳过 {total_skipped} 条")

if __name__ == "__main__":
    crawl()