#!/usr/bin/env python3
"""宜丰县人民政府 - 通知公告 爬虫
API: POST /searchManuscript
"""

import json
import re
from datetime import datetime, date
from urllib.parse import urljoin
import requests
import sqlite3
import os

BASE_URL = "http://www.jxyf.gov.cn"
API_URL = urljoin(BASE_URL, "/searchManuscript")
SITE_NAME = "宜丰县-通知公告"
CHANNEL_ID = "1996835172745060352"
WEBSITE_CODE = "yfxrmzf"
CUTOFF_DATE = date(2023, 6, 16)
DB_PATH = os.environ.get("DB_PATH", "/root/search.db")

def get_conn():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.row_factory = sqlite3.Row
    return conn

def clean_title(title):
    if not title:
        return ""
    title = re.sub(r'<[^>]+>', '', title)
    title = title.replace('&nbsp;', ' ').strip()
    return re.sub(r'\s+', ' ', title)

def extract_pub_date(pub_date_str):
    if not pub_date_str:
        return None
    m = re.match(r'(\d{4}-\d{2}-\d{2})', str(pub_date_str))
    return m.group(1) if m else None

def fetch_page(page):
    params = {"current": page, "pageSize": 20, "webSiteCode": [WEBSITE_CODE], "channelTreeIds": [CHANNEL_ID]}
    try:
        resp = requests.post(API_URL, json=params, timeout=30, headers={"User-Agent": "Mozilla/5.0"})
        resp.raise_for_status()
        return resp.json().get("data", {})
    except Exception as e:
        print("  请求第{}页失败: {}".format(page, e), flush=True)
        return None

def parse_article(item):
    try:
        article_id = str(item.get("id", ""))
        if not article_id:
            return None
        title = clean_title(item.get("showTitle") or item.get("title", ""))
        pd_str = item.get("pubDate", "")
        pub_date = extract_pub_date(pd_str)
        if not pub_date:
            return None
        pub_dt = datetime.strptime(pub_date, "%Y-%m-%d").date()
        if pub_dt < CUTOFF_DATE:
            return None
        content_obj = item.get("content", {})
        content = content_obj.get("content", "") if isinstance(content_obj, dict) else ""
        urls_str = item.get("urls", "{}")
        try:
            urls = json.loads(urls_str) if isinstance(urls_str, str) else urls_str
        except:
            urls = {}
        detail_url = urls.get("pc", "")
        if detail_url and not detail_url.startswith("http"):
            detail_url = urljoin(BASE_URL, detail_url)
        page_url = detail_url or "{}/article/{}".format(BASE_URL, article_id)
        source = item.get("contentSource", "")
        af_str = item.get("articleFiles", "[]")
        try:
            afs = json.loads(af_str) if isinstance(af_str, str) else af_str
        except:
            afs = []
        attachments = []
        for af in afs:
            if isinstance(af, dict):
                fname = af.get("fileName", "")
                fpath = af.get("filePath", "") or af.get("url", "")
                if fpath and not fpath.startswith("http"):
                    fpath = urljoin(BASE_URL, fpath)
                if fname and fpath:
                    attachments.append('<a href="{}" target="_blank">{}</a>'.format(fpath, fname))
        if attachments:
            content += '<div class="attachments"><p><strong>附件：</strong></p>' + '<br>'.join(attachments) + '</div>'
        return {"site_name": SITE_NAME, "title": title, "page_url": page_url,
                "content": content, "pub_date": pub_date, "source": source}
    except Exception as e:
        print("  解析异常: {}".format(e), flush=True)
        return None

def main():
    print("=== {} 爬虫 ===".format(SITE_NAME), flush=True)
    print("数据库: {}".format(DB_PATH), flush=True)
    conn = get_conn()
    cursor = conn.cursor()
    page = 1
    total_fetched = total_new = total_skipped = total_o3 = 0
    while True:
        print("\n--- 第{}页 ---".format(page), flush=True)
        data = fetch_page(page)
        if not data:
            break
        results = data.get("results", [])
        if not results:
            break
        print("  返回 {} 条".format(len(results)), flush=True)
        all_o3 = True
        for item in results:
            total_fetched += 1
            parsed = parse_article(item)
            if parsed is None:
                total_o3 += 1
                continue
            all_o3 = False
            try:
                cursor.execute("""
                    INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, date_rank)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                """, (parsed["site_name"], parsed["title"], parsed["page_url"],
                      parsed["content"], parsed["pub_date"], parsed["content"],
                      int(datetime.strptime(parsed["pub_date"], "%Y-%m-%d").timestamp())))
                if cursor.rowcount > 0:
                    total_new += 1
                else:
                    total_skipped += 1
            except Exception as e:
                print("  写入失败: {}".format(e), flush=True)
        conn.commit()
        if all_o3 and page > 1:
            print("  之后数据均超出3年范围", flush=True)
            break
        if len(results) < 20:
            break
        page += 1
    conn.close()
    print("\n=== 完成 ===")
    print("总抓取: {} | 新增: {} | 跳过: {} | 超3年: {}".format(total_fetched, total_new, total_skipped, total_o3))

if __name__ == "__main__":
    main()