#!/usr/bin/env python3
"""北安市人民政府 - 通知公告 爬虫
   https://www.hljba.gov.cn/bas/c100747/list.shtml
   CMS: ZG前端框架, AJAX API分页, 直接返回正文
   API: /common/search/{channelId}?_isAgg=false&_isJson=true&_pageSize=15&page=N
"""
import requests, json, sqlite3, os, sys, time, re
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "北安市-通知公告"
BASE_URL = "https://www.hljba.gov.cn"
API_URL = BASE_URL + "/common/search/7ccb692175054df19eb53e5b910162c9"
PAGE_SIZE = 15
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


def fetch_api(page):
    url = f"{API_URL}?_isAgg=false&_isJson=true&_pageSize={PAGE_SIZE}&_template=index&_rangeTimeGte=&_channelName=&page={page}"
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        if r.status_code != 200:
            print(f"  [warn] API返回 {r.status_code}")
            return None, 0
        data = r.json()
        results = data.get("data", {}).get("results", [])
        total = data.get("data", {}).get("total", 0)
        return results, total
    except Exception as e:
        print(f"  [warn] API请求失败 {page}: {str(e)[:60]}")
        return None, 0


def extract_content(item):
    """从API返回的item中提取正文（使用contentHtml保留段落结构+附件）"""
    raw = item.get("contentHtml", "") or item.get("content", "")
    if not raw:
        return ""
    raw = raw.replace("&ensp;", " ").replace("&nbsp;", " ").replace("&lt;", "<").replace("&gt;", ">")
    raw = raw.replace("&amp;", "&").replace("&quot;", '"')
    # 剥离内联标签
    raw = re.sub(r'</?(b|span|font|strong|u|i|em|o:p)[^>]*>', '', raw, flags=re.IGNORECASE)
    raw = re.sub(r'<br\s*/?>', '\n', raw)

    parts = []
    for p in re.findall(r'<p[^>]*>(.*?)</p>', raw, re.DOTALL):
        txt = re.sub(r'<[^>]+>', '', p).strip()
        if txt:
            parts.append(txt)
    if not parts:
        txt = re.sub(r'<[^>]+>', '', raw).strip()
        if txt:
            parts = [txt]

    content_text = "\n\n".join(parts)

    # 提取附件（resList）
    res_list = item.get("resList", [])
    if res_list:
        attachment_lines = ["\n\n**附件：**"]
        for rl in res_list:
            fname = rl.get("fileName", "")
            fpath = rl.get("filePathNew", "") or rl.get("filePath", "")
            if fname and fpath:
                if fpath.startswith("/"):
                    furl = BASE_URL + fpath
                else:
                    furl = BASE_URL + "/" + fpath
                attachment_lines.append(f"[{fname}]({furl})")
        if len(attachment_lines) > 1:
            content_text += "\n".join(attachment_lines)

    return content_text


def main():
    INCREMENTAL = "--full" not in sys.argv
    mode = "增量" if INCREMENTAL else "全量"
    print(f"=== {SITE_NAME} === (模式: {mode}, cutoff={CUTOFF})")

    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()
    new_total = 0
    dup_total = 0
    old_total = 0
    err_total = 0

    results, total_count = fetch_api(1)
    if results is None:
        print("无法获取数据，退出")
        conn.close()
        return
    total_pages = (total_count + PAGE_SIZE - 1) // PAGE_SIZE
    print(f"总条数: {total_count}, 总页数: {total_pages}")

    for page in range(1, total_pages + 1):
        if page > 1:
            results, _ = fetch_api(page)
            if not results:
                continue
        page_new = 0
        for item in results:
            date_str = (item.get("publishedTimeStr") or "")[:10]
            if INCREMENTAL and date_str and date_str < CUTOFF:
                old_total += 1
                continue
            url = item.get("url", "")
            if not url.startswith("http"):
                url = BASE_URL + url
            title = item.get("title", "")
            c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=?", (url,))
            if c.fetchone()[0] > 0:
                dup_total += 1
                continue
            content = extract_content(item)
            if not content or len(content) < 50:
                print(f"  [warn] 正文过短: {title[:40]}...")
                err_total += 1
                continue
            pd = date_str if date_str else datetime.now().strftime("%Y-%m-%d")
            c.execute(
                "INSERT INTO gov_raw (page_url, title, content, site_name, publish_date, source_url, category) "
                "VALUES (?, ?, ?, ?, ?, ?, ?)",
                (url, title, content, SITE_NAME, pd, url, "通知公告")
            )
            page_new += 1
            new_total += 1
        conn.commit()
        print(f"  [page {page}/{total_pages}] +{page_new} new, total: {new_total}")
        time.sleep(0.3)

    conn.close()
    print(f"\n[完成] {SITE_NAME}: 新增 {new_total} 条, 旧跳过 {old_total}, 重复 {dup_total}, 错误 {err_total}")


if __name__ == "__main__":
    main()