#!/usr/bin/env python3
"""
察哈尔右翼前旗 - 通知公告 爬虫
CMS: NmghlPagination (API POST)
"""

import os, sys, re, json, time, hashlib
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed
import requests

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "察右前旗-通知公告"
API_URL = "https://www.cyqq.gov.cn/rmt/tmpl/api/site/5c921a9a07e246f189220e98ad430dec/channelid/6ad3b180cab6428da44de94d8b5c7aa6"
PARAMS = "5186ec152b6fdbf8f9925254821388e33f1bf26f04b05fd746e5575a4111485e17dfbdc2e13666401fa60e98f783528276699fa580e601be852d2feac106fe9e0fff607a5aed63d1003c194190192c80b4299bdcb70d22818e92317f41b240666b09a4cbad3eeb99aac8bd71796d1895a0bb14c482c642ae7d2ce1d41492fb7501ecd379aae27643ae4ed1342128b7ee8d4d00a342fbdc88038447bd54b40880c2193091eb0d5614b7b03e9e766cacfd5a5ab8639a392eb94a37b75ebd810216752dbf729663b703e62ae4d766adb9ebab1dd89d2f63d6d53584676662f91b09e4451ee8e1572b61ea3ae04b4a70201bd8945cd536a47376dce5f0db44e65bc2a1142513b9411021fd1dfbd83179d1a5"
BASE_URL = "https://www.cyqq.gov.cn/tzgg"
PAGE_SIZE = 10
TOTAL_COUNT = 139

CUTOFF_DATE = datetime.strptime("2023-06-18", "%Y-%m-%d")
MAX_WORKERS = 8

session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
})

def get_conn():
    import sqlite3
    return sqlite3.connect(DB_PATH, timeout=60)

def exists(conn, url):
    cur = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
    return cur.fetchone() is not None

def insert(conn, title, content, pub_date, page_url):
    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    date_rank = 0
    try:
        d = datetime.strptime(pub_date[:10], "%Y-%m-%d")
        date_rank = int(d.strftime("%Y%m%d"))
    except:
        pass
    try:
        cur = conn.execute("""INSERT OR IGNORE INTO gov_raw 
            (site_name, title, page_url, source_url, publish_date, date_rank, 
             summary, status, category, content, visits, tags)
            VALUES (?,?,?,?,?,?,?,?,?,?,0,'')""",
            (SITE_NAME, title, page_url, "", pub_date[:10], date_rank,
             "", "published", "通知公告", content))
        return cur.rowcount > 0
    except Exception as e:
        print(f"  [DB] {e}", file=sys.stderr)
        return False

def fetch_list_page(page):
    """Fetch one page of list via API."""
    body = {"currpage": page, "pagesize": PAGE_SIZE, "params": PARAMS}
    try:
        r = session.post(API_URL, json=body, timeout=15)
        data = r.json()
        if data.get("errcode") == 0:
            return data.get("data", [])
    except Exception as e:
        print(f"  [API] 第{page}页: {e}", file=sys.stderr)
    return []

def fetch_detail(item):
    """Fetch detail page."""
    docno = item.get("docno", "")
    url = f"{BASE_URL}/{docno}.html"
    try:
        r = session.get(url, timeout=15)
        r.encoding = 'utf-8'
        html = r.text
    except:
        return None

    soup = BeautifulSoup(html, 'html.parser')

    # Title
    title = item.get("title", "")
    meta_t = soup.find('meta', attrs={'name': 'ArticleTitle'})
    if meta_t and meta_t.get('content'):
        title = meta_t['content'].strip()

    # Date (API has publishTime as epoch ms)
    pub_date = ""
    pt = item.get("publishTime")
    if pt:
        pub_date = datetime.fromtimestamp(pt / 1000).strftime("%Y-%m-%d")
    if not pub_date:
        meta_d = soup.find('meta', attrs={'name': 'PubDate'})
        if meta_d and meta_d.get('content'):
            pub_date = meta_d['content'][:10]

    # Content
    content_div = soup.find('div', id='content')
    content = str(content_div) if content_div else ''

    if content:
        return {'url': url, 'title': title, 'pub_date': pub_date, 'content': content}
    return None

def main():
    total_pages = (TOTAL_COUNT + PAGE_SIZE - 1) // PAGE_SIZE
    print(f"[信息] 察右前旗-通知公告, API分页 {total_pages}页 ({TOTAL_COUNT}条)")

    # Collect list
    all_items = []
    for pi in range(1, total_pages + 1):
        items = fetch_list_page(pi)
        all_items.extend(items)
        print(f"  第{pi}/{total_pages}页 -> {len(items)}条 (累计{len(all_items)}条)")
        time.sleep(0.3)

    print(f"[列表] 共 {len(all_items)} 条")

    if not all_items:
        print("[完成] 无数据")
        return

    # Filter 3yr
    filtered = []
    skipped_old = 0
    for item in all_items:
        pt = item.get("publishTime", 0)
        if pt:
            d = datetime.fromtimestamp(pt / 1000)
            if d < CUTOFF_DATE:
                skipped_old += 1
                continue
        filtered.append(item)
    print(f"[过滤] 超3年: {skipped_old}, 有效: {len(filtered)}")

    if not filtered:
        print("[完成] 无新增")
        return

    # Check dupes
    conn = get_conn()
    new_items = []
    skipped_dup = 0
    for item in filtered:
        docno = item.get("docno", "")
        url = f"{BASE_URL}/{docno}.html"
        if exists(conn, url):
            skipped_dup += 1
        else:
            new_items.append(item)
    conn.close()
    print(f"[去重] 已存在: {skipped_dup}, 新增: {len(new_items)}")

    if not new_items:
        print("[完成] 无新增")
        return

    # Fetch details
    print(f"[详情] 获取{len(new_items)}条...")
    ok = 0
    conn = get_conn()
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:
        futures = {ex.submit(fetch_detail, item): item for item in new_items}
        for i, f in enumerate(as_completed(futures), 1):
            r = f.result()
            if r and insert(conn, r['title'], r['content'], r['pub_date'], r['url']):
                ok += 1
            if i % 20 == 0:
                print(f"  进度: {i}/{len(new_items)} (入库{ok})")
    conn.commit()
    conn.close()

    print(f"\n{'='*60}")
    print(f"站点: {SITE_NAME}")
    print(f"新增: {ok} 条")
    print(f"跳过(超3年): {skipped_old}")
    print(f"跳过(重复): {skipped_dup}")
    print(f"{'='*60}")

if __name__ == '__main__':
    main()