#!/usr/bin/env python3
"""
恒山区人民政府-通知公告 (jixihengshan.gov.cn)
列表: AJAX API /common/search/{channelId}
详情: /hsq/{code}/YYYYMM/c06_NNNNNN.shtml
正文: div#zoomcon + UCAPCONTENT, 标题: h1.article_title
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "恒山区人民政府-通知公告"
BASE_URL = "https://www.jixihengshan.gov.cn"
CHANNEL_ID = "20b283cb2e2348d3888cc44525ba5ecb"
API_URL = f"{BASE_URL}/common/search/{CHANNEL_ID}"
PAGE_SIZE = 20
MAX_PAGES = 5
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"}

def fetch_list_page(page):
    params = {
        "_isAgg": "false", "_isJson": "true", "_pageSize": str(PAGE_SIZE),
        "_template": "index", "_rangeTimeGte": "", "_channelName": "",
        "page": str(page)
    }
    resp = requests.get(API_URL, params=params, headers=HEADERS, timeout=20, verify=False)
    data = resp.json()
    results = data.get("data", {}).get("results", [])
    total = data.get("data", {}).get("total", 0)
    items = []
    for r in results:
        title = r.get("title", "").strip()
        url = r.get("url", "")
        pub_time = r.get("publishedTimeStr", "")[:10]
        if title and url:
            if not url.startswith("http"):
                url = BASE_URL + url
            items.append((title, url, pub_time))
    return items, total

def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        html = r.text
    except:
        return None, None, None

    result = {}

    # 标题 + 日期从 articletitle
    m = re.search(r'class="articletitle"[^>]*>\s*(.*?)</div>', html, re.DOTALL)
    if m:
        block = m.group(1)
        title_m = re.match(r'([^<]*)', block)
        if title_m:
            result["title"] = title_m.group(1).strip()
        date_m = re.search(r'发布时间[：:]\s*(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日', block)
        if date_m:
            result["publish_date"] = f"{date_m.group(1)}-{int(date_m.group(2)):02d}-{int(date_m.group(3)):02d}"

    if not result.get("title"):
        m = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
        if m:
            result["title"] = re.sub(r"<[^>]+>", "", m.group(1)).strip()

    if not result.get("publish_date"):
        m = re.search(r'(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日', html)
        if m:
            result["publish_date"] = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"

    # 正文
    content = None
    m = re.search(r'id="zoomcon"[^>]*>(.*?)</div>\s*(?:<div|<!--|$)', html, re.DOTALL)
    if m:
        content = m.group(1).strip()

    if content:
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL | re.I)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL | re.I)
        content = re.sub(r"</?UCAPCONTENT[^>]*>", "", content, flags=re.DOTALL | re.I)
        result["content"] = content

    return result.get("title"), result.get("content"), result.get("publish_date")


def main(incremental=False, limit=None):
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")

    all_list = []
    for page in range(1, MAX_PAGES + 1):
        print(f"📄 第 {page} 页...", end=" ", flush=True)
        items, total = fetch_list_page(page)
        if not items:
            print("0 条")
            break
        print(f"✅ {len(items)} 条 (总计{total})")
        all_list.extend(items)
        if incremental or (page >= (total + PAGE_SIZE - 1) // PAGE_SIZE):
            break

    print(f"\n📊 列表总计: {len(all_list)} 条")

    filtered = [(t, u, d) for t, u, d in all_list if d >= THREE_YEARS_AGO]
    print(f"📅 近3年: {len(filtered)} 条")

    if incremental:
        filtered = filtered[:20]
    if limit:
        filtered = filtered[:limit]
        print(f"🧪 测试模式: 限 {len(filtered)} 条")

    all_items, seen = [], set()
    for i, (title, url, list_date) in enumerate(filtered):
        if url in seen:
            continue
        seen.add(url)
        print(f"  [{i+1}/{len(filtered)}] {title[:50]}...", end=" ", flush=True)
        dt_title, content, pub_date = fetch_detail(url)
        final_title = dt_title or title
        final_date = pub_date or list_date

        summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
        summary = re.sub(r"\s+", " ", summary)

        all_items.append({
            "site_name": SITE_NAME,
            "title": final_title,
            "url": url,
            "content": content or "",
            "pub_date": final_date,
            "summary": summary,
            "tags": SITE_NAME,
        })
        print("✅")
        time.sleep(0.3)

    if all_items:
        push_to_searchdb(all_items, "hengshanqu")
    else:
        print("  ⏭ 无数据，跳过推送")

    print(f"\n✅ 完成! 共 {len(all_items)} 条")


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--incremental", action="store_true")
    parser.add_argument("--limit", type=int)
    args = parser.parse_args()

    t0 = time.time()
    incremental = args.incremental or (len(sys.argv) > 1 and sys.argv[1] == '1')
    main(incremental=incremental, limit=args.limit)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")