#!/usr/bin/env python3
"""
台江县人民政府 - 政民互动/在线调查爬虫
JSON API 直取，含详情页 URL
"""
import sys, os, re, time, json
from datetime import datetime, timezone, timedelta
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "台江县-在线调查"
API_URL = "https://www.gztaijiang.gov.cn/IGI/opinion/web/list"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0",
    "Referer": "https://www.gztaijiang.gov.cn/zmhd/zjdc/"
}


def crawl(incremental=False, limit=None):
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")

    r = requests.get(API_URL, params={"siteId": 502077, "pageIndex": 1, "pageSize": 1000},
                     headers=HEADERS, timeout=30)
    r.raise_for_status()
    data = r.json()
    items = data.get("datas", {}).get("data", [])
    print(f"📋 API 返回 {len(items)} 条")

    parsed = []

    for item in items:
        # 日期解析
        begin_ts = item.get("beginTime", "0")
        try:
            dt = datetime.fromtimestamp(int(begin_ts) / 1000)
            date_str = dt.strftime("%Y-%m-%d")
        except:
            date_str = ""
            dt = None

        # 3年过滤
        if dt and date_str < THREE_YEARS_AGO:
            continue

        title = (item.get("theme") or "").strip()
        if not title:
            continue

        html = (item.get("htmlContent") or "").strip()
        if not html:
            continue

        # 净化 HTML（去掉 script/style）
        content = re.sub(r'<(script|style)[^>]*>.*?</\1>', '', html, flags=re.DOTALL|re.I)
        text_len = len(re.sub(r'<[^>]+>', '', content).strip())
        if text_len < 10:
            continue

        # 详情页 URL（唯一！修复旧版全部相同 URL 导致的唯一索引冲突）
        pub_url = (item.get("publishUrl") or "").strip()
        if not pub_url or pub_url == "/":
            page_url = f"https://www.gztaijiang.gov.cn/zmhd/zjdc/?id={item.get('id','0')}"
        else:
            page_url = pub_url if pub_url.startswith("http") else f"https://www.gztaijiang.gov.cn{pub_url}"

        summary = re.sub(r'<[^>]+>', '', content).strip()[:200]
        dr = int(dt.timestamp()) if dt else int(time.time())

        parsed.append({
            "site_name": SITE_NAME,
            "title": title,
            "url": page_url,
            "content": content,
            "pub_date": date_str,
            "summary": summary,
            "tags": SITE_NAME,
        })

        if limit and len(parsed) >= limit:
            break

    print(f"\n📊 过滤后: {len(parsed)} 条")

    if parsed:
        push_to_searchdb(parsed, "taijiang")
        print(f"\n✅ 完成! 共 {len(parsed)} 条入库")
    else:
        print("\n⏭ 无新数据")


if __name__ == "__main__":
    import requests
    incremental = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    limit = None
    for i, a in enumerate(sys.argv):
        if a == "--limit" and i + 1 < len(sys.argv):
            limit = int(sys.argv[i + 1])
    t0 = time.time()
    crawl(incremental=incremental, limit=limit)
    print(f"⏱ 耗时: {time.time() - t0:.1f}s")