#!/usr/bin/env python3
"""
黄石市生态环境局-民意征集 (sthjj.huangshi.gov.cn/gzhd/myzj/)
IGI 平台 JSON API: /IGI/opinion/web/list + /IGI/opinion/web/view
"""
import sys, os, re, time, json
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "黄石市生态环境局-民意征集"
BASE_URL = "https://sthjj.huangshi.gov.cn"
LIST_API = BASE_URL + "/IGI/opinion/web/list?siteId=39&pageIndex={}&pageSize=12"
DETAIL_API = BASE_URL + "/IGI/opinion/web/view?siteId=39&id={}"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch_json(url):
    for _ in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=20)
            if r.status_code == 200:
                return r.json()
        except:
            pass
        time.sleep(2)
    return None

def main(incremental=False, limit=None):
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")

    all_items, seen_ids = [], set()
    stop = False

    for pg in range(1, 100):
        if stop:
            break
        data = fetch_json(LIST_API.format(pg))
        if not data or data.get("statusCode") != 200:
            print("❌ 列表API失败或已无数据")
            break

        items = data.get("datas", {}).get("data", [])
        if not items:
            break

        # 检查本页是否所有条目都超出3年（若整页都太旧则停止）
        all_old = True
        latest_on_page = None
        for item in items:
            ts = int(str(item.get("publicTime", "0")).replace(".0", ""))
            if ts:
                dt = datetime.fromtimestamp(ts / 1000).strftime("%Y-%m-%d")
                if latest_on_page is None or dt > latest_on_page:
                    latest_on_page = dt
                if dt >= THREE_YEARS_AGO:
                    all_old = False

        if all_old and latest_on_page:
            print(f"  ⏹ 第{pg}页全部早于 {THREE_YEARS_AGO}（最晚 {latest_on_page}），停止")
            break

        dates = []
        for item in items:
            ts = int(str(item.get("publicTime", "0")).replace(".0", ""))
            if ts:
                dates.append(datetime.fromtimestamp(ts / 1000).strftime("%Y-%m-%d"))
        print(f"  📄 第{pg}页 ({min(dates) if dates else '?'}~{max(dates) if dates else '?'})")

        for item in items:
            item_id = int(item["id"])
            if item_id in seen_ids:
                continue
            seen_ids.add(item_id)

            theme = item.get("theme", "").strip()
            if not theme:
                continue

            pub_ts = int(str(item.get("publicTime", "0")).replace(".0", ""))
            pub_date = datetime.fromtimestamp(pub_ts / 1000).strftime("%Y-%m-%d") if pub_ts else ""

            if pub_date and pub_date < THREE_YEARS_AGO:
                continue

            # 获取详情正文
            print(f"  [{len(all_items)+1}] {theme[:40]}...", end=" ", flush=True)
            detail = fetch_json(DETAIL_API.format(item_id))
            content = ""
            if detail and detail.get("statusCode") == 200:
                dd = detail.get("datas", {})
                content = dd.get("htmlContent", "") or ""
                # 也可取附件
                attachs = dd.get("attachsList") or []
                if attachs:
                    for att in attachs:
                        if isinstance(att, dict) and att.get("filePath"):
                            content += f'<p><a href="{att["filePath"]}">{att.get("fileName", "附件")}</a></p>'

            summary = re.sub(r"<[^>]+>", " ", content).strip()[:300]
            summary = re.sub(r"\s+", " ", summary) if summary else theme
            print(f"✅ ({len(content)} chars)")

            all_items.append({
                "site_name": SITE_NAME,
                "title": theme,
                "url": DETAIL_API.format(item_id),
                "source_url": DETAIL_API.format(item_id),
                "content": content,
                "pub_date": pub_date,
                "summary": summary,
                "tags": SITE_NAME,
            })

            time.sleep(0.5)
            if limit and len(all_items) >= limit:
                stop = True
                break

    if all_items:
        push_to_searchdb(all_items, "huangshi_myzj")
        print(f"\n✅ 完成! 共 {len(all_items)} 条")
    else:
        print("\n⏭ 无新数据")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--incremental", action="store_true")
    parser.add_argument("--limit", type=int)
    args = parser.parse_args()
    incremental = args.incremental or (len(sys.argv) > 1 and sys.argv[1] == "1")
    t0 = time.time()
    main(incremental=incremental, limit=args.limit)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")