#!/usr/bin/env python3
"""
禹王台区通知公告 (yuwangtai.gov.cn) 爬虫
https://www.yuwangtai.gov.cn/kfsywtqwz/tzggx/pc/list.html

CMS: 数融 CMS, API 分页查询，正文已包含在返回中。
规则: 前 5 页，标题含"项目"，近 3 年
"""

import sys, os, re, json, time
from datetime import datetime, timezone, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "禹王台区通知公告"
API_URL = "https://www.yuwangtai.gov.cn/queryList"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
MAX_PAGES = 5
PAGE_SIZE = 15

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36",
    "Content-Type": "application/json",
}

import requests
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


def fetch_page(page=1):
    try:
        r = requests.post(API_URL, json={
            "current": page,
            "pageSize": PAGE_SIZE,
            "webSiteCode": ["kfsywtqwz"],
            "channelCode": ["tzggx"],
        }, headers=HEADERS, timeout=30, verify=False)
        return r.json() if r.status_code == 200 else None
    except Exception as e:
        print(f"  ❌ 请求失败: {e}")
        return None


def norm_date(d):
    if not d:
        return ""
    m = re.search(r"(\d{4})-(\d{1,2})-(\d{1,2})", str(d))
    if m:
        y, mo, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
        return f"{y:04d}-{mo:02d}-{day:02d}" if 1990 <= y <= 2099 else ""
    return ""


def clean_html(html):
    if not html:
        return ""
    cleaned = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.I)
    cleaned = re.sub(r"<style[^>]*>.*?</style>", "", cleaned, flags=re.DOTALL | re.I)
    cleaned = re.sub(r"<p>\s*<br\s*/?>\s*</p>", "", cleaned)
    if len(cleaned) > 30000:
        cleaned = cleaned[:30000] + "..."
    return cleaned.strip()


def main():
    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"   近3年截止: {THREE_YEARS_AGO}")
    print(f"   最多 {MAX_PAGES} 页, 标题含「项目」")
    print(f"{'='*50}")

    all_items = []
    seen_urls = set()

    for page in range(1, MAX_PAGES + 1):
        print(f"\n📄 第 {page} 页...", end=" ", flush=True)
        data = fetch_page(page)
        if not data or "data" not in data:
            print("❌ 无返回")
            break

        results = data["data"].get("results", [])
        if not results:
            print("0 条")
            break

        page_matched = 0
        for item in results:
            src = item.get("source", {})
            title = (src.get("showTitle") or src.get("title") or "").strip()
            title = re.sub(r"<[^>]+>", "", title).strip()

            # 过滤: 标题含"项目"
            if "项目" not in title:
                continue

            # 日期
            pub_date = norm_date(src.get("pubDate", ""))

            # 过滤: 近 3 年
            if pub_date and pub_date < THREE_YEARS_AGO:
                continue

            # URL
            urls_str = src.get("urls", "{}")
            try:
                urls = json.loads(urls_str) if isinstance(urls_str, str) else urls_str
                page_url = urls.get("pc", "")
                if page_url and not page_url.startswith("http"):
                    page_url = "https://www.yuwangtai.gov.cn" + page_url
            except:
                page_url = f"https://www.yuwangtai.gov.cn/kfsywtqwz/tzggx/pc/content/content_{src.get('id', '')}.html"
            page_url = page_url.replace("\\/", "/")

            if not page_url or page_url in seen_urls:
                continue
            seen_urls.add(page_url)

            # 正文 (API 里已有全文)
            cobj = src.get("content", {})
            content = cobj.get("content", "") if isinstance(cobj, dict) else (cobj if isinstance(cobj, str) else "")
            content = clean_html(content)

            summary = re.sub(r"<[^>]+>", " ", content).strip()
            summary = re.sub(r"\s+", " ", summary)[:300]

            all_items.append({
                "site_name": SITE_NAME,
                "title": title,
                "url": page_url,
                "content": content,
                "pub_date": pub_date,
                "summary": summary,
                "tags": SITE_NAME,
            })
            page_matched += 1

        print(f"✅ 本页匹配 {page_matched} 条")

    print(f"\n{'─'*50}")
    print(f"📊 总计匹配: {len(all_items)} 条")

    if not all_items:
        print("⏭️ 无数据，跳过推送")
        return

    print(f"\n📤 推送至服务器...")
    push_to_searchdb(all_items, "yuwangtai_tzgg")

    print(f"\n{'='*50}")
    print(f"✅ 完成! 共 {len(all_items)} 条已推送至服务器")
    print(f"{'='*50}")


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time() - t0:.1f}s\n")