#!/usr/bin/env python3
import os
"""
信阳市生态环境局-通知公告 (sthjj.xinyang.gov.cn)
列表: /xxdt/tzgg/ — 单页，<a href=.. title=..><span>标题</span><em>日期</em></a>
详情: 正文 <div class="content-txt">
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "信阳市生态环境局-通知公告"
BASE_URL = "https://sthjj.xinyang.gov.cn"
LIST_URL = f"{BASE_URL}/xxdt/tzgg/"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"}

def fetch(url):
    for _ in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
            r.encoding = "utf-8"
            if r.text.strip():
                return r.text
        except:
            pass
        time.sleep(2)
    return None

def parse_list(html):
    """提取列表 (title, url, date_str)"""
    items = []
    # 定位列表区域
    m = re.search(r'<div class="common-list division page-list">(.*?)<!--分页', html, re.DOTALL)
    if not m:
        return items
    list_html = m.group(1)
    
    # 每条: <a href="..." title="标题"><span>..</span><em>日期</em></a>
    for m2 in re.finditer(
        r'href="(http://sthjj\.xinyang\.gov\.cn/\d{4}/\d{2}-\d{2}/\d+\.html)"[^>]*title="([^"]*)"[^>]*>.*?<em>(\d{4}-\d{2}-\d{2})</em>',
        list_html, re.DOTALL
    ):
        items.append((m2.group(2).strip(), m2.group(1), m2.group(3)))
    return items

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None
    result = {}
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]+)"', html)
    if m:
        result["title"] = m.group(1).strip()
    if not result.get("title"):
        m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if m:
            t = m.group(1).replace(" - 信阳市生态环境局", "").strip()
            if t:
                result["title"] = t
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        result["pub_date"] = m.group(1)
    m = re.search(r'<div class="content-txt">(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL | re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL | re.I)
        result["content"] = content
    return result.get("title"), result.get("content"), result.get("pub_date")

def main(incremental=False, limit=None):
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")
    html = fetch(LIST_URL)
    if not html:
        print("❌ 获取失败")
        return
    items = parse_list(html)
    print(f"📄 列表: {len(items)} 条")
    items = [(t, u, d) for t, u, d in items if d >= THREE_YEARS_AGO]
    print(f"📅 近3年: {len(items)} 条")
    if incremental:
        items = items[:10]
        print(f"🔄 增量模式: 取前 {len(items)} 条")
    if limit:
        items = items[:limit]
    all_items, seen = [], set()
    for i, (title, url, list_date) in enumerate(items):
        if url in seen:
            continue
        seen.add(url)
        print(f"  [{i+1}/{len(items)}] {title[:50]}...", end=" ", flush=True)
        dt_title, content, pub_date = fetch_detail(url)
        final_title = dt_title or title
        final_date = pub_date or list_date
        summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
        summary = re.sub(r"\s+", " ", summary)
        all_items.append({
            "site_name": SITE_NAME,
            "title": final_title,
            "url": url,
            "content": content or "",
            "pub_date": final_date,
            "summary": summary,
            "tags": SITE_NAME,
        })
        print("✅")
        time.sleep(0.3)
    if all_items:
        push_to_searchdb(all_items, "xinyang_sthj")
    print(f"\n✅ 完成! 共 {len(all_items)} 条")
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--incremental", action="store_true")
    parser.add_argument("--limit", type=int)
    args = parser.parse_args()
    incremental = args.incremental or (len(sys.argv) > 1 and sys.argv[1] == "1")
    t0 = time.time()
    main(incremental=incremental, limit=args.limit)