#!/usr/bin/env python3
"""
东乡区人民政府 - 信息公开 (jxfzdx.gov.cn)
TRS CMS 系统
列表: /col/col21311/index.html  → 分页 index_N.html
详情: /art/YYYY/MM/DD/art_XXXX_XXXXXXX.html
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "东乡区人民政府"
BASE_URL = "http://www.jxfzdx.gov.cn"
LIST_PATH = "/col/col21311/index.html"
MAX_PAGES = 5
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"}


def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"❌ fetch fail: {e}")
        return None


def parse_list(html):
    items = []
    for m in re.finditer(r'<record>.*?<!\[CDATA\[(.*?)\]\]>.*?</record>', html, re.DOTALL):
        cdata = m.group(1)
        a = re.search(r'<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"', cdata)
        date_m = re.search(r'<b[^>]*>(\d{4}-\d{2}-\d{2})</b>', cdata)
        if a:
            href = a.group(1)
            title = a.group(2).strip()
            date_str = date_m.group(1) if date_m else ""
            if not href.startswith('http'):
                href = BASE_URL + href
            if title:
                items.append((title, href, date_str))
    return items


def get_page_url(page):
    if page == 1:
        return BASE_URL + LIST_PATH
    return f"{BASE_URL}/col/col21311/index_{page}.html"


def fetch_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None
    result = {}
    # 标题
    m = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
    if m:
        result["title"] = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    if not result.get("title"):
        m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if m:
            result["title"] = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    # 日期
    for pat in [r'发布日期[：:]\s*(\d{4}-\d{2}-\d{2})',
                r'发布时间[：:]\s*(\d{4}-\d{2}-\d{2})',
                r'(\d{4}-\d{2}-\d{2})']:
        if not result.get("publish_date"):
            m = re.search(pat, html)
            if m:
                result["publish_date"] = m.group(1)
    # 正文 — TRS 内容域
    content = ""
    for cls in ['TRS_Editor', 'content', 'article-content', 'article_content',
                'xxgk-content', 'Custom_UnionStyle', 'con_text']:
        m = re.search(r'<div[^>]*class="[^"]*' + cls + r'[^"]*"[^>]*>(.*?)</div>\s*(?:<div|</div>|<!--|$)', html, re.DOTALL)
        if m and len(m.group(1)) > 50:
            content = m.group(1).strip()
            break
    if not content or len(content) < 50:
        m = re.search(r'<div[^>]*id="zoom"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<!--.*?-->', '', content)  # TRS/HTML注释
        content = re.sub(r'<meta[^>]*>', '', content)  # meta标签
        result["content"] = content.strip()
    return result.get("title"), result.get("content"), result.get("publish_date")


def main():
    import argparse
    parser = argparse.ArgumentParser(description=f"爬虫: {SITE_NAME}")
    parser.add_argument("--limit", type=int, help="测试：只处理N条详情")
    parser.add_argument("--incremental", action="store_true", help="增量模式：仅爬第1页")
    args = parser.parse_args()
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")

    all_list = []
    max_pages = 1 if args.incremental else MAX_PAGES
    for page in range(1, max_pages + 1):
        url = get_page_url(page)
        print(f"\n📄 第 {page} 页...", end=" ", flush=True)
        html = fetch(url)
        if not html:
            print("❌")
            break
        items = parse_list(html)
        if not items:
            print("0 条")
            break
        print(f"✅ {len(items)} 条")
        all_list.extend(items)
        if args.incremental:
            break

    print(f"\n📊 列表总计: {len(all_list)} 条")
    all_items, seen = [], set()
    for i, (title, url, list_date) in enumerate(all_list):
        if url in seen:
            continue
        seen.add(url)
        if args.limit and len(all_items) >= args.limit:
            break
        print(f"  [{len(all_items)+1}/{len(all_list)}] {title[:50]}...", end=" ", flush=True)
        dt, content, date = fetch_detail(url)
        if dt:
            title = dt
        final_date = date or list_date
        summary = re.sub(r'<[^>]+>', ' ', content or '').strip()[:300]
        summary = re.sub(r'\s+', ' ', summary)
        all_items.append({
            "site_name": SITE_NAME, "title": title, "url": url,
            "content": content or "", "pub_date": final_date,
            "summary": summary, "tags": SITE_NAME,
        })
        print("✅")
        time.sleep(0.3)

    if all_items:
        push_to_searchdb(all_items, "jxfzdx")
    print(f"\n✅ 完成! 共 {len(all_items)} 条")


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")