#!/usr/bin/env python3
"""
丰城市人民政府 - 公示公告 (jxfc.gov.cn)
列表使用 JSON API 分页
URL: /fcsrmzf/gsgg/pc/list.html
"""
import sys, os, re, time, json
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "丰城市人民政府-公示公告"
BASE_URL = "http://www.jxfc.gov.cn"
LIST_PATH = "/fcsrmzf/gsgg/pc/list.html"
MAX_PAGES = 5
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"}

# Common JSON API patterns for jxfc.gov.cn
API_URL = BASE_URL + "/fcsrmzf/gsgg/pc/list"
API_PARAMS = {"pageNum": 1, "pageSize": 20}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def fetch_json(api_url, params):
    try:
        r = requests.get(api_url, params=params, headers=HEADERS, timeout=20, verify=False)
        return r.json()
    except:
        return None

def parse_list_from_json(data):
    """从 JSON API 返回中提取文章"""
    items = []
    records = data.get("data", {}).get("list", data.get("list", data.get("data", [])))
    if isinstance(records, list):
        for rec in records:
            title = rec.get("title", rec.get("name", ""))
            href = rec.get("url", rec.get("href", rec.get("linkUrl", "")))
            date_str = rec.get("publishDate", rec.get("publish_date", rec.get("createDate", rec.get("create_date", ""))))
            if not href:
                article_id = rec.get("id", rec.get("articleId", ""))
                if article_id:
                    href = f"/fcsrmzf/gsgg/pc/content_{article_id}.html"
            if title and href:
                if not href.startswith('http'):
                    href = BASE_URL + href
                # Normalize date
                if date_str and len(date_str) >= 10:
                    date_str = date_str[:10]
                items.append((title.strip(), href, date_str))
    return items

def parse_list_from_html(html):
    """兜底: 从 HTML 页面提取文章"""
    items = []
    patterns = [
        r'<a[^>]*href="([^"]*gsgg[^"]*content[^"]*\.html)"[^>]*title="([^"]*)"',
        r'<a[^>]*href="([^"]*)"[^>]*target="_blank"[^>]*>(.*?)</a>\s*<span[^>]*>(\d{4}-\d{2}-\d{2})',
        r'<li[^>]*>.*?<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>.*?(?:<span[^>]*>|\[)?(\d{4}-\d{2}-\d{2})',
    ]
    for pat in patterns:
        for m in re.finditer(pat, html, re.DOTALL):
            href = m.group(1).strip()
            title = m.group(2).strip() if len(m.groups()) >= 2 else ""
            if len(m.groups()) >= 3:
                date_str = m.group(3).strip()
            else:
                date_str = ""
            if not title:
                title = re.sub(r'<[^>]+>', '', href).strip()
            if not title or len(title) < 4:
                continue
            if not href.startswith('http'):
                href = BASE_URL + href
            items.append((title, href, date_str))
        if items:
            break
    return items

def get_page_url(page):
    if page == 1:
        return BASE_URL + LIST_PATH
    return f"{BASE_URL}/fcsrmzf/gsgg/pc/list_{page}.html"

def get_api_params(page):
    params = dict(API_PARAMS)
    params["pageNum"] = page
    return params

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None
    result = {}
    m = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
    if m:
        result["title"] = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    if not result.get("title"):
        m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if m:
            result["title"] = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    m = re.search(r'发布日期[：:]\s*(\d{4}-\d{2}-\d{2})', html)
    if m:
        result["publish_date"] = m.group(1)
    if not result.get("publish_date"):
        m = re.search(r'发布时间[：:]\s*(\d{4}-\d{2}-\d{2})', html)
        if m:
            result["publish_date"] = m.group(1)
    if not result.get("publish_date"):
        m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        if m:
            result["publish_date"] = m.group(1)
    content = ""
    for cls in ['TRS_Editor', 'content', 'article-content', 'article_content', 'xxgk-content', 'Custom_UnionStyle', 'con_text', 'main-content', 'info-content', 'text']:
        m = re.search(r'<div[^>]*class="[^"]*' + cls + r'[^"]*"[^>]*>(.*?)</div>\s*(?:<div|</div>|<!--|$)', html, re.DOTALL)
        if m and len(m.group(1)) > 50:
            content = m.group(1).strip()
            break
    if not content or len(content) < 50:
        m = re.search(r'<div[^>]*id="zoom"[^>]*>(.*?)</div>', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        result["content"] = content
    return result.get("title"), result.get("content"), result.get("publish_date")

def main():
    import argparse
    parser = argparse.ArgumentParser(description=f"爬虫: {SITE_NAME}")
    parser.add_argument("--limit", type=int, help="测试：只处理N条详情")
    parser.add_argument("--incremental", action="store_true", help="增量模式：仅爬第1页")
    args = parser.parse_args()
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")
    all_list = []
    max_pages = 1 if args.incremental else MAX_PAGES
    # Try JSON API first
    api_worked = False
    for page in range(1, max_pages + 1):
        print(f"\n📄 第 {page} 页 (API)...", end=" ", flush=True)
        data = fetch_json(API_URL, get_api_params(page))
        if data and isinstance(data, dict):
            items = parse_list_from_json(data)
            if items:
                print(f"✅ {len(items)} 条 (JSON)")
                all_list.extend(items)
                api_worked = True
                if args.incremental:
                    break
                continue
        # Fallback to HTML pagination
        url = get_page_url(page)
        print(f"-> HTML...", end=" ", flush=True)
        html = fetch(url)
        if not html:
            print("❌")
            break
        items = parse_list_from_html(html)
        if not items:
            print("0 条")
            break
        print(f"✅ {len(items)} 条 (HTML)")
        all_list.extend(items)
        if args.incremental:
            break
    print(f"\n📊 列表总计: {len(all_list)} 条")
    all_items, seen = [], set()
    for i, (title, url, list_date) in enumerate(all_list):
        if url in seen:
            continue
        seen.add(url)
        if args.limit and len(all_items) >= args.limit:
            break
        print(f"  [{len(all_items)+1}/{len(all_list)}] {title[:50]}...", end=" ", flush=True)
        dt, content, date = fetch_detail(url)
        if dt:
            title = dt
        final_date = date or list_date
        summary = re.sub(r'<[^>]+>', ' ', content or '').strip()[:300]
        summary = re.sub(r'\s+', ' ', summary)
        all_items.append({
            "site_name": SITE_NAME, "title": title, "url": url,
            "content": content or "", "pub_date": final_date,
            "summary": summary, "tags": SITE_NAME,
        })
        print("✅")
        time.sleep(0.3)
    if all_items:
        push_to_searchdb(all_items, "jxfc_gsgg")
    print(f"\n✅ 完成! 共 {len(all_items)} 条")

if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")