#!/usr/bin/env python3
"""
crawl_fengxin_qyfw.py — 奉新县-企业服务 爬虫
========================================
列表: POST http://www.fengxin.gov.cn/queryList (JSON API)
详情: <div class="zfxxgk-article">...</div>
分页: current 参数（共125条）
"""
import re, sys, os, time, json
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "奉新县-企业服务"
BASE_URL  = "http://www.fengxin.gov.cn"
API_URL   = "http://www.fengxin.gov.cn/queryList"
CHANNEL_ID = "1996830091433844736"

MAX_PAGES      = 5
PAGE_SIZE      = 15
REQUEST_DELAY  = 0.5
BATCH_TAG      = "fengxin_qyfw"

def parse_list(page_num):
    import requests
    payload = {
        "current": page_num, "pageSize": PAGE_SIZE,
        "webSiteCode": ["fxxrmzf"], "channelId": [CHANNEL_ID],
        "notReturnContent": True
    }
    try:
        resp = requests.post(API_URL, json=payload, timeout=20,
                             headers={"User-Agent": "Mozilla/5.0"})
        data = resp.json()
        results = data.get("data", {}).get("results", [])
        total = data.get("data", {}).get("total", 0)
        items = []
        for item in results:
            source = item.get("source", {})
            title = re.sub(r'<[^>]*>', '', source.get("title", "") or source.get("showTitle", "")).strip()
            if not title: continue
            urls_raw = source.get("urls", "{}")
            try: detail_url = json.loads(urls_raw).get("pc", "")
            except: detail_url = ""
            if not detail_url: continue
            detail_url = urljoin(BASE_URL, detail_url)
            pub_date = source.get("pubDate", "")[:10]
            items.append((title, detail_url, pub_date))
        return items, total
    except Exception as e:
        print(f"  ❌ API请求失败: {e}")
        return [], 0

def parse_detail(url, default_title="", default_date=""):
    import requests
    from bs4 import BeautifulSoup
    try:
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            "Referer": BASE_URL + "/fxxrmzf/qyfw/pc/list.html",
        }
        resp = requests.get(url, headers=headers, timeout=20)
        html = resp.text
        title = default_title
        date_str = default_date
        if not date_str:
            m = re.search(r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\'](\d{4}-\d{2}-\d{2})', html, re.I)
            if m: date_str = m.group(1)
        content = ""
        soup = BeautifulSoup(html, 'lxml')
        article = soup.select_one('.zfxxgk-article')
        if article:
            for tag in article.select('script, style'): tag.decompose()
            content = article.decode_contents()
            content = re.sub(r'\s+(class|style|id)="[^"]*"', '', content)
            content = re.sub(r'\n\s*\n', '\n', content).strip()
        return title or default_title, date_str or default_date, content
    except Exception as e:
        print(f"    ❌ 详情页失败: {e}")
        return default_title, default_date, ""

def main():
    all_items = []
    seen_urls = set()
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")
    for page in range(1, MAX_PAGES + 1):
        print(f"\n📄 第 {page} 页...", end=" ")
        items, total = parse_list(page)
        if not items: break
        print(f"✅ {len(items)} 条")
        for title, url, date_str in items:
            if url in seen_urls: continue
            seen_urls.add(url)
            print(f"  [{len(all_items)+1}/{total}] {title[:40]}...", end=" ")
            t, d, content = parse_detail(url, title, date_str)
            record = {"title": t, "url": url, "pub_date": d, "content": content, "site_name": SITE_NAME, "source": BATCH_TAG}
            all_items.append(record)
            print(f"✅ [{d}]" if (d and content and len(content) > 30) else f"⚠️ [{d}]")
            time.sleep(REQUEST_DELAY)
        if len(items) < PAGE_SIZE: break
        time.sleep(REQUEST_DELAY)
    if all_items:
        print(f"\n{'─'*50}\n📊 总计: {len(all_items)} 条\n📤 推送至服务器...")
        push_to_searchdb(all_items, BATCH_TAG)
        print(f"\n{'='*50}\n✅ 完成! 共 {len(all_items)} 条\n{'='*50}")

if __name__ == "__main__":
    main()