#!/usr/bin/env python3
"""
crawl_zhangshu_hjbh.py — 樟树市-环境保护 爬虫
============================================
列表: POST http://www.zhangshu.gov.cn/queryList (JSON API)
详情: <div class="zfxxgk-article">...</div>
分页: current 参数（共588条, 40页）
"""
import re, sys, os, time, json
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import (
    fetch_page, parse_date, clean_html,
    extract_date_from_html, extract_content_by_selector,
    save_jsonl, push_to_searchdb
)

# ═══════════════════════════════════════════
#  配置区
# ═══════════════════════════════════════════

SITE_NAME = "樟树市-环境保护"
BASE_URL  = "http://www.zhangshu.gov.cn"
API_URL   = "http://www.zhangshu.gov.cn/queryList"
CHANNEL_ID = "1996810615334215680"  # 环境保护频道

# 抓取参数
MAX_PAGES      = 5       # 最多5页
PAGE_SIZE      = 15
REQUEST_DELAY  = 0.5     # 请求间隔（秒）
BATCH_TAG      = "zhangshu_hjbh"

# ═══════════════════════════════════════════
#  列表解析
# ═══════════════════════════════════════════

def parse_list(page_num):
    """调用API，返回列表页中所有条目 [(title, url, date), ...]"""
    payload = {
        "current": page_num,
        "pageSize": PAGE_SIZE,
        "webSiteCode": ["zssrmzf"],
        "channelId": [CHANNEL_ID],
        "notReturnContent": True
    }

    try:
        import requests
        resp = requests.post(API_URL, json=payload, timeout=20,
                             headers={"User-Agent": "Mozilla/5.0"})
        if resp.status_code != 200:
            print(f"  ⚠ API返回状态码 {resp.status_code}")
            return [], 0

        data = resp.json()
        results = data.get("data", {}).get("results", [])
        total = data.get("data", {}).get("total", 0)
        items = []

        for item in results:
            source = item.get("source", {})
            title = source.get("title", "") or source.get("showTitle", "")
            # 清洗HTML标签
            title = re.sub(r'<[^>]*>', '', title).strip()
            if not title:
                continue

            # 详情页URL
            urls_raw = source.get("urls", "{}")
            try:
                urls = json.loads(urls_raw)
                detail_url = urls.get("pc", "")
            except:
                detail_url = ""
            if not detail_url:
                continue
            detail_url = urljoin(BASE_URL, detail_url)

            # 日期
            pub_date = source.get("pubDate", "")
            # 格式 "2026-06-01 08:58" → "2026-06-01"
            date_str = pub_date[:10] if pub_date else ""

            items.append((title, detail_url, date_str))

        return items, total

    except Exception as e:
        print(f"  ❌ API请求失败: {e}")
        return [], 0


# ═══════════════════════════════════════════
#  详情解析
# ═══════════════════════════════════════════

def parse_detail(url, default_title="", default_date=""):
    """抓取详情页，返回 (title, date, content)"""
    try:
        import requests
        resp = requests.get(url, timeout=20,
                            headers={"User-Agent": "Mozilla/5.0"})
        if resp.status_code != 200:
            return default_title, default_date, ""

        html = resp.text

        # 标题: 优先从API传入的
        title = default_title
        if not title:
            m = re.search(r'<title>(.*?)</title>', html)
            if m:
                title = m.group(1).split('-')[0].strip()

        # 日期: 优先从API传入的
        date_str = default_date
        if not date_str:
            # 从<meta name="PubDate" content="YYYY-MM-DD">提取
            m = re.search(r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\'](\d{4}-\d{2}-\d{2})', html, re.I)
            if not m:
                m = re.search(r'pubDate["\']?\s*[:=]\s*["\']?(\d{4}-\d{2}-\d{2})', html, re.I)
            if not m:
                m = re.search(r'发布时间[：:]\s*(\d{4}[-/]\d{2}[-/]\d{2})', html)
            if m:
                date_str = m.group(1).replace('/', '-')

        # 正文: zfxxgk-article 容器
        content = ""
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(html, 'lxml')
        article = soup.select_one('.zfxxgk-article')
        if article:
            # 移除无用元素
            for tag in article.select('script, style, .article-tool, .bshare-custom'):
                tag.decompose()
            content = article.decode_contents()
            # 保留结构化HTML但清理多余属性
            content = re.sub(r'\s+(class|style|id|lang|data-[^=])="[^"]*"', '', content)
            # 清理多余空行
            content = re.sub(r'\n\s*\n', '\n', content).strip()
        else:
            # 兜底: 取main_content
            main = soup.select_one('.main_content')
            if main:
                for tag in main.select('script, style'):
                    tag.decompose()
                content = main.decode_contents()
                content = re.sub(r'\s+(class|style|id|lang|data-[^=])="[^"]*"', '', content)
                content = re.sub(r'\n\s*\n', '\n', content).strip()

        # 如果content为空，尝试用纯文本
        if not content or len(content) < 50:
            text = soup.get_text(separator='\n')
            lines = [l.strip() for l in text.split('\n') if l.strip()]
            # 取正文段落（跳过导航和页脚）
            meaningful = [l for l in lines if len(l) > 20 and '政府信息公开' not in l]
            content = '\n'.join(meaningful) if meaningful else ''

        return title, date_str, content

    except Exception as e:
        print(f"    ❌ 详情页失败: {e}")
        return default_title, default_date, ""


# ═══════════════════════════════════════════
#  主流程
# ═══════════════════════════════════════════

def main():
    import requests
    all_items = []
    seen_urls = set()

    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"{'='*50}")

    for page in range(1, MAX_PAGES + 1):
        print(f"\n📄 第 {page} 页...", end=" ")
        items, total = parse_list(page)

        if not items:
            print("无数据，停止")
            break

        print(f"✅ {len(items)} 条")

        for title, url, date_str in items:
            if url in seen_urls:
                continue
            seen_urls.add(url)

            print(f"  [{len(all_items)+1}/{total}] {title[:40]}...", end=" ")
            t, d, content = parse_detail(url, title, date_str)

            record = {
                "title": t,
                "url": url,
                "pub_date": d,
                "content": content,
                "site_name": SITE_NAME,
                "source": BATCH_TAG,
            }
            all_items.append(record)

            if content and len(content) > 30:
                print(f"✅ [{d}]")
            else:
                print(f"⚠️ 正文短或空 [{d}]")

            time.sleep(REQUEST_DELAY)

        # 判断是否还有下一页
        if len(items) < PAGE_SIZE:
            break
        time.sleep(REQUEST_DELAY)

    # ─── 推送 ───
    if not all_items:
        print("\n⚠️ 没有数据")
        return

    print(f"\n{'─'*50}")
    print(f"📊 总计: {len(all_items)} 条待推送")
    print(f"\n📤 推送至服务器...")
    push_to_searchdb(all_items, BATCH_TAG)

    print(f"\n{'='*50}")
    print(f"✅ 完成! 共 {len(all_items)} 条已推送至服务器")
    print(f"{'='*50}\n")


if __name__ == "__main__":
    main()