#!/usr/bin/env python3
"""
息烽县-园区信息 (xifeng.gov.cn)
列表: /tzxf/xfgyy/yqxx/ (第1页), /tzxf/xfgyy/yqxx/index_{n}.html
详情: /tzxf/xfgyy/yqxx/YYYYMM/tYYYYMMDD_NNNNNNNN.html
正文: div.content_1 > font#Zoom
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
from urllib.parse import urljoin
import requests, urllib3

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "息烽县-园区信息"
BASE_URL = "https://www.xifeng.gov.cn"
LIST_PATH = "/tzxf/xfgyy/yqxx/"
MAX_PAGES = 5
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}


def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  ⚠ {e}")
        return None


def parse_list(html):
    """解析列表页，返回 [(title, url, date_str), ...]"""
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")
    items = []

    # 列表在 div.NewsList > ul > li
    news_list = soup.select_one("div.NewsList")
    if not news_list:
        return items

    for li in news_list.select("ul > li"):
        a_tag = li.select_one("a[href]")
        if not a_tag:
            continue

        href = a_tag.get("href", "").strip()
        if not href:
            continue

        # 完整URL
        url = href if href.startswith("http") else urljoin(BASE_URL, href)

        # 标题：优先取 title 属性，其次取文本
        title = a_tag.get("title", "").strip()
        if not title or len(title) < 4:
            title = a_tag.get_text(strip=True)
        if not title or len(title) < 4:
            continue
        # 清理多余空白
        title = re.sub(r"\s+", " ", title).strip()

        # 日期：<span>YYYY-MM-DD</span>
        date_span = li.select_one("span")
        pub_date = ""
        if date_span:
            raw = date_span.get_text(strip=True)
            m = re.match(r"(\d{4})-(\d{1,2})-(\d{1,2})", raw)
            if m:
                pub_date = f"{int(m.group(1)):04d}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"

        items.append((title, url, pub_date))

    return items


def fetch_detail(url):
    """获取详情页标题、正文、发布日期"""
    html = fetch(url)
    if not html:
        return None, None, None

    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")

    # 标题
    title = ""
    h1 = soup.select_one("div.body_contenter div.title h1")
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        h1 = soup.select_one("h1")
        if h1:
            title = h1.get_text(strip=True)
    if not title:
        m = re.search(r"<title>(.*?)</title>", html, re.DOTALL)
        if m:
            title = m.group(1).strip()
    title = re.sub(r"\s+", " ", title).strip()

    # 正文: div.content_1 > font#Zoom
    content = ""
    zoom = soup.select_one("div.content_1 font#Zoom")
    if not zoom:
        zoom = soup.select_one("div.content_1")
    if not zoom:
        zoom = soup.select_one("font#Zoom")
    if zoom:
        for tag in zoom.find_all(["script", "style"]):
            tag.decompose()
        content = str(zoom)
    else:
        # 兜底
        body = soup.select_one("body")
        if body:
            for tag in body.find_all(["script", "style"]):
                tag.decompose()
            content = str(body)

    # 发布日期: var pubdata='YYYY-MM-DD' 或 发布时间：YYYY-MM-DD
    pub_date = ""
    m = re.search(r"pubdata\s*=\s*'(\d{4}-\d{1,2}-\d{1,2})'", html)
    if m:
        yr, mo, dy = m.group(1).split("-")
        pub_date = f"{int(yr):04d}-{int(mo):02d}-{int(dy):02d}"
    else:
        m = re.search(r"发布时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})", html)
        if m:
            yr, mo, dy = m.group(1).split("-")
            pub_date = f"{int(yr):04d}-{int(mo):02d}-{int(dy):02d}"

    return title, content, pub_date


def main():
    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"{'='*50}")

    # ── 爬取列表页 ──
    all_entries = []  # [(title, url, date), ...]
    seen_urls = set()
    for page in range(1, MAX_PAGES + 1):
        if page == 1:
            url = BASE_URL + LIST_PATH
        else:
            url = BASE_URL + f"/tzxf/xfgyy/yqxx/index_{page}.html"

        print(f"\n📄 第 {page} 页 ({url})...", end=" ", flush=True)
        html = fetch(url)
        if not html:
            print("❌ 请求失败")
            break

        entries = parse_list(html)
        # 去重
        new_entries = []
        for title, entry_url, date_str in entries:
            if entry_url not in seen_urls:
                seen_urls.add(entry_url)
                new_entries.append((title, entry_url, date_str))

        if not new_entries:
            print(f"0 条新条目（共 {len(entries)} 条，全部重复）")
            break

        print(f"✅ {len(new_entries)} 条")
        all_entries.extend(new_entries)

    print(f"\n📊 列表总计: {len(all_entries)} 条")

    if not all_entries:
        print("  ⏭ 无条目，退出")
        return

    # ── 爬取详情页 ──
    all_items = []
    for i, (list_title, detail_url, list_date) in enumerate(all_entries):
        print(f"  [{i+1}/{len(all_entries)}] {list_title[:50]}...", end=" ", flush=True)

        dt, content, detail_date = fetch_detail(detail_url)
        final_title = dt or list_title
        final_date = detail_date or list_date

        # 清理标题中的HTML标签
        final_title = re.sub(r"<[^>]+>", "", final_title).strip()
        final_title = re.sub(r"\s+", " ", final_title)

        summary = re.sub(r"<[^>]+>", " ", content or "").strip()
        summary = re.sub(r"\s+", " ", summary)[:300]

        all_items.append({
            "site_name": SITE_NAME,
            "title": final_title,
            "url": detail_url,
            "content": content or "",
            "pub_date": final_date,
            "summary": summary,
            "tags": SITE_NAME,
        })
        print("✅")
        time.sleep(0.3)

    # ── 推送 ──
    if all_items:
        push_to_searchdb(all_items, "xifeng_yqxx")
    else:
        print("  ⏭ 无数据，跳过推送")

    print(f"\n✅ 完成! 共 {len(all_items)} 条")


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")