#!/usr/bin/env python3
"""
crawl_xjcbcr_yjzj.py — 察布查尔县-意见征集 爬虫
=========================================
列表: https://www.xjcbcr.gov.cn/xjcbcr/c114141/list.shtml
结构: <table> → <tr id="line1457_0"> → <a class="c1457">标题</a> + <span class="timestyle1457">日期</span>
分页: list.shtml → list_2.shtml → list_3.shtml ... (共26页)
详情: <div id="vsb_content"><div class="v_news_content"> + <meta name="ArticleTitle"> + <meta name="PubDate">

用法:
    python3 crawl_xjcbcr_yjzj.py             # 全量爬（最多5页）
    python3 crawl_xjcbcr_yjzj.py --limit=5   # 测试只跑5条
"""

import re, sys, os, time
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import (
    fetch_page, parse_date, clean_html,
    extract_date_from_html, extract_content_by_selector,
    save_jsonl, push_to_searchdb
)

# ═══════════════════════════════════════════
#  配置区
# ═══════════════════════════════════════════

SITE_NAME = "察布查尔县-意见征集"
BASE_URL  = "https://www.xjcbcr.gov.cn"
LIST_PATH = "/xjcbcr/c114141/list.shtml"

MAX_PAGES = 5  # 最近5页

# 详情页正文 CSS 选择器
DETAIL_SELECTOR = "#vsb_content, .v_news_content"

# 列表项正则（提取 <tr> 块）
# 结构: <tr id="line1457_0">
#         <td><img src="dot.gif"></td>
#         <td><a class="c1457" href="...">标题</a></td>
#         <td><span class="timestyle1457">2026-05-19</span></td>
#       </tr>
LI_PATTERN = re.compile(
    r'<a class="c1457"\s+href="([^"]+)"[^>]*title=\'([^\']+)\'>\s*(.*?)</a>'
    r'.*?<span class="timestyle1457">\s*(\d{4}-\d{2}-\d{2})',
    re.DOTALL
)


# ═══════════════════════════════════════════
#  工具函数
# ═══════════════════════════════════════════

def _abs_urls(html, base_url):
    """将 HTML 中的相对路径（src/href）转为绝对 URL"""
    def _abs(m):
        attr = m.group(1)
        val = m.group(2)
        if val.startswith(('http://', 'https://', '//', 'data:', 'javascript:', '#', 'tel:', 'mailto:')):
            return m.group(0)
        abs_val = urljoin(base_url, val)
        return f'{attr}="{abs_val}"'
    html = re.sub(r'(src|href)="([^"]+)"', _abs, html, flags=re.I)
    return html


def norm_date(d):
    """标准化日期为 YYYY-MM-DD"""
    if not d:
        return ""
    m = re.search(r"(\d{4})-(\d{1,2})-(\d{1,2})", str(d))
    if m:
        y, mo, day = int(m.group(1)), int(m.group(2)), int(m.group(3))
        return f"{y:04d}-{mo:02d}-{day:02d}" if 1990 <= y <= 2099 else ""
    return ""


# ═══════════════════════════════════════════
#  列表页提取
# ═══════════════════════════════════════════

def get_list_items(html):
    """从列表页 HTML 提取条目（标题、URL、日期）"""
    items = []
    for m in LI_PATTERN.finditer(html):
        href = m.group(1).strip()
        title = m.group(2).strip()  # 从 title 属性获取完整标题
        pub_date = m.group(4).strip()

        if not title or not href:
            continue

        # 如果没有 title 属性中的标题，用标签内容
        if not title:
            title = m.group(3).strip()
            title = re.sub(r'<[^>]+>', '', title).strip()

        full_url = urljoin(BASE_URL, href)
        items.append({
            "title": title,
            "url": full_url,
            "pub_date": norm_date(pub_date),
        })
    return items


def get_page_items(page=1):
    """获取指定页码的列表条目"""
    if page == 1:
        url = urljoin(BASE_URL, LIST_PATH)
    else:
        # list.shtml → list_2.shtml, list_3.shtml ...
        list_name = LIST_PATH.replace(".shtml", f"_{page}.shtml")
        url = urljoin(BASE_URL, list_name)

    html = fetch_page(url)
    if not html:
        print(f"  ❌ 获取列表页失败: {url}")
        return []
    return get_list_items(html)


# ═══════════════════════════════════════════
#  详情页提取
# ═══════════════════════════════════════════

def get_detail(url):
    """访问详情页，返回正文 HTML 和发布时间"""
    html = fetch_page(url)
    if not html:
        return {"content": "", "pub_date": "", "title": ""}

    # 从 meta 标签提取标题
    title = ""
    m_title = re.search(r'<meta name="ArticleTitle" content="([^"]+)"', html)
    if m_title:
        title = m_title.group(1).strip()

    # 从 <title> 标签提取（兜底）
    if not title:
        m_title = re.search(r'<title>([^<]+)</title>', html)
        if m_title:
            title = m_title.group(1).strip()

    # 提取正文 — 使用 CSS 选择器
    content = extract_content_by_selector(html, DETAIL_SELECTOR)
    if not content or len(content) < 20:
        # 备选正则
        m = re.search(
            r'<div id="vsb_content">\s*<div class="v_news_content">(.*?)</div>\s*</div>',
            html, re.DOTALL
        )
        if m:
            content = clean_html(m.group(1))

    if content:
        content = _abs_urls(content, url)

    # 提取日期（meta 优先）
    pub_date = ""
    m_date = re.search(r'<meta name="PubDate" content="([^"]+)"', html)
    if m_date:
        pub_date = norm_date(m_date.group(1))
    if not pub_date:
        pub_date = extract_date_from_html(html)

    return {"content": content, "pub_date": pub_date, "title": title}


# ═══════════════════════════════════════════
#  主流程
# ═══════════════════════════════════════════

def crawl(max_items=None):
    """全量爬取（最多5页）"""
    print(f"  📡 抓取列表页...")

    all_items = []
    for page in range(1, MAX_PAGES + 1):
        items = get_page_items(page)
        if page == 1:
            label = LIST_PATH.split("/")[-1]
        else:
            label = LIST_PATH.replace(".shtml", f"_{page}.shtml").split("/")[-1]
        print(f"    {label}: {len(items)} 条")
        all_items.extend(items)
        if len(items) == 0:
            break

    # 去重
    seen = set()
    unique = []
    for it in all_items:
        if it["url"] not in seen:
            seen.add(it["url"])
            unique.append(it)

    print(f"  去重后: {len(unique)} 条")

    if max_items:
        unique = unique[:max_items]

    # 爬正文
    collected = []
    ok, fail = 0, 0
    for i, item in enumerate(unique, 1):
        detail = get_detail(item["url"])
        title = detail.get("title") or item["title"]
        pub_date = detail.get("pub_date") or item.get("pub_date", "")
        content = detail["content"]

        # 清理正文 HTML → 纯文本摘要
        content_text = content or ""
        summary = re.sub(r"<[^>]+>", " ", content_text).strip()
        summary = re.sub(r"\s+", " ", summary)[:300]

        entry = {
            "site_name": SITE_NAME,
            "title": title,
            "url": item["url"],
            "content": content_text,
            "pub_date": pub_date,
            "summary": summary,
            "tags": SITE_NAME,
        }
        collected.append(entry)
        save_jsonl(entry)
        ok += 1
        if i % 5 == 0 or i == len(unique):
            print(f"  [{i}/{len(unique)}] ✅{ok} ❌{fail}")

    # 推送到服务器 search.db
    if collected:
        push_to_searchdb(collected, batch_label="xjcbcr_yjzj")
    print(f"  ✅ 完成: 共{ok}条")
    return ok, fail


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description=f"爬虫: {SITE_NAME}")
    parser.add_argument("--limit", type=int, help="测试：只跑 N 条")
    args = parser.parse_args()

    print(f"\n📡 [{SITE_NAME}] 开始爬取 (最多{MAX_PAGES}页)")
    t0 = time.time()
    crawl(max_items=args.limit)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s\n")