#!/usr/bin/env python3
"""
crawl_TEMPLATE.py — 爬虫模板
==========================
按站写爬虫的统一模式。

规则：
    - 只爬列表前 5 页
    - 只取标题含"项目" + 近 3 年的数据
    - 正文保留 HTML 格式
    - 结果写 eia.db（search_app 自动可见）

用法:
    python3 crawl_TEMPLATE.py             # 全量爬（最多5页）
    python3 crawl_TEMPLATE.py --limit=5   # 测试只跑5条
    python3 crawl_TEMPLATE.py --stats     # 看统计

修改指南：
    改下面「配置区」的 3-5 项即可开工。
"""

import re, sys, os, time

# ── 导入共享库 ──
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import (
    fetch_page, parse_date, clean_html, THREE_YEARS_AGO,
    extract_date_from_html, extract_content_by_selector, extract_content_by_regex,
    import_to_eia, save_jsonl, get_eia_conn
)

# ═══════════════════════════════════════════
#  配置区（每站必改）
# ═══════════════════════════════════════════

SITE_NAME = "站点名称"
BASE_URL  = "http://example.com/"
LIST_URL  = "http://example.com/list/"

# 列表页：从 HTML 中提取 {url, title} 的正则
# 分组1=url  分组2=title
LIST_PATTERN = r'<a[^>]*href=["\'](/detail/\d+\.html)["\'][^>]*>([^<]+)</a>'

# 【可选】详情页正文容器 CSS 选择器（优先用这个）
DETAIL_SELECTOR = "div.article-content, div.content, #content, .TRS_Editor"

# 【可选】详情页正文容器正则（当 CSS 选择器不work时用）
DETAIL_PATTERN = r'<div[^>]*class="article-content"[^>]*>(.*?)</div>\s*<div'

# 【可选】分页URL模式
# 如果设置了这个，爬虫会自动爬第2页、第3页...
# 变量 {n} 会被替换为页码
PAGE_URL_PATTERN = None  # 如 "http://example.com/list_{n}.html"

# ═══════════════════════════════════════════
#  列表页提取
# ═══════════════════════════════════════════

def get_list_items(html):
    """从列表页 HTML 提取所有条目
    
    返回: [{"title": "...", "url": "..."}, ...]
    """
    items = []
    for m in re.finditer(LIST_PATTERN, html):
        href = m.group(1).strip()
        title = m.group(2).strip()
        if not href.startswith('http'):
            from urllib.parse import urljoin
            href = urljoin(LIST_URL, href)
        if title and href:
            items.append({"title": title, "url": href})
    return items


# ═══════════════════════════════════════════
#  详情页提取
# ═══════════════════════════════════════════

def get_detail(url):
    """访问详情页，返回正文 HTML 和发布时间
    
    返回: {"content": "...", "pub_date": "YYYY-MM-DD"}
    """
    html = fetch_page(url)
    if not html:
        return {"content": "", "pub_date": ""}

    # 1. 尝试 CSS 选择器（保留 HTML 结构）
    content = extract_content_by_selector(html, DETAIL_SELECTOR)

    # 2. 尝试正则容器
    if not content or len(content) < 50:
        content = extract_content_by_regex(html, DETAIL_PATTERN)

    # 3. 兜底：手动指定正则区域（按站定制）
    #    eg: m = re.search(r'<!--正文-->(.*?)<!--结束-->', html, re.DOTALL)
    #    if m: content = clean_html(m.group(1))

    # 日期
    pub_date = extract_date_from_html(html)

    return {"content": content, "pub_date": pub_date}


# ═══════════════════════════════════════════
#  主流程（一般不改）
# ═══════════════════════════════════════════

def crawl(max_items=None):
    """全量爬取（最多5页，近3年，含项目）"""
    urls = [LIST_URL]
    if PAGE_URL_PATTERN:
        for n in range(2, 6):
            urls.append(PAGE_URL_PATTERN.replace("{n}", str(n)))

    all_items = []
    seen_urls = set()
    for page_url in urls:
        html = fetch_page(page_url)
        if not html:
            continue
        items = get_list_items(html)
        # 过滤：标题含"项目" + 近3年
        filtered = []
        for it in items:
            if "项目" not in it["title"]:
                continue
            if it.get("pub_date") and it["pub_date"] < THREE_YEARS_AGO:
                continue
            filtered.append(it)
        new_items = [i for i in filtered if i["url"] not in seen_urls]
        for i in filtered:
            seen_urls.add(i["url"])
        all_items.extend(new_items)

    print(f"  列表页: 提取 {len(all_items)} 条")

    if max_items:
        all_items = all_items[:max_items]

    # 爬正文
    conn = get_eia_conn()
    ok, fail = 0, 0
    for i, item in enumerate(all_items, 1):
        detail = get_detail(item["url"])
        entry = {
            "site_name": SITE_NAME,
            "title": item["title"],
            "url": item["url"],
            "content": detail["content"],
            "pub_date": detail["pub_date"],
            "tags": SITE_NAME,
        }
        save_jsonl(entry)
        if import_to_eia(entry, conn):
            ok += 1
        else:
            fail += 1
        if i % 5 == 0 or i == len(all_items):
            print(f"  [{i}/{len(all_items)}] ✅{ok} ❌{fail}")

    conn.close()
    print(f"  ✅ 完成: 新增{ok}, 跳过{fail}")
    return ok, fail


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description=f"爬虫: {SITE_NAME}")
    parser.add_argument("--limit", type=int, help="测试：只跑 N 条")
    parser.add_argument("--stats", action="store_true", help="查看 eia.db 统计")
    args = parser.parse_args()

    if args.stats:
        conn = get_eia_conn()
        total = conn.execute("SELECT COUNT(*) FROM projects").fetchone()[0]
        print(f"📊 eia.db: {total} 条项目")
        conn.close()
        sys.exit(0)

    print(f"\n📡 [{SITE_NAME}] 开始爬取 (最多5页, 近3年, 含\"项目\")")
    t0 = time.time()
    crawl(max_items=args.limit)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s\n")