#!/usr/bin/env python3
"""
spider.py — 统一爬虫（列表+正文）
==================================
一个脚本处理所有阶段：

  # 初始化（已跑完）
  python3 spider.py --init                  # 全量爬列表
  python3 spider.py --init --file=已采集.txt # 从指定文件初始化

  # 正文回填
  python3 spider.py --backfill              # 回填所有待处理的正文
  python3 spider.py --backfill --limit=100  # 只回填100条

  # 每日增量
  python3 spider.py --daily                 # 爬首页 → 找新条目 → 回填正文
  python3 spider.py --daily --file=重要站.txt # 只跑指定站点

  # 其他
  python3 spider.py --stats                 # 数据库统计
  python3 spider.py --url <URL>             # 单站测试
"""

import json, os, re, sys, hashlib, time, sqlite3, warnings, argparse
from datetime import datetime, timezone, timedelta
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed

import requests
from parsel import Selector

warnings.filterwarnings("ignore", category=requests.packages.urllib3.exceptions.InsecureRequestWarning)

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
DB_PATH = os.path.join(BASE_DIR, "list_crawl.db")
CONFIG_PATH = os.path.join(BASE_DIR, "sites_cache.json")
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36",
}

# ═══════════════════════════════════════════
#  列表页提取
# ═══════════════════════════════════════════
GENERIC_TITLE_SELECTORS = [
    "h1 a", "h2 a", "h3 a", "h4 a",
    "li a", "td a",
    ".title a", ".list a",
    ".news-title a", ".article-title a",
    "article a", ".item a",
]

SEMANTIC_DATE_SELECTORS = [
    "time::attr(datetime)", "time::text",
    "[class*=date]::text", "[class*=Date]::text",
    "[class*=time]::text", "[class*=Time]::text",
    "[class*=pub]::text", "[class*=riqi]::text", "[class*=sj]::text",
    "span.date::text", "em.date::text", "p.date::text",
]

WEAK_DATE_SELECTORS = ["span::text", "em::text", "i::text"]

DATE_RE = re.compile(
    r'(\d{4})[年\-/\.](\d{1,2})[月\-/\.](\d{1,2})'
    r'|(?<!\d)(\d{4})(\d{2})(\d{2})(?!\d)'
)

ONCLICK_RE = re.compile(r"""location\.href\s*=\s*['"]([^'"]+)['"]""")
DATA_TIME_RE = re.compile(r'data-time\s*=\s*["\']([^"\']+)["\']')

# 正文容器模式（来自 base_crawler.py）
CONTENT_PATTERNS = [
    r'<div[^>]*class="article-content\s+article-content-body"[^>]*id="zoomcon"[^>]*>(.*?)</div>\s*<div[^>]*class="article-(?:reldocuments|auxiliary|extended)"',
    r'id="zoomcon"[^>]*>(.*?)</div>\s*<div[^>]*class="article-(?:reldocuments|auxiliary|extended)"',
    r'<!--正文内容-->(.*?)<!--结束-->',
    r'<font[^>]*id="?Zoom"?[^>]*>(.*?)</font>',
    r'<div[^>]*class="article[^"]*"[^>]*>(.*?)</div>\s*</div>\s*</div>',
    r'<div class="text-center mb-3">(.*?)<div class="footer_cont',
    r'<div[^>]*class="wzcon"[^>]*>(.*?)</div>\s*<div',
    r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>\s*</div>',
    r'id="postmessage_\d+"[^>]*>(.*?)</div>\s*<div[^>]*id="postmessage_',
    r'<td[^>]*class="t_f"[^>]*>(.*?)</td>',
    r'class="t_fsz"[^>]*>(.*?)<div[^>]*class="(?:sign|pstatus|ptg)"',
    r'<div[^>]*class="v_news_content"[^>]*>(.*?)</div>\s*</div>\s*</div>',
]

# 详情页日期提取
PUB_DATE_META = re.compile(r'<meta[^>]*name=["\']?(?:publish_date|pubdate|date|ArticleTitleDate)["\']?[^>]*content=["\']([^"\']+)["\']', re.I)
PUB_DATE_META2 = re.compile(r'<meta[^>]*content=["\']([^"\']+)["\'][^>]*name=["\']?(?:publish_date|pubdate|date)["\']?', re.I)
TIME_TAG = re.compile(r'<time[^>]*datetime=["\']([^"\']+)["\']')
DATE_IN_HTML = re.compile(r'(\d{4})[年\-/\.](\d{1,2})[月\-/\.](\d{1,2})(?:日)?')


def parse_date_str(raw: str):
    if not raw: return None
    m = DATE_RE.search(raw.strip())
    if not m: return None
    if m.group(1):
        y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
    else:
        y, mo, d = int(m.group(4)), int(m.group(5)), int(m.group(6))
    if not (1990 <= y <= 2099 and 1 <= mo <= 12 and 1 <= d <= 31): return None
    return f"{y:04d}-{mo:02d}-{d:02d}"


def auto_selector(url: str) -> str:
    if re.search(r'list-(\d+)-\d+', url):
        m = re.search(r'list-(\d+)-\d+', url)
        return f'a[href*="content-{m.group(1)}"]'
    if '/art/' in url: return 'a[href*="/art/"]'
    if url.endswith('.shtml') or '/shtml/' in url: return 'a[href$=".shtml"]'
    if url.endswith('.jhtml') or 'jhtml' in url: return 'a[href$=".jhtml"]'
    if '/info/' in url: return 'a[href*="/info/"]'
    if 'content_' in url or re.search(r'/\d+/index\.htm', url): return 'a[href*="content_"]'
    m2 = re.search(r'/(xxgk|zwgk|news|gk)/[^/]*', url)
    if m2: return f'a[href*="/{m2.group(1)}/"]'
    m3 = re.search(r'/(\d+)/index\.s?html?$', url)
    if m3: return f'a[href*="/{m3.group(1)}/"]'
    m4 = re.search(r'/(\d+)/default\.s?html?$', url)
    if m4: return f'a[href*="/info/"], a[href*="content_"], a[href*="/{m4.group(1)}/"]'
    return ""


def _extract_link_date(sel) -> str:
    container = sel
    for xp in ('ancestor::li[1]', 'ancestor::tr[1]', 'ancestor::dd[1]', 'ancestor::dt[1]', 'ancestor::article[1]'):
        node = sel.xpath(xp)
        if node: container = node; break
    if container == sel:
        parent = sel.xpath('..')
        container = parent if parent else sel
    for s in SEMANTIC_DATE_SELECTORS:
        for txt in container.css(s).getall():
            d = parse_date_str(txt)
            if d: return d
    for s in WEAK_DATE_SELECTORS:
        for txt in container.css(s).getall():
            t = (txt or "").strip()
            if 0 < len(t) <= 40:
                d = parse_date_str(t)
                if d: return d
    for xp in ('following-sibling::*', 'preceding-sibling::*'):
        for sib in sel.xpath(xp):
            for s in SEMANTIC_DATE_SELECTORS:
                for txt in sib.css(s).getall():
                    d = parse_date_str(txt)
                    if d: return d
    for el in container.css("p, div, span, td, li, em"):
        txt = " ".join(el.css("::text").getall())
        if ("日期" in txt or "时间" in txt or "发布" in txt) and len(txt) <= 60:
            d = parse_date_str(txt)
            if d: return d
    outer = container.xpath('..')
    if outer:
        outer_txt = " ".join(outer.css("::text").getall())
        if len(outer_txt) <= 400:
            for s in SEMANTIC_DATE_SELECTORS:
                for txt in outer.css(s).getall():
                    d = parse_date_str(txt)
                    if d: return d
    return ""


def extract_data_attr_items(html: str, base_url: str):
    items = []
    for m in re.finditer(
        r'<(\w+)[^>]*data-url\s*=\s*["\']([^"\']+)["\'][^>]*data-title\s*=\s*["\']([^"\']+)["\'][^>]*data-time\s*=\s*["\']([^"\']+)["\'][^>]*>(.*?)</\1>',
        html, re.DOTALL
    ):
        href, title, date_str = m.group(2).strip(), m.group(3).strip(), m.group(4).strip()
        if href and title:
            if not href.startswith('http'): href = urljoin(base_url, href)
            items.append({"title": title, "url": href, "pub_date": parse_date_str(date_str) or ""})
    if len(items) >= 3: return items
    items2 = []
    for m in re.finditer(
        r'<(\w+)[^>]*data-url\s*=\s*["\']([^"\']+)["\'][^>]*data-title\s*=\s*["\']([^"\']+)["\'][^>]*>(.*?)</\1>',
        html, re.DOTALL
    ):
        href, title = m.group(2).strip(), m.group(3).strip()
        if href and title and 5 < len(title) < 150:
            block = m.group(0)
            date_str = ""
            dm = DATA_TIME_RE.search(block)
            if dm: date_str = parse_date_str(dm.group(1)) or ""
            if not href.startswith('http'): href = urljoin(base_url, href)
            items2.append({"title": title, "url": href, "pub_date": date_str})
    return items2 if len(items2) >= 3 else items


def extract_list_items(html: str, url: str, css_selector: str = ""):
    """从HTML提取列表条目"""
    sel = Selector(text=html)
    items = []

    # data-attr
    items = extract_data_attr_items(html, url)
    if items: return items, "data_attr"

    # auto_selector
    auto_sel = auto_selector(url)
    if auto_sel:
        links = sel.css(auto_sel)
        for a in links:
            text = " ".join(a.css("::text").getall()).strip()
            href = a.attrib.get("href", "")
            if 5 < len(text) < 150 and href:
                pub_date = _extract_link_date(a)
                items.append({"title": text, "url": urljoin(url, href), "pub_date": pub_date})
        if len(items) >= 3: return items, "auto"

    # custom selector
    if css_selector:
        links = sel.css(css_selector)
        for a in links:
            text = " ".join(a.css("::text").getall()).strip()
            href = a.attrib.get("href", "")
            if 5 < len(text) < 150 and href:
                pub_date = _extract_link_date(a)
                items.append({"title": text, "url": urljoin(url, href), "pub_date": pub_date})
        if len(items) >= 3: return items, "custom"

    # onclick
    for el in sel.css("[onclick]"):
        onclick = el.attrib.get("onclick", "")
        m = ONCLICK_RE.search(onclick)
        if not m: continue
        href = m.group(1).strip()
        if not href or href.startswith("javascript"): continue
        text = re.sub(r'\s+', ' ', " ".join(el.css("::text").getall()).strip()).strip()
        if 5 <= len(text) <= 300:
            pub_date = _extract_link_date(el)
            items.append({"title": text, "url": urljoin(url, href), "pub_date": pub_date})
    if len(items) >= 3: return items, "onclick"

    # generic
    for css_sel in GENERIC_TITLE_SELECTORS:
        links = sel.css(css_sel)
        if not links: continue
        candidates = []
        for a in links:
            text = " ".join(a.css("::text").getall()).strip()
            href = a.attrib.get("href", "")
            if 5 < len(text) < 150 and href:
                pub_date = _extract_link_date(a)
                candidates.append((text, urljoin(url, href), pub_date))
        if len(candidates) >= 2:
            for text, href, pub_date in candidates:
                items.append({"title": text, "url": href, "pub_date": pub_date})
            break
        if len(candidates) >= 1:
            for text, href, pub_date in candidates:
                items.append({"title": text, "url": href, "pub_date": pub_date})
            break
    return items, "generic"


def filter_list_items(items, page_date=""):
    """三规则过滤"""
    filtered = []
    for item in items:
        if "项目" not in item["title"]: continue
        date_str = item["pub_date"] or page_date
        if date_str and date_str < THREE_YEARS_AGO: continue
        filtered.append(item)
    return filtered


def fetch_page(url, timeout=20):
    try:
        r = requests.get(url, headers=HEADERS, timeout=timeout, verify=False)
        if r.status_code == 200:
            r.encoding = "utf-8"
            return r.text
        return None
    except:
        return None


def extract_detail(html, url):
    """从详情页提取正文和日期"""
    if not html:
        return {"content": "", "date": ""}

    # 正文
    content = ""
    for pat in CONTENT_PATTERNS:
        m = re.search(pat, html, re.DOTALL)
        if m:
            raw = m.group(1)
            raw = re.sub(r' style="[^"]*"', '', raw)
            raw = re.sub(r' class="[^"]*"', '', raw)
            raw = re.sub(r'<span[^>]*>|</span>', '', raw)
            raw = re.sub(r'&nbsp;', ' ', raw)
            raw = re.sub(r'\n{3,}', '\n\n', raw)
            content = raw.strip()
            if len(content) > 50: break

    # 兜底：p标签
    if not content or len(content) < 50:
        sel = Selector(text=html)
        ps = sel.css("p::text").getall()
        text = " ".join(p.strip() for p in ps if p.strip())
        if len(text) > 100: content = text[:50000]
        else:
            body_text = " ".join(sel.css("body ::text").getall()).strip()
            if body_text: content = body_text[:50000]

    # 日期
    date = ""
    dm = PUB_DATE_META.search(html)
    if not dm: dm = PUB_DATE_META2.search(html)
    if dm:
        d = dm.group(1).strip()
        m = DATE_IN_HTML.search(d)
        if m: date = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    if not date:
        dm = TIME_TAG.search(html)
        if dm:
            d = dm.group(1).strip()[:10]
            m = DATE_IN_HTML.search(d)
            if m: date = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    if not date:
        dm = DATE_IN_HTML.search(html)
        if dm: date = f"{dm.group(1)}-{int(dm.group(2)):02d}-{int(dm.group(3)):02d}"

    return {"content": content[:50000], "date": date}


def crawl_list_page(site_cfg, max_pages=5):
    """爬取单个站点的列表页"""
    url = site_cfg["url"]
    name = site_cfg.get("name", url)
    css_selector = site_cfg.get("selector", "").strip()
    site_key = hashlib.md5(url.encode()).hexdigest()[:12]

    page_url = url
    all_items = []
    seen_urls = set()
    first_page_date = ""
    method_used = "none"

    for page in range(1, max_pages + 1):
        if page > 1:
            page_url = _build_page_url(url, page)
            if not page_url or page_url in seen_urls: break
        seen_urls.add(page_url)

        html = fetch_page(page_url)
        if not html: break

        items, method = extract_list_items(html, page_url, css_selector)
        if page == 1:
            method_used = method
            first_page_date = extract_page_date(html)
        if not items: break

        new_items = [i for i in items if i["url"] not in seen_urls]
        for i in items: seen_urls.add(i["url"])
        if not new_items: break

        all_items.extend(new_items)
        if page == 1:
            dates = [i["pub_date"] for i in new_items if i["pub_date"]]
            if dates and min(dates) < THREE_YEARS_AGO: break

    filtered = filter_list_items(all_items, first_page_date)
    return {"key": site_key, "name": name, "url": url, "total": len(all_items),
            "filtered": len(filtered), "method": method_used, "items": filtered}


def _build_page_url(base_url, page):
    if page == 1: return base_url
    for pat in [
        (r'(.*?/)(index|default|list)\.s?html?$', lambda m, n: f"{m.group(1)}index_{n}.html"),
        (r'(.*?/)(index|default|list)\.s?html?$', lambda m, n: f"{m.group(1)}{m.group(2)}_{n}.shtml"),
    ]:
        m = re.match(pat[0], base_url)
        if m: return pat[1](m, page)
    return None


def extract_page_date(html):
    sel = Selector(text=html)
    for s in SEMANTIC_DATE_SELECTORS:
        val = sel.css(s).get("")
        d = parse_date_str(val.strip() if val else "")
        if d: return d
    return ""


# ═══════════════════════════════════════════
#  数据库
# ═══════════════════════════════════════════
def init_db():
    conn = sqlite3.connect(DB_PATH)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS sites (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_key TEXT UNIQUE,
            name TEXT,
            url TEXT,
            selector TEXT DEFAULT '',
            method TEXT DEFAULT '',
            status TEXT DEFAULT 'pending',
            last_crawled TEXT,
            total_items INTEGER DEFAULT 0,
            project_items INTEGER DEFAULT 0,
            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    """)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS items (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_key TEXT,
            site_name TEXT,
            title TEXT,
            url TEXT UNIQUE,
            pub_date TEXT,
            content TEXT DEFAULT '',
            status TEXT DEFAULT 'list_only',
            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    """)
    conn.execute("""
        CREATE TABLE IF NOT EXISTS crawl_log (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_key TEXT,
            site_name TEXT,
            method TEXT,
            total INTEGER,
            filtered INTEGER,
            duration_sec REAL,
            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_items_url ON items(url)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_items_status ON items(status)")
    conn.commit()
    return conn


def import_sites():
    conn = init_db()
    if conn.execute("SELECT COUNT(*) FROM sites").fetchone()[0] > 0:
        print("  sites 表已有数据，跳过")
        conn.close()
        return
    with open(CONFIG_PATH, encoding="utf-8") as f:
        data = json.load(f)
    imported = 0
    for s in data.get("sites", []):
        url = s.get("url", "", timeout=30).strip()
        if not url: continue
        site_key = hashlib.md5(url.encode()).hexdigest()[:12]
        try:
            conn.execute("INSERT OR IGNORE INTO sites (site_key, name, url, selector) VALUES (?, ?, ?, ?)",
                         (site_key, s.get("name", url, timeout=30)[:200], url, s.get("selector", "", timeout=30)))
            if conn.total_changes > 0: imported += 1
        except: pass
    conn.commit()
    conn.close()
    print(f"  ✅ 导入 {imported} 个站点")


def save_list_results(result, duration):
    conn = sqlite3.connect(DB_PATH)
    conn.execute("UPDATE sites SET method=?, status=?, last_crawled=CURRENT_TIMESTAMP, total_items=?, project_items=? WHERE site_key=?",
                 (result["method"], "done" if result["filtered"] > 0 else "empty",
                  result["total"], result["filtered"], result["key"]))
    new_count = 0
    for item in result["items"]:
        try:
            conn.execute("INSERT OR IGNORE INTO items (site_key, site_name, title, url, pub_date) VALUES (?, ?, ?, ?, ?)",
                         (result["key"], result["name"], item["title"], item["url"], item.get("pub_date", "")))
            if conn.total_changes > 0: new_count += 1
        except: pass
    conn.execute("INSERT INTO crawl_log (site_key, site_name, method, total, filtered, duration_sec) VALUES (?, ?, ?, ?, ?, ?)",
                 (result["key"], result["name"], result["method"], result["total"], result["filtered"], duration))
    conn.commit()
    conn.close()
    return new_count


def backfill_items(items, workers=5):
    """回填正文"""
    if not items: return [], 0, 0
    ok, date_ok, fail = 0, 0, 0
    results = []

    with ThreadPoolExecutor(max_workers=workers) as executor:
        futures = {}
        for item_id, url, existing_date in items:
            futures[executor.submit(_backfill_one, item_id, url, existing_date)] = item_id
        for i, future in enumerate(as_completed(futures), 1):
            item_id, data, status = future.result()
            results.append((item_id, data, status))
            if status == "completed": ok += 1
            elif status == "date_only": date_ok += 1
            else: fail += 1
            if i % 100 == 0:
                print(f"  [{i}/{len(items)}] ✅{ok} 🕐{date_ok} ❌{fail}")

    # 入库
    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()
    nc, nd = 0, 0
    for item_id, data, status in results:
        if data:
            if data.get("content"):
                cur.execute("UPDATE items SET content=?, pub_date=COALESCE(NULLIF(pub_date,''), ?), status='completed' WHERE id=?",
                           (data["content"], data["date"], item_id))
                nc += 1
                if data["date"]: nd += 1
            elif data.get("date"):
                cur.execute("UPDATE items SET pub_date=?, status='date_fixed' WHERE id=?", (data["date"], item_id))
                nd += 1
        elif status == "fetch_failed":
            cur.execute("UPDATE items SET status='fetch_failed' WHERE id=?", (item_id,))
    conn.commit()
    conn.close()
    return results, nc, nd


def _backfill_one(item_id, url, existing_date):
    html = fetch_page(url)
    if not html: return item_id, None, "fetch_failed"
    detail = extract_detail(html, url)
    content = detail["content"]
    date = detail["date"] or existing_date
    if content: return item_id, {"content": content, "date": date}, "completed"
    if date and not existing_date: return item_id, {"content": "", "date": date}, "date_only"
    return item_id, None, "no_content"


# ═══════════════════════════════════════════
#  Stats
# ═══════════════════════════════════════════
def print_stats():
    conn = init_db()
    total_sites = conn.execute("SELECT COUNT(*) FROM sites").fetchone()[0]
    done_sites = conn.execute("SELECT COUNT(*) FROM sites WHERE status='done'").fetchone()[0]
    empty_sites = conn.execute("SELECT COUNT(*) FROM sites WHERE status='empty'").fetchone()[0]
    total_items = conn.execute("SELECT COUNT(*) FROM items").fetchone()[0]
    list_only = conn.execute("SELECT COUNT(*) FROM items WHERE status='list_only'").fetchone()[0]
    completed = conn.execute("SELECT COUNT(*) FROM items WHERE status='completed'").fetchone()[0]
    fetch_failed = conn.execute("SELECT COUNT(*) FROM items WHERE status='fetch_failed'").fetchone()[0]
    with_date = conn.execute("SELECT COUNT(*) FROM items WHERE pub_date != ''").fetchone()[0]
    with_content = conn.execute("SELECT COUNT(*) FROM items WHERE content != ''").fetchone()[0]
    print(f"\n{'='*50}")
    print(f"  📊 list_crawl.db 统计")
    print(f"{'='*50}")
    print(f"  站点: {total_sites} (有数据{done_sites} / 空{empty_sites})")
    print(f"  条目: {total_items}")
    print(f"    ├─ 列表待处理: {list_only}")
    print(f"    ├─ 正文完成:   {completed}")
    print(f"    ├─ 抓取失败:   {fetch_failed}")
    print(f"    └─ 已有日期:   {with_date}")
    print(f"  已有正文: {with_content}")
    print(f"{'='*50}")
    if done_sites > 0:
        rows = conn.execute("SELECT name, project_items, method FROM sites WHERE status='done' ORDER BY project_items DESC LIMIT 10").fetchall()
        print(f"\n  项目数据Top10:")
        for r in rows:
            print(f"    {r[0][:50]:50s} {r[1]:3d}条 [{r[2]}]")
    conn.close()


# ═══════════════════════════════════════════
#  Main
# ═══════════════════════════════════════════
def main():
    parser = argparse.ArgumentParser(description="统一爬虫（列表+正文）")
    parser.add_argument("--init", action="store_true", help="初始化：全量爬列表")
    parser.add_argument("--backfill", action="store_true", help="正文回填")
    parser.add_argument("--daily", action="store_true", help="每日增量（爬首页+回填正文）")
    parser.add_argument("--stats", action="store_true", help="数据库统计")
    parser.add_argument("--url", help="单站测试")
    parser.add_argument("--name", help="站点名")
    parser.add_argument("--selector", help="CSS选择器")
    parser.add_argument("--file", help="URL列表文件")
    parser.add_argument("--limit", type=int, default=0, help="限制条数")
    parser.add_argument("--pages", type=int, default=1, help="翻页数")
    parser.add_argument("--workers", type=int, default=5, help="并发数")
    args = parser.parse_args()

    if args.stats:
        print_stats()
        return

    # ─── 单站测试 ───
    if args.url:
        site_cfg = {"url": args.url, "name": args.name or args.url, "selector": args.selector or ""}
        t0 = time.time()
        result = crawl_list_page(site_cfg, args.pages or 5)
        duration = time.time() - t0
        new_count = save_list_results(result, duration)
        print(f"\n{'='*50}")
        print(f"  📍 {result['name']}")
        print(f"  URL: {result['url']}  [{result['method']}]")
        print(f"  {result['total']}条 → {result['filtered']}条含项目 ({duration:.1f}s)")
        if result["items"]:
            print(f"  含项目条目:")
            for i, item in enumerate(result["items"][:15], 1):
                print(f"    {i:2d}. [{item['pub_date'] or '???'}] {item['title'][:65]}")

        # 回填这批条目的正文
        if result["items"]:
            print(f"\n  📋 回填正文...")
            items_for_backfill = []
            for item in result["items"]:
                row = conn.execute("SELECT id, url, pub_date FROM items WHERE url=?", (item["url"],)).fetchone()
                if row and row[2] == 'list_only':
                    items_for_backfill.append((row[0], row[1], row[2] or ""))
            if items_for_backfill:
                _, nc, nd = backfill_items(items_for_backfill, args.workers)
                print(f"  ✅ 回填: 正文{nc}条, 日期{nd}条")
        return

    # ─── 初始化 ───
    if args.init:
        import_sites()
        conn = sqlite3.connect(DB_PATH)
        rows = conn.execute("SELECT site_key, name, url, selector FROM sites ORDER BY id").fetchall()
        conn.close()

        url_filter = set()
        if args.file:
            with open(args.file) as f:
                for line in f:
                    line = line.strip()
                    if '\t' in line: url_filter.add(line.split('\t')[1].strip().rstrip('/'))
                    elif line and not line.startswith('#'): url_filter.add(line.rstrip('/'))
            rows = [r for r in rows if r[2].strip().rstrip('/') in url_filter]
            if args.limit: rows = rows[:args.limit]

        print(f"\n📋 初始化 {len(rows)} 个站点 (并发={args.workers})")
        t_start = time.time()
        ok, empty, fail = 0, 0, 0

        def crawl_one(row):
            site_cfg = {"url": row[2], "name": row[1], "selector": row[3] or ""}
            t0 = time.time()
            result = crawl_list_page(site_cfg, args.pages or 5)
            duration = time.time() - t0
            new = save_list_results(result, duration)
            return result, duration, new

        with ThreadPoolExecutor(max_workers=args.workers) as executor:
            futures = {executor.submit(crawl_one, row): row for row in rows}
            for i, future in enumerate(as_completed(futures), 1):
                try:
                    result, duration, _ = future.result()
                    if result["filtered"] > 0:
                        ok += 1; tag = f"✅ {result['filtered']}条/{result['total']}条 [{result['method']}]"
                    elif result["total"] > 0:
                        empty += 1; tag = f"📭 无项目/{result['total']}条 [{result['method']}]"
                    else:
                        empty += 1; tag = f"⬜ 0条 [{result['method']}]"
                    print(f"  [{i:4d}/{len(rows)}] {tag} {result['name'][:40]} {duration:.1f}s")
                except Exception as e:
                    fail += 1
                    print(f"  [{i:4d}/{len(rows)}] ❌ {str(e)[:50]}")

        print(f"\n  📊 完成: ✅ {ok} / 📭 {empty} / ❌ {fail}  耗时 {time.time()-t_start:.0f}s")
        print_stats()
        return

    # ─── 正文回填 ───
    if args.backfill:
        conn = sqlite3.connect(DB_PATH)
        items = conn.execute(
            "SELECT id, url, COALESCE(pub_date,'') FROM items WHERE status='list_only' ORDER BY id LIMIT ?",
            (args.limit or 999999,)
        ).fetchall()
        conn.close()
        if not items:
            print("  没有待回填的条目")
            print_stats()
            return
        print(f"\n📋 回填 {len(items)} 条 (并发={args.workers})")
        t0 = time.time()
        results, nc, nd = backfill_items(items, args.workers)
        print(f"\n  📊 完成: ✅正文{nc} / 🕐日期{nd} / ❌{len(items)-nc-nd}")
        print(f"  ⏱ {time.time()-t0:.0f}s (平均{(time.time()-t0)/len(items):.1f}s/条)")
        print_stats()
        return

    # ─── 每日增量 ───
    if args.daily:
        conn = init_db()
        if args.file:
            with open(args.file) as f:
                urls = set()
                for line in f:
                    line = line.strip()
                    if '\t' in line: urls.add(line.split('\t')[1].strip().rstrip('/'))
                    elif line and not line.startswith('#'): urls.add(line.rstrip('/'))
            rows = conn.execute("SELECT site_key, name, url, selector FROM sites").fetchall()
            rows = [r for r in rows if r[2].strip().rstrip('/') in urls]
        else:
            rows = conn.execute("SELECT site_key, name, url, selector FROM sites WHERE status='done' ORDER BY last_crawled").fetchall()
        conn.close()

        print(f"\n📋 每日增量: {len(rows)} 站 (并发={args.workers})")
        t_start = time.time()
        new_total = 0
        new_items_for_backfill = []

        def daily_check(row):
            site_cfg = {"url": row[2], "name": row[1], "selector": row[3] or ""}
            result = crawl_list_page(site_cfg, max_pages=1)
            return result

        with ThreadPoolExecutor(max_workers=args.workers) as executor:
            futures = {executor.submit(daily_check, row): row for row in rows}
            for future in as_completed(futures):
                try:
                    result = future.result()
                    new_count = save_list_results(result, 0)
                    if new_count > 0:
                        new_total += new_count
                        print(f"  ✅ {result['name'][:40]} +{new_count}新条目")
                        # 收集新条目用于回填
                        if result["items"]:
                            for item in result["items"]:
                                new_items_for_backfill.append(item)
                except: pass

        # 回填新条目的正文
        if new_items_for_backfill:
            print(f"\n  📋 回填 {len(new_items_for_backfill)} 条新条目正文...")
            conn = sqlite3.connect(DB_PATH)
            items = []
            for item in new_items_for_backfill[:args.limit or 999999]:
                row = conn.execute("SELECT id, url, COALESCE(pub_date,'') FROM items WHERE url=?", (item["url"],)).fetchone()
                if row: items.append((row[0], row[1], row[2]))
            conn.close()
            if items:
                _, nc, nd = backfill_items(items[:50], args.workers)  # 最多回填50篇新正文
                print(f"  ✅ 回填正文: {nc}条")

        print(f"\n  📊 增量完成: +{new_total} 新条目  耗时 {time.time()-t_start:.0f}s")
        return

    parser.print_help()


if __name__ == "__main__":
    main()