#!/usr/bin/env python3
"""
spider_phase2.py — 正文回填
===========================
读取 list_crawl.db 中的列表数据，爬取详情页回填正文和日期。

用法:
  python3 spider_phase2.py --limit=50          # 试跑50条
  python3 spider_phase2.py --batch             # 跑所有
  python3 spider_phase2.py --missing-date      # 只跑缺日期的
  python3 spider_phase2.py --stats             # 只看统计
"""

import sqlite3, os, re, time, warnings
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime

import requests
from parsel import Selector

warnings.filterwarnings("ignore", category=requests.packages.urllib3.exceptions.InsecureRequestWarning)

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
DB_PATH = os.path.join(BASE_DIR, "list_crawl.db")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36",
}

# ── 正文容器模式（来自 base_crawler.py） ──
CONTENT_PATTERNS = [
    r'<div[^>]*class="article-content\s+article-content-body"[^>]*id="zoomcon"[^>]*>(.*?)</div>\s*<div[^>]*class="article-(?:reldocuments|auxiliary|extended)"',
    r'id="zoomcon"[^>]*>(.*?)</div>\s*<div[^>]*class="article-(?:reldocuments|auxiliary|extended)"',
    r'<!--正文内容-->(.*?)<!--结束-->',
    r'<font[^>]*id="?Zoom"?[^>]*>(.*?)</font>',
    r'<div[^>]*class="article[^"]*"[^>]*>(.*?)</div>\s*</div>\s*</div>',
    r'<div class="text-center mb-3">(.*?)<div class="footer_cont',
    r'<div[^>]*class="wzcon"[^>]*>(.*?)</div>\s*<div',
    r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>\s*</div>',
    r'id="postmessage_\d+"[^>]*>(.*?)</div>\s*<div[^>]*id="postmessage_',
    r'<td[^>]*class="t_f"[^>]*>(.*?)</td>',
    r'class="t_fsz"[^>]*>(.*?)<div[^>]*class="(?:sign|pstatus|ptg)"',
    r'<div[^>]*class="v_news_content"[^>]*>(.*?)</div>\s*</div>\s*</div>',
]

# 日期正则（从详情页提取）
DATE_IN_HTML = re.compile(r'(\d{4})[年\-/\.](\d{1,2})[月\-/\.](\d{1,2})(?:日)?')
PUB_DATE_META = re.compile(r'<meta[^>]*name=["\']?(?:publish_date|pubdate|date|ArticleTitleDate)["\']?[^>]*content=["\']([^"\']+)["\']', re.I)
PUB_DATE_META2 = re.compile(r'<meta[^>]*content=["\']([^"\']+)["\'][^>]*name=["\']?(?:publish_date|pubdate|date)["\']?', re.I)
TIME_TAG = re.compile(r'<time[^>]*datetime=["\']([^"\']+)["\']')


def fetch_page(url, timeout=20):
    try:
        r = requests.get(url, headers=HEADERS, timeout=timeout, verify=False)
        if r.status_code == 200:
            r.encoding = "utf-8"
            return r.text
        return None
    except:
        return None


def clean_html(html):
    """清理HTML，保留结构化标签"""
    if not html: return ""
    html = re.sub(r' style="[^"]*"', '', html)
    html = re.sub(r' class="[^"]*"', '', html)
    html = re.sub(r'<span[^>]*>|</span>', '', html)
    html = re.sub(r'&nbsp;', ' ', html)
    html = re.sub(r'<br\s*/?>', '\n', html)
    html = re.sub(r'\n{3,}', '\n\n', html)
    return html.strip()


def extract_detail(html, url):
    """从详情页提取正文和日期"""
    if not html:
        return {"content": "", "date": ""}

    # ── 正文 ──
    content = ""
    for pat in CONTENT_PATTERNS:
        m = re.search(pat, html, re.DOTALL)
        if m:
            content = clean_html(m.group(1))
            if len(content) > 50:
                break

    # 兜底：找页面里最大的文本块（<p>标签）
    if not content or len(content) < 50:
        sel = Selector(text=html)
        ps = sel.css("p::text").getall()
        text = " ".join(p.strip() for p in ps if p.strip())
        if len(text) > 100:
            content = text[:50000]
        else:
            # 最后兜底：body 里的所有文本
            body_text = " ".join(sel.css("body ::text").getall()).strip()
            if body_text:
                content = body_text[:50000]

    # ── 日期（优先级：meta标签 > <time> > 正则） ──
    date = ""
    dm = PUB_DATE_META.search(html)
    if not dm:
        dm = PUB_DATE_META2.search(html)
    if dm:
        d = dm.group(1).strip()
        m = DATE_IN_HTML.search(d)
        if m:
            date = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    if not date:
        dm = TIME_TAG.search(html)
        if dm:
            d = dm.group(1).strip()[:10]
            m = DATE_IN_HTML.search(d)
            if m:
                date = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    if not date:
        dm = DATE_IN_HTML.search(html)
        if dm:
            date = f"{dm.group(1)}-{int(dm.group(2)):02d}-{int(dm.group(3)):02d}"

    return {"content": content[:50000], "date": date}


def process_item(item_id, url, existing_date):
    """处理单个详情页"""
    html = fetch_page(url)
    if not html:
        return item_id, None, "fetch_failed"

    detail = extract_detail(html, url)
    content = detail["content"]
    date = detail["date"] or existing_date

    if content:
        return item_id, {"content": content, "date": date}, "completed"
    elif date and not existing_date:
        return item_id, {"content": "", "date": date}, "date_only"
    else:
        return item_id, None, "no_content"


def update_db(items_updates):
    """批量更新数据库"""
    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()
    new_dates = 0
    new_contents = 0
    for item_id, data, status in items_updates:
        if data:
            if data.get("content"):
                cur.execute(
                    "UPDATE items SET content=?, pub_date=COALESCE(NULLIF(pub_date,''), ?), status='completed' WHERE id=?",
                    (data["content"], data["date"], item_id)
                )
                new_contents += 1
                if data["date"]:
                    new_dates += 1
            elif data.get("date"):
                cur.execute(
                    "UPDATE items SET pub_date=?, status='date_fixed' WHERE id=?",
                    (data["date"], item_id)
                )
                new_dates += 1
        else:
            if status == "fetch_failed":
                cur.execute("UPDATE items SET status='fetch_failed' WHERE id=?", (item_id,))
    conn.commit()
    conn.close()
    return new_contents, new_dates


def get_items_to_process(limit=0, missing_date_only=False):
    """获取待处理的条目"""
    conn = sqlite3.connect(DB_PATH)
    if missing_date_only:
        rows = conn.execute(
            "SELECT id, url, pub_date FROM items WHERE status='list_only' AND pub_date='' ORDER BY id LIMIT ?",
            (limit or 999999,)
        ).fetchall()
    else:
        rows = conn.execute(
            "SELECT id, url, pub_date FROM items WHERE status='list_only' ORDER BY id LIMIT ?",
            (limit or 999999,)
        ).fetchall()
    conn.close()
    return rows


def print_stats():
    conn = sqlite3.connect(DB_PATH)
    total = conn.execute("SELECT COUNT(*) FROM items").fetchone()[0]
    list_only = conn.execute("SELECT COUNT(*) FROM items WHERE status='list_only'").fetchone()[0]
    completed = conn.execute("SELECT COUNT(*) FROM items WHERE status='completed'").fetchone()[0]
    fetch_failed = conn.execute("SELECT COUNT(*) FROM items WHERE status='fetch_failed'").fetchone()[0]
    date_fixed = conn.execute("SELECT COUNT(*) FROM items WHERE status='date_fixed'").fetchone()[0]
    with_date = conn.execute("SELECT COUNT(*) FROM items WHERE pub_date != ''").fetchone()[0]
    with_content = conn.execute("SELECT COUNT(*) FROM items WHERE content != ''").fetchone()[0]
    print(f"\n{'='*50}")
    print(f"  📊 Phase 2 进度")
    print(f"{'='*50}")
    print(f"  总条目:       {total}")
    print(f"  待处理:       {list_only}")
    print(f"  正文完成:     {completed}")
    print(f"  仅补充日期:   {date_fixed}")
    print(f"  抓取失败:     {fetch_failed}")
    print(f"  ──────────────")
    print(f"  已有日期:     {with_date}")
    print(f"  已有正文:     {with_content}")
    print(f"{'='*50}")
    conn.close()


def main():
    import argparse
    parser = argparse.ArgumentParser(description="正文回填 Phase 2")
    parser.add_argument("--limit", type=int, default=0, help="处理条数")
    parser.add_argument("--batch", action="store_true", help="全量处理")
    parser.add_argument("--missing-date", action="store_true", help="只处理缺日期的")
    parser.add_argument("--stats", action="store_true", help="只看统计")
    parser.add_argument("--workers", type=int, default=5, help="并发数")
    args = parser.parse_args()

    if args.stats:
        print_stats()
        return

    items = get_items_to_process(args.limit, args.missing_date)
    if not items:
        print("  没有待处理的条目")
        print_stats()
        return

    print(f"\n📋 待处理: {len(items)} 条 (并发={args.workers})")
    t_start = time.time()
    ok, date_ok, fail = 0, 0, 0

    with ThreadPoolExecutor(max_workers=args.workers) as executor:
        futures = {executor.submit(process_item, i[0], i[1], i[2]): i for i in items}
        results = []
        for i, future in enumerate(as_completed(futures), 1):
            item_id, data, status = future.result()
            results.append((item_id, data, status))
            if status == "completed":
                ok += 1
                tag = f"✅ 正文+日期"
            elif status == "date_only":
                date_ok += 1
                tag = "🕐 仅日期"
            elif status == "no_content":
                fail += 1
                tag = "⬜ 无正文"
            else:
                fail += 1
                tag = "❌ 请求失败"
            if i % 50 == 0 or i <= 10:
                print(f"  [{i:4d}/{len(items)}] {tag}")

    # 批量入库
    new_contents, new_dates = update_db(results)
    total_time = time.time() - t_start

    print(f"\n  📊 完成: ✅正文{ok} / 🕐仅日期{date_ok} / ❌失败{fail}")
    print(f"  💾 入库: 新增正文{new_contents}条, 补充日期{new_dates}条")
    print(f"  ⏱ 耗时: {total_time:.0f}s (平均{total_time/len(items):.1f}s/条)")
    print_stats()


if __name__ == "__main__":
    main()