#!/usr/bin/env python3
"""
EIAFans (环评爱好者) 爬虫 — Discuz! X3.4 论坛
====================
全量/增量两种模式：
  python3 crawl_eiafans.py              # 增量（默认：只爬最新10页）
  python3 crawl_eiafans.py --full       # 全量：2078页
  python3 crawl_eiafans.py --pages=50   # 指定页数
  python3 crawl_eiafans.py --sync       # 同步已有数据到服务器

⚠️ 注意：详情页有JS反爬封锁，只能获取列表页数据（标题+URL+日期）
   搜索结果显示"详情请跳转原站查看"

数据流：本地 quality_results.db → SSH → 服务器 search.db
"""

import sys, os, re, json, sqlite3, hashlib, time, ssl, urllib.request, base64, subprocess
from datetime import datetime
from urllib.parse import urljoin
from html import unescape

# ─── 配置 ───
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
QUALITY_DB = os.path.join(BASE_DIR, "quality_results.db")
SERVER_SSH = "root@1.94.217.116"
SERVER_DB = "/root/quality_results.db"
SERVER_SEARCH_DB = "/root/search.db"

BASE_URL = "http://www.eiafans.com"
FORUM_URL = "http://www.eiafans.com/forum-64-{page}.html"

TOTAL_PAGES = 2078
PLACEHOLDER_CONTENT = "⚠️ 该数据来自环评爱好者论坛，详情页有反爬保护，请点击链接跳转原站查看正文内容。"
PLACEHOLDER_SUMMARY = "环评爱好者论坛公示信息 — 详情请跳转原站查看"

# ─── HTTP 工具 ───
CTX = ssl._create_unverified_context()
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "Chrome/125.0.0.0 Safari/537.36",
}

def http_get(url, timeout=20):
    """返回 (text, final_url) — GBK 自动转 UTF-8"""
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=timeout, context=CTX)
        raw = resp.read()
        text = raw.decode("gbk", errors="replace")
        return text, resp.geturl()
    except Exception as e:
        return None, str(e)

# ─── 列表页解析 ───
def parse_list_page(html):
    """从 Discuz! 列表页解析所有帖子，提取标题、URL、日期"""
    threads = []
    seen_tids = set()
    
    # 找到 normalthread 的区域（排除 sticky/pinned 公告）
    # 匹配 tbody id="normalthread_{tid}"
    for m in re.finditer(
        r'<tbody[^>]*id="normalthread_(\d+)"[^>]*>(.*?)</tbody>',
        html, re.DOTALL
    ):
        tid = m.group(1)
        block = m.group(2)
        
        if tid in seen_tids:
            continue
        seen_tids.add(tid)
        
        # 标题：class="s xst"
        title_m = re.search(r'class="s xst"[^>]*>(.*?)</a>', block, re.DOTALL)
        if not title_m:
            continue
        title = re.sub(r'<[^>]+>', '', title_m.group(1)).strip()
        if not title or len(title) < 5:
            continue
        
        # URL：thread-{tid}-1-{page}.html
        url_m = re.search(r'href="(thread-\d+-1-\d+\.html)"', block)
        thread_url = urljoin(BASE_URL, url_m.group(1)) if url_m else f"{BASE_URL}/thread-{tid}-1-1.html"
        
        # 日期：从 <td class="by"> 中的 <em> 提取
        date = ""
        date_m = re.search(r'<em[^>]*>\s*<a[^>]*>\s*(\d{4}-\d{1,2}-\d{1,2})', block)
        if date_m:
            date = date_m.group(1)
        
        # 备选日期：无 <a> 包裹的日期格式
        if not date:
            date_m = re.search(r'(\d{4}-\d{1,2}-\d{1,2})\s+\d{2}:\d{2}', block)
            if date_m:
                date = date_m.group(1)
        
        threads.append({
            "tid": tid,
            "url": f"{BASE_URL}/thread-{tid}-1-1.html",
            "title": title,
            "date": date,
        })
    
    return threads

# ─── 数据库操作 ───
def init_db():
    conn = sqlite3.connect(QUALITY_DB)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA synchronous=NORMAL")
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS quality_results (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            site_name TEXT NOT NULL,
            domain TEXT,
            title TEXT,
            url TEXT UNIQUE,
            content TEXT,
            publish_date TEXT,
            summary TEXT,
            crawled_at TEXT DEFAULT (datetime('now','localtime'))
        );
        CREATE TABLE IF NOT EXISTS eiafans_threads (
            tid TEXT PRIMARY KEY,
            title TEXT,
            url TEXT,
            publish_date TEXT,
            crawled_at TEXT DEFAULT (datetime('now','localtime'))
        );
    """)
    conn.commit()
    conn.close()

import queue, threading

_write_queue = queue.Queue()
_write_thread_started = False

def _write_worker():
    while True:
        task = _write_queue.get()
        if task is None:
            break
        func, args, kwargs = task
        try:
            func(*args, **kwargs)
        except Exception as e:
            print(f"    ❌ DB写入错误: {e}")
        _write_queue.task_done()

def _ensure_write_thread():
    global _write_thread_started
    if not _write_thread_started:
        t = threading.Thread(target=_write_worker, daemon=True)
        t.start()
        _write_thread_started = True

def _db_save_quality(name, domain, title, url, content, date, summary):
    conn = sqlite3.connect(QUALITY_DB)
    conn.execute("PRAGMA journal_mode=WAL")
    try:
        conn.execute("DELETE FROM quality_results WHERE url=?", (url,))
        conn.execute(
            "INSERT INTO quality_results (site_name, domain, title, url, content, publish_date, summary) VALUES (?,?,?,?,?,?,?)",
            (name, domain, title, url, content or "", date or "", summary or ""),
        )
        conn.commit()
    except Exception as e:
        print(f"    ❌ DB: {e}")
    finally:
        conn.close()

def _db_mark_thread(tid, title, url, date):
    conn = sqlite3.connect(QUALITY_DB)
    conn.execute("PRAGMA journal_mode=WAL")
    try:
        conn.execute(
            "INSERT OR IGNORE INTO eiafans_threads (tid, title, url, publish_date) VALUES (?,?,?,?)",
            (tid, title, url, date),
        )
        conn.commit()
    except Exception as e:
        print(f"    ❌ DB: {e}")
    finally:
        conn.close()

def is_thread_crawled(tid):
    conn = sqlite3.connect(QUALITY_DB)
    try:
        conn.execute("PRAGMA journal_mode=WAL")
        r = conn.execute("SELECT 1 FROM eiafans_threads WHERE tid=?", (tid,)).fetchone()
        return r is not None
    finally:
        conn.close()

def mark_thread_crawled(tid, title, url, date):
    _ensure_write_thread()
    _write_queue.put((_db_mark_thread, (tid, title, url, date), {}))

def save_to_quality(name, domain, title, url, content, date, summary):
    _ensure_write_thread()
    _write_queue.put((_db_save_quality, (name, domain, title, url, content, date, summary), {}))

# ─── 同步到服务器 ───
def sync_to_server():
    """将本地数据直接写入 search.db（服务器本地模式）"""
    print("\n📤 同步到 search.db...")

    conn = sqlite3.connect(QUALITY_DB)
    rows = conn.execute(
        "SELECT title, url, content, publish_date, summary FROM crawl_results ORDER BY id"
    ).fetchall()
    conn.close()

    if not rows:
        print("  本地没有数据")
        return

    dst = sqlite3.connect("/root/search.db")
    dst.execute("PRAGMA journal_mode=WAL")

    site_name = "eiafans"
    new_count = 0
    for r in rows:
        title, url, content, pub_date, summary = r
        try:
            dst.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(title, page_url, content, publish_date, summary, site_name, tags) "
                "VALUES (?,?,?,?,?,?,?)",
                (title, url, (content or "")[:8000], pub_date or "",
                 (summary or "")[:300], site_name, "")
            )
            if dst.total_changes > 0:
                new_count += 1
        except Exception as e:
            print(f"  Error: {e}")

    if new_count > 0:
        dst.commit()
        # Update FTS
        dst.execute(
            "INSERT INTO gov_search(rowid,title,site_name,summary) "
            "SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r "
            "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (site_name,))
        dst.commit()

    total = dst.execute(
        "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (site_name,)).fetchone()[0]
    dst.close()

    print(f"  OK {new_count}/{len(rows)} 条同步到 search.db (DB共{total}条)")


# ─── 站点爬取 ───
def crawl_one_page(page_num, out_file=None):
    """爬取单页列表 — 只提取标题、URL、日期（详情页有JS反爬）"""
    list_url = FORUM_URL.format(page=page_num)
    
    html, _ = http_get(list_url)
    if not html:
        print(f"  ❌ 列表页 {page_num} 失败")
        return 0
    
    threads = parse_list_page(html)
    if not threads:
        print(f"  ⚠️ 列表页 {page_num} 无帖子")
        return 0
    
    # 修正URL：始终指向 thread-{tid}-1-1.html
    for t in threads:
        t["url"] = f"{BASE_URL}/thread-{t['tid']}-1-1.html"
    
    count = 0
    for t in threads:
        if is_thread_crawled(t["tid"]):
            continue
        
        # 存储（内容占位，详情JS反爬无法获取）
        save_to_quality(
            "环评爱好者", "www.eiafans.com",
            t["title"], t["url"],
            PLACEHOLDER_CONTENT,
            t["date"],
            f"{PLACEHOLDER_SUMMARY}\n📎 原帖链接：{t['url']}",
        )
        mark_thread_crawled(t["tid"], t["title"], t["url"], t["date"])
        count += 1
        
        meta = f" | {t['date']}" if t["date"] else ""
        print(f"    ✅ {t['title'][:35]}...{meta}")
        
        if out_file:
            out_file.write(json.dumps({
                "site_name": "环评爱好者",
                "title": t["title"],
                "publish_date": t["date"],
                "page_url": t["url"],
                "content": t["title"],
                "content_text": f"{PLACEHOLDER_SUMMARY} — {t['url']}",
            }, ensure_ascii=False) + "\n")
            out_file.flush()
    
    return count

def crawl(pages_to_crawl, page_start=1):
    """全量/增量爬取所有页"""
    conn = sqlite3.connect(QUALITY_DB)
    existing = conn.execute("SELECT COUNT(*) FROM eiafans_threads").fetchone()[0]
    conn.close()
    print(f"📊 已有记录: {existing} 个帖子")
    
    start_time = time.time()
    total_new = 0
    
    for page in range(page_start, page_start + pages_to_crawl):
        if page > TOTAL_PAGES:
            print(f"\n⏹ 已达最大页数 {TOTAL_PAGES}")
            break
        
        print(f"\n📃 第 {page}/{TOTAL_PAGES} 页...", end=" ")
        sys.stdout.flush()
        
        n = crawl_one_page(page)
        total_new += n
        
        if n > 0:
            print(f"  ➕+{n} 条")
        else:
            print("  ➖ 无新增")
        
        # 进度报告
        if page % 100 == 0 or page == page_start + pages_to_crawl - 1:
            elapsed = time.time() - start_time
            rate = total_new / max(elapsed, 1)
            print(f"\n📊 【进度】第{page}页 | 累计{total_new}条 | {elapsed:.0f}s | {rate:.0f}条/秒")
    
    elapsed = time.time() - start_time
    print(f"\n{'='*50}")
    print(f"✅ 完成！共处理 {min(pages_to_crawl, TOTAL_PAGES - page_start + 1)} 页，新增 {total_new} 条")
    print(f"⏱ 耗时: {elapsed:.0f}秒 ({elapsed/60:.1f}分钟)")
    print(f"⚡ 速率: {total_new/max(elapsed,1):.1f} 条/秒")
    
    return total_new

# ─── 主入口 ───
if __name__ == "__main__":
    import argparse
    
    parser = argparse.ArgumentParser(description="环评爱好者爬虫")
    parser.add_argument("--full", action="store_true", help="全量爬取所有2078页")
    parser.add_argument("--pages", type=int, default=10, help="爬取页数 (默认10)")
    parser.add_argument("--start", type=int, default=1, help="起始页数 (默认1)")
    parser.add_argument("--sync", action="store_true", help="仅同步已有数据到服务器")
    
    args = parser.parse_args()
    
    if args.sync:
        sync_to_server()
        sys.exit(0)
    
    pages = TOTAL_PAGES if args.full else args.pages
    
    init_db()
    
    print(f"\n{'='*50}")
    print(f"🚀 环评爱好者爬虫 — {'全量' if args.full else f'增量({pages}页)'}")
    print(f"{'='*50}")
    print(f"  模式: {'全量' if args.full else '增量'}")
    print(f"  页数: {pages} (第{args.start}页起)")
    print(f"  数据: 列表页标题+URL+日期 (详情页有JS反爬)")
    print(f"{'='*50}")
    
    total_new = crawl(pages, page_start=args.start)
    
    if total_new > 0:
        sync_to_server()
    
    print(f"\n{'='*50}")
    print(f"🏁 完成")