#!/usr/bin/env python3
"""Run crawlers from index 59 onwards."""
import json, os, subprocess, sys, time, re

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
CONFIG_PATH = os.path.join(BASE_DIR, 'daily_crawl_config.json')
VENV_ACTIVATE = os.path.join(BASE_DIR, 'venv', 'bin', 'activate')

with open(CONFIG_PATH, encoding='utf-8') as f:
    crawlers = json.load(f)['crawlers']

total_ok = total_fail = total_new = 0
results = []

for i, c in enumerate(crawlers):
    if i < 59:  # Skip first 59 (already ran)
        continue
    if not c.get('enabled', True) or not c.get('incremental', False):
        continue
    
    script_path = os.path.join(BASE_DIR, c['script'])
    if not os.path.exists(script_path):
        print(f"  ❌ 脚本不存在: {script_path}")
        continue
    
    args = c.get('args', '')
    venv_python = f'source "{VENV_ACTIVATE}" && python3'
    cmd = f'cd "{BASE_DIR}" && {venv_python} "{script_path}" {args}'
    
    print(f"\n{'='*60}", flush=True)
    print(f"  [{i+1}/85] 📍 {c['name']}", flush=True)
    print(f"  {c['script']} | {c.get('sync_mode','?')}", flush=True)
    print(f"{'='*60}", flush=True)
    
    try:
        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=300)
        out = (r.stdout or '') + (r.stderr or '')
        code = r.returncode
    except subprocess.TimeoutExpired:
        out = "[TIMEOUT 300s]"
        code = -1
    except Exception as e:
        out = f"[ERROR] {e}"
        code = -1
    
    new_count = 0
    for line in out.split('\n'):
        stripped = line.strip()
        if any(k in stripped for k in ['新增', '完成', 'Error', 'ERROR', '✅', '❌', '⚠️', '🏁', '📊', '推送', '同步', 'OK:', '新增']):
            print(f"     {stripped[:150]}", flush=True)
        m = re.search(r'新增[：:]?\s*(\d+)', stripped)
        if m:
            new_count = max(new_count, int(m.group(1)))
    
    if code == 0:
        total_ok += 1
        total_new += new_count
        results.append({'name': c['name'], 'status': 'OK', 'new': new_count, 'sync': c.get('sync_mode','?')})
    else:
        total_fail += 1
        results.append({'name': c['name'], 'status': 'FAIL', 'new': 0, 'sync': c.get('sync_mode','?')})
        print(f"  ❌ 失败 (exit={code})", flush=True)
    
    time.sleep(0.3)

print(f"\n{'='*60}")
print(f"  📊 结果: 成功 {total_ok} / 失败 {total_fail}")
print(f"  📈 新增条目: {total_new}")
print(f"{'='*60}")

print("\n📋 各站点增量汇总:")
for r in results:
    icon = '✅' if r['status'] == 'OK' else '❌'
    print(f"  {icon} {r['name']}: {r['new']} 条新增 [{r['sync']}]")