#!/usr/bin/env python3
"""Batch run crawlers efficiently - use venv python directly"""
import json
import subprocess
import os
import sys
import time

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
CONFIG_PATH = os.path.join(BASE_DIR, 'daily_crawl_config.json')
VENV_PYTHON = os.path.join(BASE_DIR, 'venv', 'bin', 'python3')

def load_config():
    with open(CONFIG_PATH, encoding='utf-8') as f:
        return json.load(f)['crawlers']

def run_one(script, args=''):
    script_path = os.path.join(BASE_DIR, script)
    cmd = f'cd "{BASE_DIR}" && "{VENV_PYTHON}" "{script_path}" {args}'
    try:
        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=600)
        out = r.stdout + r.stderr
        return out, r.returncode
    except subprocess.TimeoutExpired:
        return "[TIMEOUT 600s]", -1
    except Exception as e:
        return f"[ERROR] {e}", -1

def extract_summary(out):
    lines = []
    for line in out.split('\n'):
        if any(k in line for k in ['新增','完成','Error','ERROR','入库','上传','推送','爬取','列表','同步','💾','📤','✅','❌','🔍','found','skip','跳过','条→','条 ']):
            lines.append(line.strip())
    return lines[-8:] if lines else []

# ====== MAIN ======
crawlers = load_config()
direct = [c for c in crawlers if c.get('enabled') and c.get('incremental') and c.get('sync_mode') == 'direct_server']
scrapy = [c for c in crawlers if c.get('enabled') and c.get('incremental') and c.get('sync_mode') == 'scrapy_to_searchdb']

results = {'direct': [], 'scrapy': []}

print(f"{'='*60}")
print(f"PHASE 1: {len(direct)} direct_server crawlers")
print(f"{'='*60}")
for i, c in enumerate(direct):
    name = c['name']
    script = c['script']
    args = c.get('args', '')
    print(f"[{i+1}/{len(direct)}] {name}...", end=' ', flush=True)
    
    try:
        out, code = run_one(script, args)
    except Exception as e:
        print(f'❌ Exception: {e}')
        results['direct'].append({'name': name, 'status': '❌', 'code': -1, 'summary': [str(e)]})
        continue
    
    summary = extract_summary(out)
    status = '✅' if code == 0 else '❌'
    print(f'{status}', flush=True)
    
    for s in summary:
        print(f"    {s}")
    
    results['direct'].append({
        'name': name, 'status': status, 'code': code, 'summary': summary
    })

# Report direct
direct_ok = sum(1 for r in results['direct'] if r['code'] == 0)
print(f"\n{'='*60}")
print(f"DIRECT SERVER: {direct_ok}/{len(direct)} succeeded")
print(f"{'='*60}")
for r in results['direct']:
    s = ' | '.join(r['summary'][:3]) if r['summary'] else ''
    print(f"  {r['status']} {r['name']}: {s[:120]}")

print(f"\n{'='*60}")
print(f"PHASE 2: {len(scrapy)} scrapy_to_searchdb crawlers")
print(f"{'='*60}")

for i, c in enumerate(scrapy):
    name = c['name']
    script = c['script']
    args = c.get('args', '')
    print(f"[{i+1}/{len(scrapy)}] {name}...", end=' ', flush=True)
    
    try:
        out, code = run_one(script, args)
    except Exception as e:
        print(f'❌ Exception: {e}')
        results['scrapy'].append({'name': name, 'status': '❌', 'code': -1, 'summary': [str(e)]})
        continue
    
    summary = extract_summary(out)
    status = '✅' if code == 0 else '❌'
    print(f'{status}', flush=True)
    
    for s in summary:
        print(f"    {s}")
    
    results['scrapy'].append({
        'name': name, 'status': status, 'code': code, 'summary': summary
    })

# Final summary
scrapy_ok = sum(1 for r in results['scrapy'] if r['code'] == 0)
total_ok = direct_ok + scrapy_ok
total = len(direct) + len(scrapy)
print(f"\n{'='*60}")
print(f"TOTAL: {total_ok}/{total} succeeded")
print(f"{'='*60}")

# Print scrapy details with sync info
for r in results['scrapy']:
    s = ' | '.join(r['summary'][:4]) if r['summary'] else ''
    print(f"  {r['status']} {r['name']}: {s[:150]}")

print(f"\n{'='*60}")
print("DONE")