#!/usr/bin/env python3
"""Batch run crawlers efficiently - split into groups"""
import json
import subprocess
import os
import sys
import time

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
CONFIG_PATH = os.path.join(BASE_DIR, 'daily_crawl_config.json')
VENV = os.path.join(BASE_DIR, 'venv', 'bin', 'activate')

def load_config():
    with open(CONFIG_PATH, encoding='utf-8') as f:
        return json.load(f)['crawlers']

def run_one(script, args=''):
    cmd = f'cd "{BASE_DIR}" && source "{VENV}" && python3 "{script}" {args}'
    try:
        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=600)
        out = r.stdout + r.stderr
        return out, r.returncode
    except subprocess.TimeoutExpired:
        return "[TIMEOUT 600s]", -1
    except Exception as e:
        return f"[ERROR] {e}", -1

def extract_summary(out):
    lines = []
    for line in out.split('\n'):
        if any(k in line for k in ['新增','完成','Error','ERROR','入库','上传','推送','爬取','列表','同步','💾','📤','✅','❌','🔍','found','skip','跳过','条→','条 ']):
            lines.append(line.strip())
    return lines[-8:] if lines else [out.strip()[-200:]]

# ====== MAIN ======
crawlers = load_config()
direct = [c for c in crawlers if c.get('enabled') and c.get('incremental') and c.get('sync_mode') == 'direct_server']
scrapy = [c for c in crawlers if c.get('enabled') and c.get('incremental') and c.get('sync_mode') == 'scrapy_to_searchdb']

results = {'direct': [], 'scrapy': []}

# Run direct_server first (they are independent and auto-sync)
print(f"{'='*60}")
print(f"PHASE 1: {len(direct)} direct_server crawlers")
print(f"{'='*60}")
for i, c in enumerate(direct):
    name = c['name']
    script = c['script']
    args = c.get('args', '')
    print(f"\n[{i+1}/{len(direct)}] {name} ({script})")
    sys.stdout.flush()
    
    out, code = run_one(script, args)
    summary = extract_summary(out)
    status = '✅' if code == 0 else '❌'
    
    for s in summary:
        print(f"  {s}")
    
    results['direct'].append({
        'name': name, 'status': status, 'code': code, 'summary': summary
    })
    
    if code != 0:
        # Print full error output for failures
        err_lines = out.strip().split('\n')
        for l in err_lines[-10:]:
            if 'Traceback' in l or 'Error' in l or 'error' in l or 'Exception' in l:
                print(f"  ⚠️  {l}")
    
    time.sleep(0.5)

# Report
direct_ok = sum(1 for r in results['direct'] if r['code'] == 0)
print(f"\n{'='*60}")
print(f"DIRECT SERVER DONE: {direct_ok}/{len(direct)} succeeded")
print(f"{'='*60}")
for r in results['direct']:
    s = ' | '.join(r['summary'][:3]) if r['summary'] else '(no output)'
    print(f"  {r['status']} {r['name']}: {s[:150]}")

print(f"\n{'='*60}")
print(f"PHASE 2: {len(scrapy)} scrapy_to_searchdb crawlers")
print(f"{'='*60}")

# Now run scrapy_to_searchdb crawlers  
for i, c in enumerate(scrapy):
    name = c['name']
    script = c['script']
    args = c.get('args', '')
    print(f"\n[{i+1}/{len(scrapy)}] {name} ({script} {args})")
    sys.stdout.flush()
    
    out, code = run_one(script, args)
    summary = extract_summary(out)
    status = '✅' if code == 0 else '❌'
    
    for s in summary:
        print(f"  {s}")
    
    results['scrapy'].append({
        'name': name, 'status': status, 'code': code, 'summary': summary
    })
    
    if code != 0:
        err_lines = out.strip().split('\n')
        for l in err_lines[-10:]:
            if 'Traceback' in l or 'Error' in l or 'error' in l or 'Exception' in l:
                print(f"  ⚠️  {l}")
    
    time.sleep(0.5)

# Final summary
scrapy_ok = sum(1 for r in results['scrapy'] if r['code'] == 0)
print(f"\n{'='*60}")
print(f"SCRAPY DONE: {scrapy_ok}/{len(scrapy)} succeeded")
print(f"{'='*60}")
for r in results['scrapy']:
    s = ' | '.join(r['summary'][:3]) if r['summary'] else '(no output)'
    print(f"  {r['status']} {r['name']}: {s[:150]}")

print(f"\n{'='*60}")
print(f"FINAL: Direct={direct_ok}/{len(direct)} | Scrapy={scrapy_ok}/{len(scrapy)} | Total OK={direct_ok+scrapy_ok}/{len(direct)+len(scrapy)}")
print(f"{'='*60}")