#!/bin/bash
cd ~/gov_crawler
source venv/bin/activate

echo "=== Running direct_server crawlers ==="
python3 -c "
import json, subprocess, os, sys

cfg = json.load(open('daily_crawl_config.json', encoding='utf-8'))
crawlers = [c for c in cfg['crawlers'] if c.get('enabled') and c.get('incremental') and c.get('sync_mode') == 'direct_server']

total = 0
failed = 0
results = []

for c in crawlers:
    script = c['script']
    args = c.get('args', '')
    name = c['name']
    print(f'=== {name} ===')
    cmd = f'cd {os.getcwd()} && source venv/bin/activate && python3 \"{script}\" {args}'
    try:
        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=600)
        out = r.stdout + r.stderr
        code = r.returncode
    except subprocess.TimeoutExpired:
        out = '[TIMEOUT 600s]'
        code = -1
    except Exception as e:
        out = f'[ERROR] {e}'
        code = -1
    
    # Extract key info
    summary_lines = []
    for line in out.split('\n'):
        if any(k in line for k in ['新增','完成','Error','ERROR','error','✅','❌','成功','失败','found','新增记录','已存在']):
            summary_lines.append(line.strip())
    
    status = '✅' if code == 0 else '❌'
    print(f'  {status} exit={code}')
    for sl in summary_lines[-5:]:
        print(f'    {sl}')
    results.append({'name': name, 'status': status, 'code': code, 'summary': summary_lines})
    total += 1
    if code != 0:
        failed += 1

print()
print('='*60)
print(f'DIRECT SERVER SUMMARY: Success {total-failed}/{total}')
for r in results:
    s = ' | '.join(r['summary'][-3:]) if r['summary'] else ''
    print(f'  {r[\"status\"]} {r[\"name\"]}: {s}')
" 2>&1
echo "EXIT_CODE=$?"