#!/usr/bin/env python3
"""Batch crawler executor - reads daily_crawl_config.json and runs all enabled crawlers - FAST version"""
import json, subprocess, sys, os, re
from datetime import datetime

LOG = "/tmp/crawl_run.log"

def log(msg):
    ts = datetime.now().strftime("%H:%M:%S")
    line = f"[{ts}] {msg}"
    print(line, flush=True)
    with open(LOG, "a") as f:
        f.write(line + "\n")

with open("daily_crawl_config.json", "r") as f:
    cfg = json.load(f)

enabled = [c for c in cfg["crawlers"] if c.get("enabled", False)]
total = len(enabled)
results = []

log(f"Starting FAST crawl of {total} enabled crawlers (30s timeout)")

for i, c in enumerate(enabled):
    name = c["name"]
    script = c["script"]
    args = c.get("args", "").strip()
    
    if script.startswith("node "):
        cmd_parts = [s.strip() for s in script.split()]
        if args:
            cmd_parts.append(args)
    else:
        cmd_parts = ["python3", script]
        if args:
            cmd_parts.append(args)
    
    log(f"[{i+1}/{total}] {name}")
    
    try:
        r = subprocess.run(cmd_parts, capture_output=True, text=True, timeout=30, cwd="/root/gov_crawler")
        stdout = r.stdout.strip()
        stderr = r.stderr.strip()
        
        new_count = 0
        for line in stdout.split("\n"):
            m = re.search(r"新增[了]?(\d+)", line)
            if m: new_count += int(m.group(1))
            m = re.search(r"增量[了]?(\d+)", line)
            if m: new_count += int(m.group(1))
            m = re.search(r"new[:\s]*(\d+)", line.lower())
            if m: new_count += int(m.group(1))
        
        status = "OK"
        if r.returncode != 0:
            status = f"FAIL(rc={r.returncode})"
        
        results.append((name, status, new_count))
        log(f"  -> {status} | +{new_count}")
        
    except subprocess.TimeoutExpired:
        results.append((name, "TIMEOUT", 0))
        log(f"  -> TIMEOUT (30s)")
    except Exception as e:
        results.append((name, "ERROR", 0))
        log(f"  -> ERROR: {str(e)[:80]}")

# Final report
log("")
log("========== FINAL REPORT ==========")
total_new = sum(r[2] for r in results)
ok_count = sum(1 for r in results if r[1] == "OK")
fail_rc = sum(1 for r in results if r[1].startswith("FAIL"))
timeout_count = sum(1 for r in results if r[1] == "TIMEOUT")
error_count = sum(1 for r in results if r[1] == "ERROR")

log(f"Total crawlers run: {total}")
log(f"  Successful: {ok_count}")
log(f"  Failed (exit code): {fail_rc}")
log(f"  Errors: {error_count}")
log(f"  Timeouts: {timeout_count}")
log(f"Total new items added: {total_new}")

# Sites with new items
added_sites = [(r[0], r[2]) for r in results if r[2] > 0]
if added_sites:
    log(f"\nSites with new items ({len(added_sites)}):")
    for name, n in added_sites:
        log(f"  +{n}  {name}")

# Issues
issues = [r for r in results if r[1] != "OK"]
if issues:
    log(f"\nSites with issues ({len(issues)}):")
    for name, status, _ in issues:
        log(f"  [{status}] {name}")

log(f"\nLog: {LOG}")
print("\n=== FAST CRAWL COMPLETE ===", flush=True)