#!/usr/bin/env python3
import json, subprocess, sys, os, time, re
os.chdir("/root/gov_crawler")

with open("daily_crawl_config.json") as f:
    cfg = json.load(f)

crawlers = [c for c in cfg["crawlers"] if c.get("enabled", False)]
total = len(crawlers)

def run_one(name, script, args_str):
    if script.startswith("crawl_"):
        cmd = ["timeout", "120", "python3", script]
    else:
        cmd = ["timeout", "120"] + script.split()
    if args_str and args_str.strip():
        cmd.extend(args_str.split())
    start = time.time()
    try:
        r = subprocess.run(cmd, capture_output=True, text=True, timeout=130)
        elapsed = time.time() - start
        out = r.stdout or ""
        err = r.stderr or ""
        new_n = 0
        for line in out.split("\n"):
            m = re.search(r"新[增插].{0,10}?(\d+)\s*条", line)
            if m: new_n += int(m.group(1))
            m = re.search(r"added\s+(\d+)", line, re.I)
            if m: new_n += int(m.group(1))
        if r.returncode == 0:
            return (True, new_n, round(elapsed, 1))
        else:
            return (False, 0, err.strip()[-200:] or "exit " + str(r.returncode))
    except subprocess.TimeoutExpired:
        return (False, 0, "TIMEOUT")
    except Exception as e:
        return (False, 0, str(e)[:200])

total_new = 0
errors = []
results = []

for i, c in enumerate(crawlers, 1):
    name = c["name"]
    script = c["script"]
    args_str = c.get("args", "") or ""
    line = "[%d/%d] %s... " % (i, total, name)
    sys.stdout.write(line)
    sys.stdout.flush()
    
    ok, n, info = run_one(name, script, args_str)
    total_new += n
    results.append({"name": name, "script": script, "ok": ok, "new": n, "info": str(info)})
    
    if ok:
        print("+%d [%.1fs]" % (n, info))
    else:
        print("FAIL: %s" % str(info)[:120])
        errors.append("%s: %s" % (name, str(info)[:120]))

print()
print("=== CRAWL SUMMARY ===")
print("Sites: %d" % total)
print("New items: %d" % total_new)
print("Failures: %d" % len(errors))
for e in errors[:30]:
    print("  - " + e)

json.dump({"total": total, "new": total_new, "errors": errors[:50], "results": results, "ts": time.strftime("%Y-%m-%d %H:%M:%S")},
          open("/tmp/daily_done.json","w"), ensure_ascii=False, indent=2)
print()
print("Results saved to /tmp/daily_done.json")