#!/usr/bin/env python3
import json, subprocess, sys, os, re
from collections import defaultdict
from datetime import datetime

os.chdir('/root/gov_crawler')

with open('daily_crawl_config.json', 'r') as f:
    config = json.load(f)

crawlers = config['crawlers']
enabled = [c for c in crawlers if c.get('enabled', False)]

log_dir = "logs/daily_" + datetime.now().strftime('%Y%m%d_%H%M%S')
os.makedirs(log_dir, exist_ok=True)

results = []
total_new = 0
success_count = 0
fail_count = 0
total = len(enabled)

print("Total crawlers: " + str(total))
print("Start: " + datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
print("=" * 60)

for idx, c in enumerate(enabled):
    script = c['script']
    name = c.get('name', 'unknown')
    args = c.get('args', '')
    group = c.get('group', c.get('source', 'other'))

    if script.startswith('node '):
        cmd = script
        if args:
            cmd += ' ' + args
    else:
        cmd = 'python3 ' + script
        if args:
            cmd += ' ' + args

    print("\n[{}/{}] {}".format(idx+1, total, name))
    print("    > " + cmd)
    sys.stdout.flush()

    try:
        result = subprocess.run(
            cmd,
            shell=True,
            cwd='/root/gov_crawler',
            capture_output=True,
            text=True,
            timeout=300
        )

        stdout = result.stdout
        stderr = result.stderr
        combined = stdout + stderr

        new_count = 0
        for p in [
            r'\u65b0\u589e\D*?(\d+)', r'\u65b0\u589e\u4e86?\s*(\d+)', r'(\d+)\s*\u6761?\u65b0\u589e',
            r'successfully\w*\s*(\d+)', r'saved\s*(\d+)',
            r'\u63d2\u5165\D*?(\d+)', r'\u5bfc\u5165\D*?(\d+)',
            r'\u63d0\u4ea4\u6210\u529f\D*?(\d+)', r'\u722c\u53d6\u5b8c\u6210\D*?(\d+)',
        ]:
            m = re.findall(p, combined, re.IGNORECASE)
            if m:
                new_count = max(new_count, max(int(x) for x in m))
                break

        if result.returncode == 0:
            success_count += 1
            total_new += new_count
            status = 'OK'
        else:
            fail_count += 1
            status = 'FAIL'

        preview = combined[:150].replace('\n', ' | ')
        results.append({
            'name': name, 'script': script, 'status': status,
            'new_count': new_count, 'group': group,
            'preview': preview, 'rc': result.returncode
        })
        print("    [{}] new: {}".format(status, new_count))

        if result.returncode != 0:
            print("    ERR: " + stderr[:200])

    except subprocess.TimeoutExpired:
        fail_count += 1
        results.append({
            'name': name, 'script': script, 'status': 'TIMEOUT',
            'new_count': 0, 'group': group,
            'preview': 'Timeout 300s', 'rc': -1
        })
        print("    [TIMEOUT]")

    except Exception as e:
        fail_count += 1
        results.append({
            'name': name, 'script': script, 'status': 'ERROR',
            'new_count': 0, 'group': group,
            'preview': str(e)[:200], 'rc': -2
        })
        print("    [ERROR] " + str(e))

# Report
end_time = datetime.now()
print("\n" + "=" * 60)
print("Complete! Time: " + end_time.strftime('%Y-%m-%d %H:%M:%S'))
print("Success: {}, Failed: {}, Total: {}".format(success_count, fail_count, total))
print("Total new items: {}".format(total_new))
print("=" * 60)

group_stats = defaultdict(lambda: {'ok': 0, 'fail': 0, 'new': 0, 'total': 0})
for r in results:
    g = r['group']
    group_stats[g]['total'] += 1
    if r['status'] == 'OK':
        group_stats[g]['ok'] += 1
    else:
        group_stats[g]['fail'] += 1
    group_stats[g]['new'] += r['new_count']

print("\n--- By Group ---")
print("{:<30} {:>6} {:>6} {:>6} {:>6}".format('Group', 'Total', 'OK', 'Fail', 'New'))
print("-" * 60)
for g, s in sorted(group_stats.items()):
    print("{:<30} {:>6} {:>6} {:>6} {:>6}".format(g, s['total'], s['ok'], s['fail'], s['new']))

report_path = os.path.join(log_dir, 'report.txt')
with open(report_path, 'w') as f:
    f.write("Daily Crawl Report\n")
    f.write("Time: {}\n".format(end_time.strftime('%Y-%m-%d %H:%M:%S')))
    f.write("Total: {}, OK: {}, Fail: {}\n".format(total, success_count, fail_count))
    f.write("Total New: {}\n\n".format(total_new))
    f.write("--- By Group ---\n")
    f.write("{:<30} {:>6} {:>6} {:>6} {:>6}\n".format('Group', 'Total', 'OK', 'Fail', 'New'))
    f.write("-" * 60 + "\n")
    for g, s in sorted(group_stats.items()):
        f.write("{:<30} {:>6} {:>6} {:>6} {:>6}\n".format(g, s['total'], s['ok'], s['fail'], s['new']))
    f.write("\n--- Detail ---\n")
    for r in results:
        f.write("[{}] {}: new={} | {}\n".format(r['status'], r['name'], r['new_count'], r['preview']))

print("\nReport saved: " + report_path)

failed = [r for r in results if r['status'] != 'OK']
if failed:
    print("\n--- Failed ({}) ---".format(len(failed)))
    for r in failed:
        print("  [{}] {}: {}".format(r['status'], r['name'], r['preview'][:120]))

top_new = sorted(results, key=lambda r: r['new_count'], reverse=True)
top_new = [r for r in top_new if r['new_count'] > 0]
if top_new:
    print("\n--- Top New ---")
    for r in top_new[:10]:
        print("  +{} {}".format(r['new_count'], r['name']))