#!/usr/bin/env python3
"""
validate_crawler_config.py
校验 daily_crawl_config.json 与磁盘脚本文件的双向一致性。
在服务器上每天跑一次，发现不一致就告警。

用法: python3 validate_crawler_config.py
"""

import json, os, sys

GOV_DIR = '/root/gov_crawler/'
CONFIG_FILE = os.path.join(GOV_DIR, 'daily_crawl_config.json')


def main():
    errors = []

    # 1. Load config — it's a list of crawler entries directly
    with open(CONFIG_FILE) as f:
        crawlers = json.load(f)

    if not isinstance(crawlers, list):
        crawlers = crawlers.get('crawlers', [])

    # 2. Build config index (script -> entry)
    config_scripts = {}  # script name -> entry
    for entry in crawlers:
        script = entry.get('script', '')
        if script:
            config_scripts[script] = entry

    # 3. Scan disk for all crawl scripts (Python + Node.js)
    disk_scripts = set()
    for fname in os.listdir(GOV_DIR):
        if fname.startswith('crawl_') and (fname.endswith('.py') or fname.endswith('.js')):
            disk_scripts.add(fname)

    # 4. Check: scripts in config but MISSING on disk
    config_missing = []
    for script in sorted(config_scripts.keys()):
        if script.startswith('node '):
            actual_file = script[5:]
        else:
            actual_file = script
        if actual_file not in disk_scripts:
            config_missing.append(script)

    if config_missing:
        errors.append('配置中存在但磁盘上缺失 ({}个):'.format(len(config_missing)))
        for s in config_missing:
            errors.append('    ' + s)

    # 5. Check: scripts on disk but NOT in config
    skip_list = {'crawl_TEMPLATE.py', 'crawl_yidu_backup.py'}
    config_script_names = set()
    for s in config_scripts.keys():
        if s.startswith('node '):
            config_script_names.add(s[5:])
        else:
            config_script_names.add(s)

    disk_orphans = sorted(disk_scripts - config_script_names - skip_list)
    if disk_orphans:
        n = len(disk_orphans)
        errors.append('磁盘上存在但配置中缺失 ({}个):'.format(n))
        for s in disk_orphans[:10]:
            errors.append('    ' + s)
        if n > 10:
            errors.append('    ... 还有{}个'.format(n - 10))

    # 6. Summary
    print('爬虫配置校验报告')
    print('=' * 40)
    print('配置爬虫数: {}'.format(len(crawlers)))
    print('磁盘脚本数: {} (排除模板/备份后: {})'.format(
        len(disk_scripts), len(disk_scripts - skip_list)))
    print('配置唯一脚本: {}'.format(len(config_scripts)))

    if errors:
        print()
        print('发现 {} 个问题:'.format(len(errors)))
        for e in errors:
            print('  ' + e)
        return 1
    else:
        print()
        print('配置与磁盘完全一致，无孤儿脚本')
        return 0


if __name__ == '__main__':
    sys.exit(main())