#!/usr/bin/env python3
"""
配置文件去重工具 - v2（改进版）
只删除真正重复的条目：name + script + args 三者完全一致才视为重复。
同名但不同脚本/args 的保留（它们是不同的爬虫）。
"""
import json
from datetime import datetime
from collections import defaultdict

CONFIG_PATH = "/root/gov_crawler/daily_crawl_config.json"

def make_key(c):
    """唯一标识：(name, script, args)"""
    args = c.get("args", "")
    if isinstance(args, list):
        args = " ".join(str(a) for a in args)
    return (c["name"], c.get("script", ""), str(args))

def read_config():
    with open(CONFIG_PATH) as f:
        return json.load(f)

def write_config(config):
    with open(CONFIG_PATH, "w") as f:
        json.dump(config, f, ensure_ascii=False, indent=2)

def main():
    config = read_config()
    crawlers = config["crawlers"]
    print(f"Before: {len(crawlers)} entries")

    by_key = defaultdict(list)
    for i, c in enumerate(crawlers):
        by_key[make_key(c)].append((i, c))

    to_remove = []
    for key, entries in by_key.items():
        if len(entries) > 1:
            for idx, entry in entries[1:]:
                to_remove.append(idx)
                print(f"  REMOVE: {entry['name']} | {entry.get('script','')} | args={entry.get('args','')}")

    for idx in sorted(to_remove, reverse=True):
        crawlers.pop(idx)

    config["crawlers"] = crawlers
    config["_更新日期"] = datetime.now().strftime("%Y-%m-%d %H:%M")
    write_config(config)

    print(f"After: {len(crawlers)} entries (removed {len(to_remove)} true duplicates)")
    print(f"\nSkipped (legitimate same-name-different-script):")
    by_name = defaultdict(list)
    for c in crawlers:
        by_name[c["name"]].append((c.get("script",""), c.get("args","")))
    for name, entries in sorted(by_name.items()):
        if len(entries) > 1:
            print(f"  {name}: {len(entries)} variants")
            for script, args in entries:
                print(f"    script={script} | args={args}")

if __name__ == "__main__":
    main()