#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""qc_diag_persistent.py —— 持续型失败脚本的真实报错取证 + 按签名归类"""
import json
import os
import re
import sqlite3
from collections import Counter, defaultdict

D = "/root/gov_crawler/"
db = sqlite3.connect("file:/root/search.db?mode=ro", uri=True, timeout=30)
db.execute("PRAGMA busy_timeout=30000")

TO = json.load(open(D + "qc_out/qc_timeout_20260925.json", encoding="utf-8"))
pers = [x for x in TO if x["分类"] == "持续型"]
print("持续型 %d 个\n" % len(pers))


def sig(t):
    """把报错文本压成签名，便于归类"""
    t = re.sub(r"\d+", "N", (t or "")[:400])
    t = re.sub(r"[a-zA-Z0-9_./-]{20,}", "<TOK>", t)
    t = re.sub(r"\s+", " ", t)
    for k in ["Traceback (most recent call last)", "ModuleNotFoundError", "ImportError",
              "KeyError", "TypeError", "AttributeError", "IndexError", "ValueError",
              "SyntaxError", "NameError", "UnicodeDecodeError", "ConnectionError",
              "SSLError", "MaxRetriesExceededError", "Timeout", "JSONDecodeError",
              "sqlite3.", "requests.exceptions", "urllib3", "FileNotFoundError",
              "empty list", "no attribute", "cannot identify image", "PermissionError"]:
        if k in (t or ""):
            return k
    return "其他"


groups = defaultdict(list)
for x in pers:
    sc = x["script"]
    rows = list(db.execute(
        "SELECT run_date, run_time, status, error_detail, new_count, elapsed_seconds "
        "FROM run_logs WHERE script_name=? ORDER BY run_date DESC, run_time DESC LIMIT 4", (sc,)))
    ok = db.execute("SELECT MAX(run_date) FROM run_logs WHERE script_name=? AND status='成功'", (sc,)).fetchone()[0]
    n_ok = db.execute("SELECT COUNT(*) FROM run_logs WHERE script_name=? AND status='成功'", (sc,)).fetchone()[0]
    err = next((r[3] for r in rows if r[3]), "")
    x["last_ok"] = ok or "从未成功"
    x["n_ok"] = n_ok
    x["err"] = err
    x["last3"] = [(r[0], r[2], (r[3] or "")[:70]) for r in rows[:3]]
    groups[sig(err)].append(x)

print("=== 按错误签名归类 ===")
for k, v in sorted(groups.items(), key=lambda kv: -len(kv[1])):
    print("  %-32s %2d 个: %s" % (k, len(v), ", ".join(x["script"] for x in v)[:150]))

print("\n=== 逐个详情（含真实报错）===")
for x in pers:
    print("\n── %s ｜ %s ｜ 运行%d 失败%d(%s) ｜ 成功 %d 次，末次成功 %s" % (
        x["script"], x["类型"], x["运行"], x["失败"], x["失败率"], x["n_ok"], x["last_ok"]))
    print("   站点:", (x["站点"] or "?")[:40], "| 组:", x["组"])
    p = D + x["script"]
    if os.path.exists(p):
        st = os.stat(p)
        import datetime
        print("   脚本存在 | %d 字节 | 改于 %s" % (st.st_size, datetime.date.fromtimestamp(st.st_mtime)))
        import py_compile
        try:
            py_compile.compile(p, doraise=True, cfile="/tmp/_qc.pyc")
            print("   py_compile ✅")
        except Exception as e:
            print("   py_compile ❌ %s" % str(e)[:90])
    else:
        print("   ⚠️ 脚本文件不存在")
    if x["err"]:
        print("   error_detail:", re.sub(r"\s+", " ", x["err"])[:330])
    else:
        print("   error_detail: （空）")
        for d0, s0, e0 in x["last3"]:
            print("      %s %s %s" % (d0, s0, e0))

json.dump([{k: v for k, v in x.items() if k != "last3"} for x in pers],
          open(D + "qc_out/persistent_diag.json", "w", encoding="utf-8"),
          ensure_ascii=False, indent=1)
print("\n已写 qc_out/persistent_diag.json")
db.close()