#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""qc_scope_dis.py —— QC 体检·Dis-db 侧摸底（只读）
核心问题：Script_name ↔ 配置编号 是否真的对得上（编号 = config 数组 1-based index）"""
import json
import os
import re
import subprocess
from collections import Counter

DB = "22e399b8-3c5b-80f5-b6dc-c6d96cafa1bd"
TOK = "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"
CFG = "/Users/chniir0000outlook.com/Crawler/gov_crawler/daily_crawl_config.json"
D = "/Users/chniir0000outlook.com/Crawler/gov_crawler/"


def api(url, method="GET", body=None):
    cmd = ["curl", "-sS", "-X", method, "-H", "Authorization: Bearer " + TOK,
           "-H", "Notion-Version: 2022-06-28", "-H", "Content-Type: application/json",
           "--max-time", "60", url]
    if body is not None:
        cmd += ["-d", json.dumps(body, ensure_ascii=False)]
    p = subprocess.run(cmd, capture_output=True, timeout=90)
    try:
        return json.loads(p.stdout.decode("utf-8", "ignore"))
    except Exception:
        return {"_raw": p.stdout.decode("utf-8", "ignore")[:200]}


cfg = json.load(open(CFG, encoding="utf-8"))
cr = cfg["crawlers"] if isinstance(cfg, dict) and "crawlers" in cfg else cfg
print("本地 config 条目数:", len(cr))

print("\n=== dump Dis-db（分页，防静默截断）===")
rows, cur, pages = [], None, 0
while True:
    body = {"page_size": 100}
    if cur:
        body["start_cursor"] = cur
    r = api("https://api.notion.com/v1/databases/%s/query" % DB, "POST", body)
    got = r.get("results", [])
    rows += got
    pages += 1
    if not r.get("has_more"):
        break
    cur = r.get("next_cursor")
print("  记录数: %d（%d 页）| has_more 收敛 ✅" % (len(rows), pages))


def txt(pr, k):
    v = pr.get(k) or {}
    t = v.get("rich_text") or v.get("title") or []
    return "".join(x.get("plain_text", "") for x in t).strip()


def num(pr, k):
    return (pr.get(k) or {}).get("number")


recs = []
for pg in rows:
    pr = pg["properties"]
    recs.append({
        "id": pg["id"], "name": txt(pr, "站点名"), "url": (pr.get("Url") or {}).get("url") or "",
        "script": txt(pr, "Script_name"), "no": num(pr, "配置编号"),
        "note": txt(pr, "备注") or str(num(pr, "备注") or ""),
        "scrapy": txt(pr, "Scrapy"),
    })

both = [r for r in recs if r["script"] and r["no"]]
only_s = [r for r in recs if r["script"] and not r["no"]]
only_n = [r for r in recs if not r["script"] and r["no"]]
none = [r for r in recs if not r["script"] and not r["no"]]
print("\n=== ① 填写情况 ===")
print("  脚本+编号都有: %d" % len(both))
print("  只有 Script_name: %d" % len(only_s))
print("  只有 编号: %d" % len(only_n))
print("  两个都空: %d" % len(none))
print("  备注 取值分布:", Counter(r["note"] for r in recs).most_common(5))

print("\n=== ② 核心：编号 ↔ 脚本 是否一一对应（编号=config[编号-1]）===")
ok = bad_no = bad_idx = bad_s = 0
samples_idx, samples_no = [], []
for r in both:
    n = int(r["no"])
    if not (1 <= n <= len(cr)):
        bad_no += 1
        samples_no.append(r)
        continue
    expect = (cr[n - 1].get("script") or "")
    if expect == r["script"]:
        ok += 1
    else:
        bad_idx += 1
        samples_idx.append((r, expect))
        continue
    if not os.path.exists(D + r["script"]):
        bad_s += 1
print("  ✅ 完全自洽: %d" % ok)
print("  ⚠️ 编号越界(> %d): %d" % (len(cr), bad_no))
print("  ⚠️ 编号指向的是别的脚本: %d" % bad_idx)
print("  ⚠️ 脚本名对应但磁盘不存在: %d" % bad_s)
for r, exp in samples_idx[:8]:
    print("     #%s %s | Dis写=%s | config[%s]=%s" % (r["no"], r["name"][:20], r["script"], int(r["no"]), exp or "(空)"))
for r in samples_no[:5]:
    print("     编号越界 → #%s %s | %s" % (r["no"], r["name"][:20], r["script"]))

print("\n=== ③ 倒序看新注册的一批（编号最大 12 条）===")
for r in sorted([x for x in recs if x["no"]], key=lambda x: -int(x["no"]))[:12]:
    n = int(r["no"])
    exp = (cr[n - 1].get("script") or "(越界)") if 1 <= n <= len(cr) else "(越界)"
    flag = "✅" if exp == r["script"] else "❌"
    print("  %s #%-5s %-24s Dis=%-26s config=%s" % (flag, n, r["name"][:24], r["script"] or "(空)", exp))

print("\n=== ④ Dis 有记录但库内/磁盘都对不上的 ===")
print("  Dis 里出现过的脚本数:", len(set(r["script"] for r in recs if r["script"])))
miss_disk = sorted(set(r["script"] for r in recs if r["script"] and not os.path.exists(D + r["script"])))
print("  ⚠️ Dis 写了但磁盘没这个脚本:", len(miss_disk), miss_disk[:10])
reg = set(c.get("script") for c in cr)
print("  ⚠️ Dis 写了但 config 里没有的脚本:", len(set(r["script"] for r in recs if r["script"]) - reg),
      sorted(set(r["script"] for r in recs if r["script"]) - reg)[:8])
neg = sorted(set(r["script"] for r in recs if r["script"] and not os.path.exists(D + r["script"]))
             & set(x for x in os.listdir(D) if x.endswith(".py")))
print("  （其中在磁盘上确实存在的）:", len(neg))