#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""qc_dis_offset.py —— 挖 Dis「配置编号」的真实口径（缓存 dump，做偏移直方图）"""
import json
import os
import subprocess
from collections import Counter

DB = "22e399b8-3c5b-80f5-b6dc-c6d96cafa1bd"
TOK = "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"
CACHE = "/tmp/dis_dump_20260925.json"
CFG = "/Users/chniir0000outlook.com/Crawler/gov_crawler/daily_crawl_config.json"
D = "/Users/chniir0000outlook.com/Crawler/gov_crawler/"


def api(url, body=None):
    cmd = ["curl", "-sS", "-X", "POST", "-H", "Authorization: Bearer " + TOK,
           "-H", "Notion-Version: 2022-06-28", "-H", "Content-Type: application/json",
           "--max-time", "60", url]
    if body is not None:
        cmd += ["-d", json.dumps(body, ensure_ascii=False)]
    p = subprocess.run(cmd, capture_output=True, timeout=90)
    return json.loads(p.stdout.decode("utf-8", "ignore"))


if os.path.exists(CACHE):
    recs = json.load(open(CACHE, encoding="utf-8"))
    print("用缓存 %s（%d 条）" % (CACHE, len(recs)))
else:
    recs, cur = [], None
    while True:
        body = {"page_size": 100}
        if cur:
            body["start_cursor"] = cur
        r = api("https://api.notion.com/v1/databases/%s/query" % DB, body)
        recs += r.get("results", [])
        if not r.get("has_more"):
            break
        cur = r.get("next_cursor")
    out = []
    for pg in recs:
        pr = pg["properties"]

        def txt(k):
            v = pr.get(k) or {}
            t = v.get("rich_text") or v.get("title") or []
            return "".join(x.get("plain_text", "") for x in t).strip()
        out.append({"name": txt("站点名"), "url": (pr.get("Url") or {}).get("url") or "",
                    "script": txt("Script_name"), "no": (pr.get("配置编号") or {}).get("number")})
    json.dump(out, open(CACHE, "w", encoding="utf-8"), ensure_ascii=False)
    recs = out
    print("已缓存", CACHE, len(recs))

cfg = json.load(open(CFG, encoding="utf-8"))
cr = cfg["crawlers"] if isinstance(cfg, dict) and "crawlers" in cfg else cfg

# 建 script → [1-based positions]
pos = {}
for i, c in enumerate(cr):
    pos.setdefault(c.get("script") or "", []).append(i + 1)

both = [r for r in recs if r.get("script") and r.get("no")]
print("\n=== 假设检验：Dis编号 N 与 config 里该脚本的位置 P 的关系 ===")
h = Counter()
notfound = 0
for r in both:
    ps = pos.get(r["script"])
    if not ps:
        notfound += 1
        continue
    n = int(r["no"])
    # 取最接近 N 的那个位置
    best = min(ps, key=lambda p: abs(p - n))
    h[best - n] += 1
print("  差值(P - N) 直方图 top12:", h.most_common(12))
print("  Dis 写的脚本在 config 里查无:", notfound)

# 单一位置脚本的纯净偏移
pure = [r for r in both if len(pos.get(r["script"], [])) == 1]
hp = Counter()
for r in pure:
    p = pos[r["script"]][0]
    hp[p - int(r["no"])] += 1
print("  仅取「该脚本在 config 只出现一次」的 %d 条，差值直方图:" % len(pure))
for d, c in hp.most_common(12):
    print("     P-N = %+5d → %d 条" % (d, c))

print("\n=== 抽样看几条「P-N」不同的 ===")
seen = set()
for r in pure:
    p = pos[r["script"]][0]
    d = p - int(r["no"])
    if d not in seen and len(seen) < 8:
        seen.add(d)
        print("  N=%-5s P=%-5s (P-N=%+d) %-26s %s" % (r["no"], p, d, r["name"][:26], r["script"]))

print("\n=== 再看：Dis编号 与 config 的 group 是否相关（编号是否按组内序号）===")
gpos = {}
for i, c in enumerate(cr):
    gpos.setdefault(c.get("group") or "", []).append((i + 1, c.get("script")))
# 抽 3 个组看组内序号
for g in ["安徽", "江西", "浙江"]:
    seq = gpos.get(g, [])
    print("  组 %s 共 %d 条，前 5 个(组内序:1-based config号): %s" % (
        g, len(seq), ["%d:%s" % (k + 1, s) for k, (cno, s) in enumerate(seq[:5])]))