#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""site_coverage_domain.py — 按「站点(域名)」判覆盖健康度（自校准）

为什么要换口径
--------------
2026-09-21 实证：按 script_name 判定会系统性误判，原因有二 ——
  ① 812 个脚本手写 INSERT 时漏写 script_name → 记录落在空名下，按脚本查不到；
  ② page_url 全局 UNIQUE + INSERT OR IGNORE → 同一页面被哪个脚本先插就归谁
     （实测 crawl_szepia.py 的内容挂在 crawl_eiacloud.py 名下）。
所以「某站点有没有在进数据」必须按**域名**判，与归属无关。

判定（自校准，无需人工配置）
  阈值 = max(7 天, 3 × 该站历史中位发布间隔)
  滞后 = 今天 - 库内最新发布日期
"""
import argparse
import collections
import datetime as dt
import os
import re
import sqlite3
import statistics

DB = os.getenv("SEARCH_DB", "/root/search.db")


def domain_of(u):
    m = re.match(r"^[a-z]+://([^/]+)", u or "", re.I)
    if not m:
        return None
    d = m.group(1).lower().split(":")[0]
    if d.startswith("www."):
        d = d[4:]
    return d


def norm_date(s):
    m = re.match(r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})", (s or "").strip())
    if not m:
        return None
    try:
        return dt.date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
    except ValueError:
        return None


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--days", type=int, default=400)
    ap.add_argument("--top", type=int, default=20)
    ap.add_argument("--min-records", type=int, default=20, help="少于此记录数的站点不判")
    a = ap.parse_args()

    today = dt.date.today()
    since = today - dt.timedelta(days=a.days)
    db = sqlite3.connect(DB, timeout=300)
    db.execute("PRAGMA busy_timeout=300000")
    c = db.cursor()

    print("=" * 104)
    print("按站点(域名)的覆盖健康度  窗口 %s ~ %s" % (since, today))
    print("=" * 104)

    # 流式聚合（生产机内存小，不用 fetchall）
    agg = collections.defaultdict(lambda: {"dates": collections.Counter(),
                                           "n": 0, "scripts": collections.Counter(),
                                           "undated": 0})
    cur = c.execute("""SELECT page_url, publish_date, script_name FROM gov_raw
                       WHERE page_url IS NOT NULL AND page_url != ''""")
    for url, pd_, sc in cur:
        d = domain_of(url)
        if not d:
            continue
        g = agg[d]
        g["n"] += 1
        dt_ = norm_date(pd_)
        if dt_:
            g["dates"][dt_] += 1
        else:
            g["undated"] += 1
        if sc:
            g["scripts"][sc] += 1
    print("域名数: %d" % len(agg))

    # 运行侧证据（近14天）
    d14 = (today - dt.timedelta(days=14)).isoformat()
    runs = {}
    for sc, tot, fails, newsum in c.execute("""SELECT script_name, COUNT(*),
               SUM(CASE WHEN status != '成功' THEN 1 ELSE 0 END), COALESCE(SUM(new_count),0)
               FROM run_logs WHERE run_date >= ? GROUP BY 1""", (d14,)):
        runs[sc] = (tot, fails or 0, newsum or 0)

    out = []
    for dom, g in agg.items():
        if g["n"] < a.min_records:
            continue
        ds = sorted(g["dates"].keys())
        if not ds:
            out.append({"dom": dom, "n": g["n"], "db_max": None, "stale": 9999,
                        "typical": 30, "thr": 90, "verdict": "无日期", "scripts": g["scripts"],
                        "undated": g["undated"]})
            continue
        db_max = ds[-1]
        stale = (today - db_max).days
        win = [d for d in ds if d >= since]
        if len(win) >= 3:
            gaps = [(win[i + 1] - win[i]).days for i in range(len(win) - 1)]
            gaps = [x for x in gaps if x > 0]
            typical = int(statistics.median(gaps)) if gaps else 30
        else:
            typical = 30
        thr = max(7, 3 * typical)
        # 运行侧：该域名涉及的所有脚本
        r14 = sum(runs.get(s, (0, 0, 0))[0] for s in g["scripts"])
        f14 = sum(runs.get(s, (0, 0, 0))[1] for s in g["scripts"])
        n14 = sum(runs.get(s, (0, 0, 0))[2] for s in g["scripts"])
        if stale > 2 * thr:
            verdict = "停滞"
        elif stale > thr:
            verdict = "观察"
        else:
            verdict = "正常"
        out.append({"dom": dom, "n": g["n"], "db_max": db_max.isoformat(), "stale": stale,
                    "typical": typical, "thr": thr, "verdict": verdict,
                    "scripts": g["scripts"], "r14": r14, "f14": f14, "n14": n14,
                    "undated": g["undated"]})

    order = {"停滞": 0, "观察": 1, "无日期": 2, "正常": 3}
    out.sort(key=lambda x: (order[x["verdict"]], -x["stale"]))
    cnt = collections.Counter(x["verdict"] for x in out)
    print()
    print("分级: " + " | ".join("%s %d" % (k, cnt[k]) for k in ("停滞", "观察", "正常", "无日期")))
    print("（只统计记录数 ≥%d 的域名，共 %d 个）" % (a.min_records, len(out)))

    for lvl in ("停滞", "观察"):
        items = [x for x in out if x["verdict"] == lvl]
        if not items:
            continue
        print()
        print("=" * 104)
        print("【%s】%d 个域名" % (lvl, len(items)))
        print("=" * 104)
        print("  %-34s %10s %-11s %5s %5s %8s %s" %
              ("域名", "记录数", "库内最新", "滞后", "阈值", "近14运行", "写入脚本"))
        for x in items[:a.top]:
            scs = ", ".join("%s(%d)" % (s, n) for s, n in x["scripts"].most_common(2))
            print("  %-34s %10d %-11s %5d %5d %8d %s" %
                  (x["dom"][:34], x["n"], x["db_max"], x["stale"], x["thr"], x["r14"], scs[:44]))
        if len(items) > a.top:
            print("  ... 还有 %d 个" % (len(items) - a.top))
    db.close()


if __name__ == "__main__":
    main()