#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""qc_d5_recheck.py —— D5 重测（新判据）：按「脚本目标域名」比对库内实际有数据的域名

不用 config 的 name / script_name，改用 url 字段的域名 vs 库内 page_url 域名集合
"""
import json, re, sqlite3
from collections import Counter
D = "/root/gov_crawler/"
db = sqlite3.connect("file:/root/search.db?mode=ro", uri=True, timeout=30)
db.execute("PRAGMA busy_timeout=30000")

print("扫描库内域名集合（一次全扫）...")
hosts = set()
for r in db.execute("SELECT DISTINCT substr(page_url, instr(page_url,'//')+2, "
                    "CASE WHEN instr(substr(page_url, instr(page_url,'//')+2),'/')>0 "
                    "THEN instr(substr(page_url, instr(page_url,'//')+2),'/')-1 ELSE 999 END) FROM gov_raw"):
    h = (r[0] or "").lower().strip()
    if h: hosts.add(h)
print("  库内域名数: %d" % len(hosts))
# 去 www. 后也能匹配
hosts_nw = set(h[4:] if h.startswith("www.") else h for h in hosts)

CFG = json.load(open(D + "daily_crawl_config.json", encoding="utf-8"))
cfg = CFG if isinstance(CFG, list) else CFG.get("tasks", CFG.get("scripts", []))

def dom(u):
    m = re.match(r"https?://([^/]+)", u or "")
    return (m.group(1).lower() if m else "")

no_data, no_url, ok = [], [], 0
for c in cfg:
    if not c.get("enabled", True): continue
    s = (c.get("script") or "").strip()
    u = c.get("url") or ""
    h = dom(u)
    if not h:
        no_url.append((s, c.get("name") or "")); continue
    hn = h[4:] if h.startswith("www.") else h
    if h in hosts or hn in hosts_nw or any(x.endswith("." + hn) for x in hosts_nw):
        ok += 1
    else:
        no_data.append((s, c.get("name") or "", u[:60]))
print("\n=== D5 重测结果（启用脚本 %d 个）===" % sum(1 for c in cfg if c.get("enabled", True)))
print("  域名命中库内数据   : %d ✅" % ok)
print("  域名不在库内（真无数据）: %d ⚠️" % len(no_data))
print("  config 无 url 字段    : %d" % len(no_url))
print("\n=== 真·无数据样例（最多 15，按脚本）===")
for s, n, u in no_data[:15]:
    print("  %-34s %-22s %s" % (s[:34], n[:22], u))
json.dump({"ok": ok, "no_data": no_data, "no_url": no_url},
          open(D + "qc_out/qc_d5_recheck_20260925.json", "w", encoding="utf-8"),
          ensure_ascii=False, indent=1)
print("\n已写 qc_out/qc_d5_recheck_20260925.json")
db.close()