#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""qc_d5_batch.py —— D5「真无数据」逐个定性（第 N 批）用法: qc_d5_batch.py 0 10"""
import json, os, re, sqlite3, ssl, sys, urllib.request
D = "/root/gov_crawler/"
b0, b1 = int(sys.argv[1]), int(sys.argv[2])
d = json.load(open(D + "qc_out/qc_d5_recheck_20260925.json", encoding="utf-8"))
tgt = d["no_data"][b0:b1]
db = sqlite3.connect("file:/root/search.db?mode=ro", uri=True, timeout=30)
db.execute("PRAGMA busy_timeout=30000")
ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
UA = {"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"}
def probe(u, referer=None):
    try:
        h = dict(UA)
        if referer: h["Referer"] = referer
        req = urllib.request.Request(u, headers=h)
        r = urllib.request.urlopen(req, timeout=12, context=ctx)
        b = r.read(200000)
        return r.status, len(b)
    except urllib.error.HTTPError as e:
        return e.code, 0
    except Exception as e:
        return type(e).__name__, 0
print("=== D5 第 %d-%d 个定性 ===" % (b0+1, b1))
for s, n, host in tgt:
    p = D + s
    src = open(p, encoding="utf-8", errors="ignore").read() if os.path.exists(p) else ""
    urls = re.findall(r"https?://[^\s\"'<>)]+", src)
    list_url = urls[0] if urls else ""
    for u in urls:
        if re.search(r"list|index|tzgg|gsgg|hpsp|notice", u, re.I): list_url = u; break
    code, ln = probe(list_url, "https://" + host + "/") if list_url else ("NO-URL", 0)
    # 该基础域下库内是否有其它主机有数据
    base = ".".join(host.split(".")[-2:]) if host.count(".") >= 2 else host
    others = db.execute("SELECT COUNT(DISTINCT substr(page_url, instr(page_url,'//')+2, "
                        "CASE WHEN instr(substr(page_url, instr(page_url,'//')+2),'/')>0 "
                        "THEN instr(substr(page_url, instr(page_url,'//')+2),'/')-1 ELSE 999 END)) "
                        "FROM gov_raw WHERE page_url LIKE ?", ("%//%"+base+"%",)).fetchone()[0]
    last = db.execute("SELECT MAX(run_date)||' '||(SELECT status FROM run_logs r2 WHERE r2.script_name=r1.script_name ORDER BY run_date DESC, run_time DESC LIMIT 1) "
                      "FROM run_logs r1 WHERE script_name=?", (s,)).fetchone()[0] or "无记录"
    print("\n── %s | %s | %s" % (s, n[:20], host))
    print("   列表URL: %s" % list_url[:78])
    print("   探测: HTTP/err=%s 字节=%s | 同域其它主机在库内=%d | 末次: %s" % (code, ln, others, last[:28]))