#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""check_danzhai.py —— step4 数据质量细查"""
import json
import re

rows = [json.loads(l) for l in open("/tmp/danzhai_p1.jsonl", encoding="utf-8")]
print("条数:", len(rows))
print()
ell = [r["title"] for r in rows if ("…" in r["title"] or r["title"].rstrip().endswith(("...", "..")))]
print("省略号/截断标题:", len(ell), ell[:3])
print("标题长度 min/max:", min(len(r["title"]) for r in rows), "/", max(len(r["title"]) for r in rows))
print()
print("=== 全部标题（逐条）===")
for i, r in enumerate(rows, 1):
    print("  %2d [%s] %s" % (i, r["pub_date"], r["title"]))
print()
print("实体残留(& &#  ):", [r["title"] for r in rows if re.search(r"&(?:amp|#\d+|nbsp)", r["title"])])
print("实体残留(正文):", [r["title"][:24] for r in rows if re.search(r"&(?:amp|#\d+|nbsp);", r["content"])])
print("附件条数:", sum(1 for r in rows if r["attachments"]))
print("正文含 <p> 条数:", sum(1 for r in rows if "<p" in r["content"]), "/", len(rows))
print("正文含 <table> 条数:", sum(1 for r in rows if "<table" in r["content"]))
print("正文长度 min/avg/max:", min(len(r["content"]) for r in rows),
      int(sum(len(r["content"]) for r in rows) / len(rows)), max(len(r["content"]) for r in rows))
print()
print("=== 首条正文全文（前 700 字）===")
print(rows[0]["content"][:700])
print()
print("=== 污染词扫描 ===")
SUS = ["返回顶部", "打印本页", "关闭窗口", "扫一扫", "上一篇", "下一篇", "字号", "分享到",
       "ArchiveGdPart", "已归档", "百度网盘提取码", "javascript", "首页", "网站地图",
       "主办单位", "版权所有", "ICP备"]
for s in SUS:
    hit = [r["title"][:20] for r in rows if s in r["content"]]
    if hit:
        print("  ⚠️ %-14s 命中 %d 条: %s" % (s, len(hit), hit[:3]))
print("  扫描完毕")