#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""2026-08-11 荆州正文尾部噪声第二轮清理:
① jzgov-qrcode-div 二维码区 (3050 条)
② zw-box-print 打印区 (51 条，嵌套结构)
③ 相关链接/文档/图片/音频/视频 空分类残留 (2 条)
用 BeautifulSoup 容错处理嵌套 div
"""
import re
import sqlite3
from bs4 import BeautifulSoup

DB = '/mnt/data/search.db'
SITE = '荆州市生态环境局'


def clean(c):
    if not c:
        return c
    soup = BeautifulSoup(c, 'html.parser')
    changed = False
    # ① 二维码区
    for el in list(soup.select('div.jzgov-qrcode-div')):
        el.decompose()
        changed = True
    for el in list(soup.select('div.jzgov-mt-40')):
        el.decompose()
        changed = True
    # ② 打印区 zw-box-print (含嵌套)
    for el in list(soup.select('div.zw-box-print')):
        el.decompose()
        changed = True
    # ③ 空分类 zw-leader-jl (相关链接/文档/图片/音频/视频)
    for el in list(soup.select('div.zw-leader-jl')):
        h3 = el.find('h3')
        if h3 and any(k in h3.get_text() for k in ['相关链接', '相关文档', '相关图片', '相关音频', '相关视频']):
            # 空分类 div: 若无附件链接则删除
            if not el.find('a', href=True):
                el.decompose()
                changed = True
    # ④ 扫一扫文字残留
    for el in list(soup.find_all(string=re.compile('扫一扫在手机上查看当前页面'))):
        p = el.find_parent('div')
        if p:
            p.decompose()
            changed = True
    if not changed:
        return c
    # 清理空段落
    out = str(soup)
    out = re.sub(r'<p>\s*</p>', '', out)
    out = re.sub(r'<!-- 扫码手机查看 -->', '', out)
    out = re.sub(r'<!--打印-分享-关闭-->', '', out)
    out = re.sub(r'\n{3,}', '\n\n', out)
    return out


conn = sqlite3.connect(DB, timeout=60)
conn.execute('PRAGMA busy_timeout=290000')

rows = conn.execute(
    "SELECT id, content FROM gov_raw WHERE site_name=? AND "
    "(content LIKE '%jzgov-qrcode%' OR content LIKE '%zw-box-print%' OR content LIKE '%zw-leader-jl%' OR content LIKE '%扫一扫%')",
    (SITE,)
).fetchall()
print(f'待清理: {len(rows)} 条')

ok = skip = 0
for rid, content in rows:
    nc = clean(content)
    if nc == content:
        skip += 1
        continue
    try:
        conn.execute('UPDATE gov_raw SET content=? WHERE id=?', (nc, rid))
        conn.commit()
        ok += 1
    except Exception as e:
        print(f'  [{rid}] ERR {e}')
print(f'完成: 清理 {ok} / 跳过 {skip} / 总 {len(rows)}')
conn.close()