#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""单条挤锁清理栩辰 42315/42352（busy_timeout=390s 挤调度器锁间隙）"""
import re
import sqlite3
import sys

sys.path.insert(0, '/root/gov_crawler')

db = sqlite3.connect('/mnt/data/search.db', timeout=390)
db.execute('PRAGMA busy_timeout=390000')


def clean(c):
    if not c:
        return c
    c = re.sub(r'<a href="javascript:[^"]*">\s*关闭\s*</a>', '', c)
    c = re.sub(r'<div class="zw-box-print">.*?</div>', '', c, flags=re.DOTALL)
    c = re.sub(
        r'<div class="zw-leader-jl">\s*<h3><span>相关(?:链接|文档|图片|音频|视频)</span></h3>.*?</div>',
        '', c, flags=re.DOTALL)

    def att_clean(m):
        paras = []
        for am in re.finditer(r'<a[^>]*href=[\'"]([^\'"]+)[\'"][^>]*>(.*?)</a>', m.group(0), re.DOTALL):
            href, text = am.group(1).strip(), re.sub(r'<[^>]+>', '', am.group(2)).strip()
            if href and text:
                paras.append('<p><a href="{}">{}</a></p>'.format(href, text))
        return '\n'.join(paras)

    c = re.sub(
        r'<div class="zw-leader-jl">\s*<h3><span>相关附件</span></h3>.*?</div>',
        att_clean, c, flags=re.DOTALL)
    return c


for rid in (42315, 42352):
    r = db.execute('SELECT content FROM gov_raw WHERE id=?', (rid,)).fetchone()
    if not r:
        print(rid, 'NOT FOUND')
        continue
    nc = clean(r[0])
    db.execute('UPDATE gov_raw SET content=? WHERE id=?', (nc, rid))
    db.commit()
    print(rid, 'CLEANED', len(nc), '字 含噪声:', '相关链接' in nc)
db.close()