#!/usr/bin/env python3
"""Fix 0-char records with <br>-based UCAP content."""
import subprocess
import sqlite3
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB = '/mnt/data/search.db'

def clean_br_content(html_content, detail_url):
    if not html_content:
        return ''
    soup = BeautifulSoup(html_content, 'html.parser')
    for tag in soup.select('script, style, link, #asbox, .zjsc'):
        tag.decompose()
    ucap = soup.find('ucapcontent')
    content = ucap if ucap else soup.select_one('div.main#new_content')
    if not content:
        content = soup.select_one('div.detail')
    if not content:
        return ''
    
    paras = []
    for child in list(content.children):
        if child.name is None:
            t = str(child).strip()
            if t and len(t) > 2:
                paras.append(t)
            continue
        if child.name in ['br']:
            continue
        if child.name in ['p', 'div', 'section', 'h1', 'h2', 'h3', 'h4']:
            t = child.get_text(strip=True)
            if t:
                paras.append(t)
    
    text = '\n\n'.join(paras)
    
    links = []
    if len(text) < 500:
        for img in content.select('img[src]'):
            src = urljoin(detail_url, img['src'])
            links.append('[图片] ' + src)
        for a in content.select('a[href]'):
            href = a['href']
            full_url = urljoin(detail_url, href)
            if any(href.lower().endswith(ext) for ext in ['.pdf','.doc','.docx','.xls','.xlsx']):
                fname = a.get_text(strip=True) or href.split('/')[-1]
                links.append('[附件: ' + fname + '] ' + full_url)
    
    result = text
    if links:
        result += '\n\n' + '\n'.join(links)
    return result.strip()


conn = sqlite3.connect(DB)
rows = conn.execute(
    "SELECT id, source_url FROM gov_raw WHERE site_name='犍为县人民政府-环境保护' AND (content IS NULL OR length(content) < 5)"
).fetchall()
print('Found ' + str(len(rows)) + ' 0-char records')

fix_fts = []
for rid, src_url in rows:
    r = subprocess.run(['curl', '-sS', '-L', '--max-time', '15', '-k', src_url],
                       capture_output=True, timeout=30)
    if r.returncode != 0 or not r.stdout:
        print('  FAIL fetch id=' + str(rid))
        continue
    html = r.stdout.decode('utf-8', errors='replace')
    body = clean_br_content(html, src_url)
    if body and len(body) > 5:
        body_esc = body.replace("'", "''")
        sql = "UPDATE gov_raw SET content='" + body_esc + "' WHERE id=" + str(rid)
        conn.execute(sql)
        conn.commit()
        fix_fts.append(rid)
        print('  FIXED id=' + str(rid) + ': ' + str(len(body)) + ' chars')
    else:
        print('  STILL EMPTY id=' + str(rid))

for rid in fix_fts:
    row = conn.execute("SELECT title, site_name, summary FROM gov_raw WHERE id=?", (rid,)).fetchone()
    if row:
        t, s, summ = row
        st = (summ or t)[:200]
        te = t.replace("'", "''")
        se = s.replace("'", "''")
        ste = st.replace("'", "''")
        sql = "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES(" + str(rid) + ", '" + te + "', '" + se + "', '" + ste + "');"
        subprocess.run(['sqlite3', DB], input=sql.encode('utf-8'), capture_output=True, timeout=10)

conn.close()
print('FTS re-synced: ' + str(len(fix_fts)) + ' records')