#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""bhna 存量省略号标题重刷 v2：优先 page_url，其次 source_url
修复 32 条早期导入 page_url 为空的省略号标题
"""
import sys, re, time, sqlite3, urllib.request

SEARCH_DB = "/root/search.db"
SITE_NAME = "港城产业园区-信息公示"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch_title(url):
    req = urllib.request.Request(url, headers=HEADERS)
    resp = urllib.request.urlopen(req, timeout=30)
    html = resp.read().decode('utf-8', errors='replace')
    title = ""
    h1 = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
    if h1:
        title = re.sub(r'<[^>]+>', '', h1.group(1)).strip()
    if not title:
        tt = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if tt:
            title = re.sub(r'<[^>]+>', '', tt.group(1)).strip()
            title = re.sub(r'\s*[-_—|]\s*[^-_—|]*$', '', title).strip()
    return title

def clean(t):
    if not t:
        return t
    return re.sub(r'[\.]{3,}|…\s*$', '', t).strip()

conn = sqlite3.connect(SEARCH_DB, timeout=60)
conn.execute("PRAGMA busy_timeout=30000")

# 省略号且有 URL（page_url 或 source_url）
cur = conn.execute(
    """SELECT id, COALESCE(NULLIF(page_url,''), NULLIF(source_url,'')), title
       FROM gov_raw WHERE site_name=? AND (title LIKE '%...%' OR title LIKE '%…%')""",
    (SITE_NAME,))
rows = cur.fetchall()
print(f"省略号总记录: {len(rows)}")

ok = skip = fail = 0
for idx, (rid, url, old_title) in enumerate(rows, 1):
    if not url:
        print(f"  [{idx}/{len(rows)}] SKIP 无任何URL: {old_title[:50]}")
        skip += 1
        continue
    try:
        title = clean(fetch_title(url))
    except Exception as e:
        print(f"  [{idx}/{len(rows)}] FAIL {url[-50:]}: {e}")
        fail += 1
        time.sleep(0.3)
        continue
    if not title or title == old_title:
        print(f"  [{idx}/{len(rows)}] SKIP 无h1或无变化: {old_title[:50]}")
        skip += 1
        time.sleep(0.3)
        continue
    conn.execute("UPDATE gov_raw SET title=? WHERE id=?", (title, rid))
    conn.commit()
    ok += 1
    print(f"  [{idx}/{len(rows)}] OK {old_title[:35]}... → {title[:55]}")
    time.sleep(0.3)

conn.close()
print(f"\nTOTAL: ok={ok} skip={skip} fail={fail}")