#!/usr/bin/env python3
"""Backfill truncated titles for 廉江开发区"""
import os, re, requests, sqlite3
from bs4 import BeautifulSoup

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "廉江开发区管委会-公示公告"
HEADERS = {"User-Agent": "Mozilla/5.0"}
import urllib3
urllib3.disable_warnings()

conn = sqlite3.connect(DB_PATH)
cur = conn.cursor()

# Get all records for this site with truncated titles (containing ...)
cur.execute("SELECT id, page_url, title FROM gov_raw WHERE site_name=? AND (title LIKE '%…%' OR title LIKE '%...%')", (SITE_NAME,))
rows = cur.fetchall()
print(f"Records with truncated titles: {len(rows)}")

fixed = 0
for i, (rid, url, old_title) in enumerate(rows, 1):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = 'utf-8'
        soup = BeautifulSoup(r.text, 'html.parser')
        full_title = ''
        if soup.title:
            t = soup.title.get_text(strip=True)
            t = re.sub(r'\s*[-–—]\s*廉江市人民政府门户网站\s*$', '', t).strip()
            full_title = t
        if full_title and full_title != old_title:
            cur.execute("UPDATE gov_raw SET title=? WHERE id=?", (full_title, rid))
            conn.commit()
            fixed += 1
            if fixed <= 3 or fixed % 50 == 0:
                print(f'  [{fixed}] id={rid}: {old_title[:40]}... → {full_title[:40]}...')
    except Exception as e:
        print(f'  Error id={rid}: {e}')
    
    if i % 50 == 0:
        print(f'  Progress: {i}/{len(rows)}')

print(f'\nFixed: {fixed}/{len(rows)} titles')
conn.close()