#!/usr/bin/env python3
"""Update existing yjq_xzsp records with proper content from yjq.gov.cn pages."""
import sys, os, json, re, time, requests, sqlite3
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = "https://www.yjq.gov.cn"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

conn = sqlite3.connect(DB_PATH)
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=30000")
c = conn.cursor()

# Find records with short content (<2000 chars) from this site
rows = c.execute(
    "SELECT rowid, page_url, length(content), title FROM gov_raw "
    "WHERE site_name LIKE ? AND length(content) < 2000 ORDER BY rowid",
    ("%弋江%行政审批%",)
).fetchall()
print(f"Found {len(rows)} records with short content to fix")

fixed = 0
err = 0
for rowid, page_url, old_len, title in rows:
    try:
        r = requests.get(page_url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
        content_div = soup.find("div", class_=lambda c: c and "j-fontContent" in c.split())
        if not content_div:
            print(f"  SKIP {rowid}: {title[:30]} - no j-fontContent")
            continue
        ch = str(content_div)
        cs = BeautifulSoup(ch, "html.parser")
        parts = []
        for el in cs.find_all(["table", "p"]):
            if el.name == "table":
                parts.append(str(el))
            elif el.name == "p" and not el.find_parent("table"):
                t = el.get_text(separator="", strip=True)
                if t:
                    parts.append(t)
        new_content = "\n\n".join(parts)
        if len(new_content) > old_len:
            c.execute("UPDATE gov_raw SET content=? WHERE rowid=?", (new_content, rowid))
            fixed += 1
            print(f"  OK {rowid}: {old_len} -> {len(new_content)} ({title[:30]})")
        else:
            print(f"  SAME {rowid}: {old_len} chars, no change ({title[:30]})")
    except Exception as e:
        print(f"  ERR {rowid}: {e}")
        err += 1
    time.sleep(0.5)
    if fixed % 5 == 0 and fixed > 0:
        conn.commit()

conn.commit()
conn.close()
print(f"Done: {fixed} fixed, {err} errors")