"""Re-crawl anyi detail pages to restore table HTML"""
import requests, sqlite3, re, time, warnings
from bs4 import BeautifulSoup
warnings.filterwarnings('ignore')

DB_PATH = "/root/search.db"
SITE_NAME = "安义县-环评公示"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Referer": "https://anyi.nc.gov.cn/ayxzf/xsthjjgggs2021/just_list.shtml",
}

def extract_content(html):
    """Extract content preserving <p> text and <table> HTML"""
    soup = BeautifulSoup(html, 'html.parser')
    
    # Try UCAPCONTENT first
    ucap = soup.find('UCAPCONTENT')
    if ucap:
        return clean_ucap_text(ucap)
    
    # Fallback to article-content div
    cont = soup.select_one('div.article-content-body, div.article-content')
    if cont:
        ucap2 = cont.find('UCAPCONTENT')
        if ucap2:
            return clean_ucap_text(ucap2)
        return clean_ucap_text(cont)
    
    return soup.get_text('\n\n', strip=True)

def clean_ucap_text(element):
    """Clean content preserving <p> text and <table> HTML"""
    html = str(element)
    html = re.sub(r'</?(?:span|o:p|font|b|strong|em|i|u|s|strike|sub|sup)[^>]*>', '', html)
    clean = BeautifulSoup(html, 'html.parser')
    
    parts = []
    for p in clean.find_all('p'):
        txt = p.get_text(strip=True)
        if txt:
            parts.append(txt)
    for t in clean.find_all('table'):
        parts.append(str(t))
    
    if not parts:
        txt = clean.get_text('\n\n', strip=True)
        return txt if txt else ""
    
    return '\n\n'.join(parts)


# Get all URLs for this site
db = sqlite3.connect(DB_PATH)
urls = [r[0] for r in db.execute("SELECT page_url FROM gov_raw WHERE site_name = ?", (SITE_NAME,)).fetchall()]
print(f"Found {len(urls)} existing records to update")

updated = 0
skipped = 0
for i, page_url in enumerate(urls):
    try:
        resp = requests.get(page_url, headers=HEADERS, timeout=30, verify=False)
        resp.encoding = 'utf-8'
    except Exception as e:
        print(f"  [{i+1}/{len(urls)}] FETCH ERR: {e}")
        continue
    
    if resp.status_code != 200 or 'error_403' in resp.text:
        print(f"  [{i+1}/{len(urls)}] BLOCKED: {page_url.split('/')[-1][:30]}")
        skipped += 1
        continue
    
    new_content = extract_content(resp.text)
    if not new_content:
        skipped += 1
        continue
    
    summary = new_content[:500]
    
    try:
        db.execute("UPDATE gov_raw SET content = ?, summary = ? WHERE page_url = ?",
                   (new_content, summary, page_url))
        db.commit()
        updated += 1
        if updated % 100 == 0:
            print(f"  [{i+1}/{len(urls)}] Updated {updated} so far...")
    except Exception as e:
        print(f"  DB ERR: {e}")
    
    # Small delay to avoid overwhelming the server
    time.sleep(0.3)

db.close()
print(f"\n✅ Done. Updated: {updated}, Skipped/Blocked: {skipped}")