"""
One-time refetch: try to extract content for existing empty-content records
from 巨野县人民政府-法定主动公开内容 (crawl_juye.py site).
"""
import os, sys, re, time, sqlite3, requests
from datetime import datetime

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}
SITE_NAME = '巨野县人民政府-法定主动公开内容'


def extract_memo_content(html):
    m = re.search(r'var\s+memo\s*=\s*"((?:[^"\\]|\\.)*)"', html, re.DOTALL)
    if m:
        raw = m.group(1)
    else:
        m = re.search(r"var\s+memo\s*=\s*'((?:[^'\\]|\\.)*)'", html, re.DOTALL)
        if m:
            raw = m.group(1)
        else:
            return ''
    raw = raw.replace(r'\/', '/')
    try:
        raw = raw.encode().decode('unicode_escape')
    except:
        pass
    return raw


def main():
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    
    # Find all empty-content records for this site
    rows = c.execute(
        "SELECT id, page_url, title FROM gov_raw WHERE site_name=? AND (content IS NULL OR content='') ORDER BY id",
        (SITE_NAME,)
    ).fetchall()
    
    print(f'Found {len(rows)} empty-content records for {SITE_NAME}')
    
    fixed = 0
    failed = 0
    
    for idx, (rid, page_url, title) in enumerate(rows):
        print(f'  [{idx+1}/{len(rows)}] id={rid}...', end=' ', flush=True)
        try:
            r = requests.get(page_url, headers=HEADERS, timeout=15)
            r.encoding = 'utf-8'
            if r.status_code != 200:
                print(f'HTTP {r.status_code}')
                failed += 1
                continue
            
            content = extract_memo_content(r.text)
            if not content:
                print('no memo content')
                failed += 1
                continue
            
            c.execute("UPDATE gov_raw SET content=? WHERE id=?", (content, rid))
            print(f'fixed ({len(content)} chars)')
            fixed += 1
            time.sleep(0.3)
            
            if fixed % 20 == 0:
                conn.commit()
        except Exception as e:
            print(f'ERR: {e}')
            failed += 1
            time.sleep(0.5)
    
    conn.commit()
    conn.close()
    print(f'\nDone! Fixed: {fixed}, Failed: {failed}')


if __name__ == '__main__':
    main()