#!/usr/bin/env python3
"""
Fix existing harvin records - re-fetch detail pages with fixed extract_detail
"""
import sys
sys.path.insert(0, '/root/gov_crawler')
from crawl_harvin import fetch, extract_detail
import sqlite3
import json
import time

DB_PATH = '/root/gov_crawler/search.db'

def fix_records():
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    
    # Get all harvin records
    c.execute("SELECT id, url, substr(content, 1, 200) FROM gov_raw WHERE url LIKE 'https://www.harvin.cn/news/%' ORDER BY id")
    rows = c.fetchall()
    print(f"Total harvin records: {len(rows)}")
    
    bad_count = 0
    fixed_count = 0
    for row_id, url, old_preview in rows:
        # Re-fetch detail
        try:
            html = fetch(url)
            result = extract_detail(html, url)
            new_content = result.get('content', '')
        except Exception as e:
            print(f"  SKIP [id={row_id}] {url}: {e}")
            continue
        
        old_content_row = c.execute("SELECT content FROM gov_raw WHERE id = ?", (row_id,)).fetchone()
        if not old_content_row:
            continue
        
        old_content = old_content_row[0] or ''
        
        # Check if content changed
        has_html_table = '<table' in old_content
        has_bad_metadata = '分类：' in old_content[:500] and '新闻中心' in old_content[:500]
        
        if has_html_table or has_bad_metadata:
            bad_count += 1
        
        if old_content != new_content and new_content.strip():
            title = result.get('title', '')
            date = result.get('date', '')
            c.execute(
                "UPDATE gov_raw SET content = ?, title = ?, publish_date = ? WHERE id = ?",
                (new_content, title, date, row_id)
            )
            fixed_count += 1
            if fixed_count <= 5:
                print(f"  FIXED [id={row_id}] {title[:50]}: old={len(old_content)}chars new={len(new_content)}chars")
                if has_html_table:
                    print(f"    Was: raw HTML table")
                if has_bad_metadata:
                    print(f"    Was: metadata in content")
        elif not new_content.strip():
            print(f"  WARN [id={row_id}] {url}: new content is empty!")
        
        time.sleep(0.3)
    
    conn.commit()
    
    print(f"\nSummary: {bad_count} bad records found, {fixed_count} fixed out of {len(rows)} total")
    
    # Rebuild FTS - gov_search uses rowid_col as FK
    print("\nRebuilding FTS indexes...")
    try:
        c.execute("""
            UPDATE gov_search SET title = (SELECT title FROM gov_raw WHERE id = gov_search.rowid_col)
            WHERE rowid_col IN (SELECT id FROM gov_raw WHERE url LIKE 'https://www.harvin.cn/news/%')
        """)
        print(f"  Updated {c.rowcount} rows in gov_search (title)")
    except Exception as e:
        print(f"  gov_search title update error: {e}")
    
    try:
        c.execute("""
            UPDATE gov_search SET summary = (SELECT substr(content, 1, 200) FROM gov_raw WHERE id = gov_search.rowid_col)
            WHERE rowid_col IN (SELECT id FROM gov_raw WHERE url LIKE 'https://www.harvin.cn/news/%')
        """)
        print(f"  Updated {c.rowcount} rows in gov_search (summary)")
    except Exception as e:
        print(f"  gov_search summary update error: {e}")
    
    conn.commit()
    conn.close()

if __name__ == '__main__':
    fix_records()
    print("\nDone.")