#!/usr/bin/env python3
"""Re-import all 3 new sites into the REAL search DB (/root/search.db -> /mnt/data/search.db)"""
import json, sqlite3, os, re, sys

DB = '/root/search.db'
sites = [
    ('harvin', '/root/gov_crawler/output/harvin.jsonl', 'https://www.harvin.cn/news/%', '祥云股份', '企业'),
    ('hbnq', '/root/gov_crawler/output/hbnq_gsgg.jsonl', 'https://www.hbnq.gov.cn/content/11049/%', '内丘县人民政府', '内丘县'),
    ('hbqj', '/root/gov_crawler/output/hbqj_eia.jsonl', 'https://www.hbqj.gov.cn/xxgk/xxgkml/szfxxgkml/gysyjs/hjbh/jsxmhjyxpjgs/%', '潜江市人民政府', '潜江市'),
]

conn = sqlite3.connect(DB)
c = conn.cursor()

for tag, path, url_pat, site_prefix, group in sites:
    if not os.path.exists(path):
        print("SKIP: %s not found" % path)
        continue
    records = [json.loads(l) for l in open(path)]
    print("\n=== %s (%d records) ===" % (tag, len(records)))
    
    c.execute("DELETE FROM gov_raw WHERE page_url LIKE ?", (url_pat,))
    print("  Deleted %d existing" % c.rowcount)
    
    ok = 0
    for i, r in enumerate(records):
        title = r['title']
        content = (r.get('content') or '').strip()
        page_url = r['link']
        pub_date = r.get('date', '') or ''
        site_name = r['site_name']
        attach = json.dumps(r.get('attachments', []), ensure_ascii=False)
        summary = re.sub(r'<[^>]+>', '', content)[:200].strip() if content else title[:200]
        
        try:
            c.execute("""
                INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date,
                    summary, status, category, content, tags, industry, attachments)
                VALUES (?,?,?,?,?,?, 'published',?,?,'','other',?)
            """, (site_name, page_url, page_url, title, pub_date, summary, group, content, attach))
            ok += 1
        except sqlite3.IntegrityError:
            pass
        
        if ok % 50 == 0 and ok > 0:
            conn.commit()
    
    conn.commit()
    print("  Imported: %d" % ok)

conn.close()
print("\n=== DONE ===")