import json, sqlite3, sys, os, re, html

jsonl_path = sys.argv[1]

def strip_html(text):
    if not text:
        return ""
    text = html.unescape(text)
    text = re.sub(r'</?(?:p|div|h[1-6]|li|tr|blockquote|section|article|table|br\s*/?)[^>]*>', chr(10), text, flags=re.IGNORECASE)
    text = re.sub(r'<[^>]+>', '', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'\n{3,}', chr(10)*2, text)
    return text.strip()

db_path = os.getenv("SEARCH_DB", "/root/search.db")
db = sqlite3.connect(db_path, timeout=60)

cursor = db.cursor()
added = 0
errors = 0

with open(jsonl_path) as f:
    for line in f:
        line = line.strip()
        if not line:
            continue
        try:
            item = json.loads(line)
            title = (item.get("title") or "")[:500]
            page_url = (item.get("page_url") or "")[:1000]
            content = strip_html(item.get("content") or "")
            publish_date = (item.get("publish_date") or "")[:20]
            site_name = (item.get("site_name") or "unknown")[:100]
            att = item.get("attachments") or []
            attachments_str = json.dumps(att, ensure_ascii=False) if att else ""
            cursor.execute(
                "INSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?)",
                (title, page_url, content, publish_date, site_name, page_url, attachments_str)
            )
            if cursor.rowcount > 0:
                added += 1
        except Exception as e:
            errors += 1

db.commit()

# Sync FTS for just the new records (incremental, no full rebuild)
cursor.execute("""
    INSERT INTO gov_search(rowid, title, site_name, summary)
    SELECT r.rowid, r.title, r.site_name, substr(r.content, 1, 500)
    FROM gov_raw r
    WHERE NOT EXISTS (SELECT 1 FROM gov_search s WHERE s.rowid = r.rowid)
      AND r.content IS NOT NULL AND r.content != ''
""")
db.commit()

total = cursor.execute("SELECT COUNT(*) FROM gov_raw").fetchone()[0]
fts_total = cursor.execute("SELECT COUNT(*) FROM gov_search").fetchone()[0]
db.close()

os.remove(jsonl_path)
print(f"✅ Imported: +{added} err{errors} raw_total={total} fts_total={fts_total}")