#!/usr/bin/env python3
"""Server-side JSONL importer for search.db"""
import json, sqlite3, sys, os, re, html

def strip_html(text):
    """Convert HTML to clean plain text."""
    if not text:
        return ""
    text = html.unescape(text)
    text = re.sub(r'</?(?:p|div|h[1-6]|li|tr|blockquote|section|article|table|br\s*/?)[^>]*>', chr(10), text, flags=re.IGNORECASE)
    text = re.sub(r'<[^>]+>', '', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'\n{3,}', chr(10)*2, text)
    return text.strip()

if len(sys.argv) < 2:
    print("Usage: import_jsonl.py <jsonl_path> [label]")
    sys.exit(1)

jsonl_path = sys.argv[1]
label = sys.argv[2] if len(sys.argv) > 2 else "data"

db_path = os.getenv("SEARCH_DB", "/root/search.db")
db = sqlite3.connect(db_path)
added = 0
errors = 0

with open(jsonl_path) as f:
    for line in f:
        line = line.strip()
        if not line:
            continue
        try:
            item = json.loads(line)
            title = (item.get("title") or "")[:500]
            page_url = (item.get("page_url") or "")[:1000]
            content = strip_html(item.get("content") or "")
            publish_date = (item.get("publish_date") or "")[:20]
            site_name = (item.get("site_name") or "unknown")[:100]

            # Attachments as JSON string
            att = item.get("attachments") or []
            attachments_str = json.dumps(att, ensure_ascii=False) if att else ""

            # Delete old FTS entry if this page_url already exists (rowid may change after REPLACE)
            db.execute("DELETE FROM gov_search WHERE rowid IN (SELECT rowid FROM gov_raw WHERE page_url = ?)", (page_url,))

            # Insert or replace raw data
            db.execute(
                "INSERT OR REPLACE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?)",
                (title, page_url, content, publish_date, site_name, page_url, attachments_str)
            )

            # Get the new rowid and insert/update FTS incrementally
            new_rowid = db.execute("SELECT last_insert_rowid()").fetchone()[0]
            summary = content[:500] if content else title[:500]
            db.execute(
                "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
                (new_rowid, title, site_name, summary)
            )

            added += 1
        except Exception as e:
            errors += 1

db.commit()

total = db.execute("SELECT COUNT(*) FROM gov_raw").fetchone()[0]
db.close()

os.remove(jsonl_path)
print(f"\u2705 {label}: +{added} err{errors} total={total}")