#!/usr/bin/env python3
"""Server-side JSONL importer for search.db"""
import json, sqlite3, sys, os, re, html

def strip_html(text):
    """Convert HTML to clean plain text, preserving link URL as 'text (url)'."""
    if not text:
        return ""
    text = html.unescape(text)
    # 先把 <a href="URL">文本</a> 转成 文本 (URL), 保 URL 可达
    text = re.sub(
        r'<a\s[^>]*href="([^"]+)"[^>]*>(.*?)</a>',
        lambda m: (re.sub(r'<[^>]+>', '', m.group(2)).strip() or '链接') + ' (' + m.group(1) + ')',
        text, flags=re.IGNORECASE | re.DOTALL
    )
    text = re.sub(
        r"<a\s[^>]*href='([^']+)'[^>]*>(.*?)</a>",
        lambda m: (re.sub(r'<[^>]+>', '', m.group(2)).strip() or '链接') + ' (' + m.group(1) + ')',
        text, flags=re.IGNORECASE | re.DOTALL
    )
    text = re.sub(r'</?(?:p|div|h[1-6]|li|tr|blockquote|section|article|table|br\s*/?)[^>]*>', chr(10), text, flags=re.IGNORECASE)
    text = re.sub(r'<[^>]+>', '', text)
    text = re.sub(r'[ \t]+', ' ', text)
    text = re.sub(r'\n{3,}', chr(10)*2, text)
    # 清理纯空白行
    text = '\n'.join(l.rstrip() for l in text.split('\n') if l.strip())
    return text.strip()

if len(sys.argv) < 2:
    print("Usage: import_jsonl.py <jsonl_path> [label]")
    sys.exit(1)

jsonl_path = sys.argv[1]
label = sys.argv[2] if len(sys.argv) > 2 else "data"

db_path = os.getenv("SEARCH_DB", "/root/search.db")
db = sqlite3.connect(db_path, timeout=120)
added = 0
errors = 0

with open(jsonl_path) as f:
    for line in f:
        line = line.strip()
        if not line:
            continue
        try:
            item = json.loads(line)
            title = (item.get("title") or "")[:500]
            page_url = (item.get("page_url") or "")[:1000]
            content = strip_html(item.get("content") or "")
            publish_date = (item.get("publish_date") or "")[:20]
            site_name = (item.get("site_name") or "unknown")[:100]

            # Attachments as JSON string
            att = item.get("attachments") or []
            attachments_str = json.dumps(att, ensure_ascii=False) if att else ""

            # Delete old FTS entry if this page_url already exists (rowid may change after REPLACE)
            db.execute("DELETE FROM gov_search WHERE rowid IN (SELECT rowid FROM gov_raw WHERE page_url = ?)", (page_url,))
            db.execute(
                "INSERT OR REPLACE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?)",
                (title, page_url, content, publish_date, site_name, page_url, attachments_str)
            )
            new_rowid = db.execute("SELECT rowid FROM gov_raw WHERE page_url = ?", (page_url,)).fetchone()
            if not new_rowid:
                raise RuntimeError("rowid lookup failed after insert")
            rowid = new_rowid[0]
            summary = content[:500] if content else title[:500]
            db.execute(
                "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
                (rowid, title, site_name, summary)
            )
            added += 1
        except Exception as e:
            errors += 1
            print(f"  ERR {line[:80]}: {e}", file=sys.stderr)

db.commit()
total = db.execute("SELECT COUNT(*) FROM gov_raw").fetchone()[0]
db.close()
try:
    os.remove(jsonl_path)
except OSError:
    pass
print(f"✅ {label}: +{added} err{errors} total={total}")