#!/usr/bin/env python3
"""import_jxyy.py - 导入 jxyy JSONL 到 gov_raw (保留 HTML 正文 + FTS 同步)
字段: title, page_url, content(HTML), publish_date, site_name, summary, date_rank, has_table, attachments, script_name
用法: python3 import_jxyy.py <jsonl_path>
"""
import json, sqlite3, sys, re, time, os
from datetime import datetime

if len(sys.argv) < 2:
    print("Usage: import_jxyy.py <jsonl_path>")
    sys.exit(1)

jsonl_path = sys.argv[1]
db_path = os.getenv("SEARCH_DB", "/root/search.db")
db = sqlite3.connect(db_path)
db.execute("PRAGMA journal_mode=WAL")
db.execute("PRAGMA busy_timeout=300000")
db.execute("PRAGMA synchronous=NORMAL")

db.execute("""CREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
    title, site_name, summary, tokenize=trigram
)""")
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_gov_raw_fts_ins AFTER INSERT ON gov_raw BEGIN
  INSERT INTO gov_search(rowid, title, site_name, summary)
  VALUES (new.id, new.title, new.site_name, new.summary);
END""")
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_gov_raw_fts_del AFTER DELETE ON gov_raw BEGIN
  DELETE FROM gov_search WHERE rowid = old.id;
END""")
db.execute("""CREATE TRIGGER IF NOT EXISTS trg_gov_raw_fts_upd AFTER UPDATE ON gov_raw BEGIN
  DELETE FROM gov_search WHERE rowid = old.id;
  INSERT INTO gov_search(rowid, title, site_name, summary)
  VALUES (new.id, new.title, new.site_name, new.summary);
END""")
db.commit()

added = 0
errors = 0
with open(jsonl_path, encoding='utf-8') as f:
    for line in f:
        line = line.strip()
        if not line:
            continue
        try:
            item = json.loads(line)
            title = (item.get("title") or "")[:500]
            page_url = (item.get("page_url") or item.get("url") or "")[:1000]
            content = (item.get("content") or "").strip()
            publish_date = (item.get("publish_date") or "")[:20]
            site_name = (item.get("site_name") or "jxyy_tzgg")[:100]
            att = item.get("attachments") or []
            attachments_str = json.dumps(att, ensure_ascii=False) if att else ""
            if not page_url or not title:
                errors += 1
                print(f"  [ERR] missing title/url: {line[:100]}")
                continue
            # summary: 去标签纯文本
            plain = re.sub(r'<[^>]+>', ' ', content)
            plain = re.sub(r'\s+', ' ', plain).strip()
            summary = plain[:200]
            # date_rank
            dr = 0
            dm = re.search(r'(\d{4})-(\d{2})-(\d{2})', publish_date)
            if dm:
                try:
                    dr = int(datetime(int(dm.group(1)), int(dm.group(2)), int(dm.group(3))).timestamp())
                except Exception:
                    dr = 0
            has_table = 1 if '<table' in content else 0
            db.execute("DELETE FROM gov_raw WHERE page_url = ?", (page_url,))
            db.execute(
                "INSERT INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments, script_name, summary, date_rank, has_table) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?, 'crawl_jxyy_tzgg.py', ?, ?, ?)",
                (title, page_url, content, publish_date, site_name, page_url, attachments_str, summary, dr, has_table)
            )
            added += 1
        except Exception as e:
            errors += 1
            print(f"  [ERR] {e}: {line[:100]}")
db.commit()
db.close()
print(f"✅ 导入完成: +{added} err{errors}")