#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""通用 JSONL → gov_raw 导入（INSERT OR IGNORE, page_url 唯一索引去重）
用法: python3 import_jsonl_to_db.py <jsonl> <site_name> <category> [--col-map 字段映射]
字段映射默认按 liyang 格式; wenshang 格式用 --fmt wenshang
"""
import json, sqlite3, os, sys, argparse

DB_PATH = "/root/search.db"

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("jsonl")
    ap.add_argument("site_name")
    ap.add_argument("category")
    ap.add_argument("--fmt", default="liyang", choices=["liyang", "wenshang"])
    args = ap.parse_args()

    if not os.path.exists(args.jsonl):
        print(f"文件不存在: {args.jsonl}")
        sys.exit(1)

    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=60000")
    cur = conn.cursor()
    added = skipped = 0
    with open(args.jsonl, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                d = json.loads(line)
            except Exception:
                skipped += 1
                continue
            if args.fmt == "wenshang":
                page_url = d.get("url") or ""
                title = d.get("title") or ""
                content = d.get("body") or ""
                publish_date = d.get("date") or ""
            else:
                page_url = d.get("page_url") or d.get("source_url") or d.get("url") or ""
                title = d.get("title") or ""
                content = d.get("content") or ""
                publish_date = d.get("publish_date") or d.get("date") or ""
            if not page_url:
                skipped += 1
                continue
            summary = (d.get("summary") or re_sub(content))[:500]
            att_json = json.dumps(d.get("attachments", []), ensure_ascii=False)
            try:
                cur.execute(
                    """INSERT OR IGNORE INTO gov_raw
                       (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments)
                       VALUES (?,?,?,?,?,?,?,?,?,?)""",
                    (args.site_name, page_url, page_url, title, publish_date, content,
                     summary, args.category, "import_jsonl_to_db", att_json)
                )
                if cur.rowcount > 0:
                    added += 1
                else:
                    skipped += 1
            except sqlite3.Error as e:
                print("  DB err:", e)
                skipped += 1
            conn.commit()
    conn.close()
    print(f"=== {args.site_name}: 新增 {added} / 跳过 {skipped} ===")

def re_sub(content):
    import re
    return re.sub(r"<[^>]+>", " ", content or "").strip()

if __name__ == "__main__":
    main()