#!/usr/bin/env python3
"""托克托县人民政府 — 部门文件 导入脚本（服务器端）
读取本地 Mac 抓取的 JSONL → 入库 search.db
用法: python3 import_tuoxian_bmwj.py /path/to/tuoxian_bmwj.jsonl
"""
import json, sys, sqlite3, os

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")


def main():
    if len(sys.argv) < 2:
        print("Usage: import_tuoxian_bmwj.py <jsonl>")
        sys.exit(1)
    path = sys.argv[1]
    conn = sqlite3.connect(SEARCH_DB)
    cur = conn.cursor()
    new = skip = 0
    with open(path, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            r = json.loads(line)
            cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (r["url"],))
            if cur.fetchone():
                skip += 1
                continue
            try:
                cur.execute(
                    """INSERT INTO gov_raw
                       (source_url, page_url, title, publish_date, site_name, content, group_name, industry, script_name)
                       VALUES (?,?,?,?,?,?,?,?,?)""",
                    (r["source_url"], r["url"], r["title"], r.get("pub_date", ""),
                     r["site_name"], r["content"], r.get("group_name", ""),
                     r.get("industry", ""), "crawl_tuoxian_bmwj.py"),
                )
                conn.commit()
                new += 1
            except Exception as e:
                conn.rollback()
                skip += 1
                print("ERR:", e, r.get("title", "")[:30])
    # FTS 增量同步
    cur.execute(
        """INSERT INTO gov_search(rowid,title,site_name,summary)
           SELECT r.id, r.title, r.site_name, substr(r.content,1,500)
           FROM gov_raw r
           WHERE r.id NOT IN (SELECT rowid FROM gov_search)
             AND r.site_name = '托克托县人民政府-部门文件'"""
    )
    conn.commit()
    conn.close()
    print(f"[import_tuoxian_bmwj] New: {new}, Skipped: {skip}")


if __name__ == "__main__":
    main()