#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""溧阳市人民政府 JSONL 导入脚本 — 把已抓取的 liyang JSONL 导入主库 gov_raw
用法: python3 import_liyang_to_db.py
来源: /tmp/liyang_xxgk_full.jsonl (284条, 信息公开) + /tmp/liyang_tzgg_full.jsonl (220条, 通知公告)
字段: title, publish_date, content, page_url, source_url, site_name, summary, date_rank, author, content_source, attachments
"""
import json, sqlite3, os, re, sys

DB_PATH = "/root/search.db"
FILES = [
    "/tmp/liyang_xxgk_full.jsonl",
    "/tmp/liyang_tzgg_full.jsonl",
]

def main():
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA busy_timeout=60000")
    cur = conn.cursor()
    total_added = total_skipped = 0
    for data_file in FILES:
        if not os.path.exists(data_file):
            print(f"文件不存在: {data_file}")
            continue
        added = skipped = 0
        site_name = "liyang_xxgk" if "xxgk" in data_file else "liyang_tzgg"
        with open(data_file, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                try:
                    d = json.loads(line)
                except Exception:
                    skipped += 1
                    continue
                page_url = d.get("page_url") or d.get("source_url") or d.get("url") or ""
                title = d.get("title") or ""
                content = d.get("content") or ""
                publish_date = d.get("publish_date") or d.get("date") or ""
                summary = d.get("summary") or content[:500]
                att_json = json.dumps(d.get("attachments", []), ensure_ascii=False)
                try:
                    cur.execute(
                        """INSERT OR IGNORE INTO gov_raw
                           (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments)
                           VALUES (?,?,?,?,?,?,?,?,?,?)""",
                        (site_name, page_url, page_url, title, publish_date, content,
                         summary, "通知公告" if "tzgg" in data_file else "政府信息公开", "import_liyang_to_db", att_json)
                    )
                    if cur.rowcount > 0:
                        added += 1
                    else:
                        skipped += 1
                except sqlite3.Error as e:
                    print("  DB err:", e)
                    skipped += 1
                conn.commit()
        print(f"=== {site_name}: 新增 {added} / 跳过 {skipped} ===")
        total_added += added
        total_skipped += skipped
    conn.close()
    print(f"总计: 新增 {total_added} / 跳过 {total_skipped}")

if __name__ == "__main__":
    main()