#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
通用 JSONL 导入脚本 (服务器端)
用法: python3 crawl_xxx.py [--file /path/data.jsonl]
配置: SITE_NAME/SCRIPT/CATEGORY/DATA_FILE 按站修改
"""
import json
import sqlite3
import os
import argparse

DB_PATH = "/root/search.db"

SITE_CONFIGS = {
    "crawl_smx_sthj.py": {
        "site_name": "三门峡市生态环境局-法定主动公开",
        "category": "法定主动公开",
        "data_file": "/root/gov_crawler/data/smx_sthj.jsonl",
    },
    "crawl_yima_jczwgk.py": {
        "site_name": "义马市-基层政务公开行政许可",
        "category": "行政许可",
        "data_file": "/root/gov_crawler/data/yima_jczwgk.jsonl",
    },
    "crawl_fengkai_hpgs.py": {
        "site_name": "封开县-建设项目环评",
        "category": "建设项目环评",
        "data_file": "/root/gov_crawler/data/fengkai_hpgs.jsonl",
    },
}

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--file", default=None)
    args = ap.parse_args()

    script = os.path.basename(__file__)
    cfg = SITE_CONFIGS.get(script)
    if not cfg:
        print(f"未找到 {script} 的配置")
        return
    data_file = args.file or cfg["data_file"]
    if not os.path.exists(data_file):
        print(f"数据文件不存在: {data_file}")
        return

    conn = sqlite3.connect(DB_PATH, timeout=290)
    conn.execute("PRAGMA busy_timeout=290000")
    cur = conn.cursor()
    added = skipped = 0
    with open(data_file, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                d = json.loads(line)
            except Exception:
                skipped += 1
                continue
            att_json = json.dumps(d.get("attachments", []), ensure_ascii=False)
            try:
                cur.execute(
                    """INSERT OR IGNORE INTO gov_raw
                       (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments)
                       VALUES (?,?,?,?,?,?,?,?,?,?)""",
                    (cfg["site_name"], d["url"], d["url"], d["title"], d["date"], d["content_html"],
                     (d["content_html"] or "")[:500], cfg["category"], script, att_json)
                )
                if cur.rowcount > 0:
                    added += 1
                else:
                    skipped += 1
            except sqlite3.Error as e:
                print("  DB err:", e)
                skipped += 1
            conn.commit()
    conn.close()
    print(f"=== {cfg['site_name']}: 新增 {added} / 跳过 {skipped} ===")

if __name__ == "__main__":
    main()