#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
njna 南京江北新区 导入脚本 (服务器端)
用法: python3 crawl_njna_xxgk.py --file /path/njna_full.jsonl
"""
import json
import sqlite3
import os
import argparse
import re

DB_PATH = "/root/search.db"

SITE_NAME = "南京江北新区管委会-政府信息公开"
CATEGORY = "政府信息公开"
SCRIPT = "crawl_njna_xxgk.py"

def strip_html(text):
    if not text:
        return ""
    t = re.sub(r"<[^>]+>", " ", text)
    t = t.replace("&nbsp;", " ").replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">")
    t = re.sub(r"\s+", " ", t).strip()
    return t

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--file", required=True)
    args = ap.parse_args()

    conn = sqlite3.connect(DB_PATH, timeout=290)
    conn.execute("PRAGMA busy_timeout=290000")
    cur = conn.cursor()
    added = skipped = 0
    with open(args.file, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            try:
                d = json.loads(line)
            except Exception:
                skipped += 1
                continue
            url = d.get("url", "")
            title = (d.get("title") or "")[:500]
            date = (d.get("date") or "")[:20]
            content = strip_html(d.get("content") or "")[:20000]
            source = (d.get("source") or "")[:100]
            att = d.get("attachments") or []
            att_json = json.dumps(att, ensure_ascii=False) if att else ""
            try:
                cur.execute(
                    """INSERT OR IGNORE INTO gov_raw
                       (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments)
                       VALUES (?,?,?,?,?,?,?,?,?,?)""",
                    (SITE_NAME, url, url, title, date, content,
                     (content or title)[:500], CATEGORY, SCRIPT, att_json)
                )
                if cur.rowcount > 0:
                    added += 1
                else:
                    # 已存在: 若有附件则更新
                    if att:
                        cur.execute(
                            "UPDATE gov_raw SET attachments=? WHERE page_url=? AND attachments=''",
                            (att_json, url))
                    skipped += 1
            except sqlite3.Error as e:
                print("  DB err:", e)
                skipped += 1
            conn.commit()
    conn.close()
    print(f"=== {SITE_NAME}: 新增 {added} / 跳过 {skipped} ===")

if __name__ == "__main__":
    main()