#!/usr/bin/env python3
"""Import sxx.gov.cn data into search.db."""
import json
import sqlite3
import glob
import os

os.chdir('/root/gov_crawler')
files = sorted(glob.glob('/root/gov_crawler/output/sxx_tzgg_*.jsonl'))
if not files:
    print('No output file found')
    exit(1)

jsonl_file = files[-1]
print(f'Importing: {jsonl_file}')

with open(jsonl_file) as f:
    records = [json.loads(line) for line in f]

print(f'Records: {len(records)}')

db_path = '/mnt/data/search.db'
conn = sqlite3.connect(db_path)
c = conn.cursor()

SITE_NAME = '濉溪经济开发区'

inserted = 0
skipped = 0
for r in records:
    url = r.get('url', '')
    title = r.get('title', '')
    content = r.get('content', '')
    publish_date = r.get('pub_date', '')
    attachments = r.get('attachments', '')
    summary = content[:200] if content else ''
    source = r.get('source', '')

    c.execute('SELECT COUNT(*) FROM gov_raw WHERE page_url=?', (url,))
    if c.fetchone()[0] > 0:
        skipped += 1
        continue

    c.execute(
        'INSERT INTO gov_raw (page_url, title, content, publish_date, site_name, summary, attachments, source_url) '
        'VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
        (url, title, content, publish_date, SITE_NAME, summary, attachments, source)
    )
    inserted += 1

conn.commit()
print(f'Inserted: {inserted}, Skipped: {skipped}')
conn.close()