#!/usr/bin/env python3
"""Import hbqj eia data into the gov DB + add config"""
import json, sqlite3, os, sys

jsonl_path = '/root/gov_crawler/output/hbqj_eia.jsonl'
results = []
with open(jsonl_path, 'r', encoding='utf-8') as f:
    for line in f:
        results.append(json.loads(line))

db_path = '/root/gov_crawler/search.db'
conn = sqlite3.connect(db_path)
cursor = conn.cursor()

cursor.execute("DELETE FROM gov_raw WHERE url LIKE 'https://www.hbqj.gov.cn/xxgk/xxgkml/szfxxgkml/gysyjs/hjbh/jsxmhjyxpjgs/%'")
print("Deleted %d existing records" % cursor.rowcount)

inserted = 0
dupes = 0
for r in results:
    try:
        cursor.execute(
            "INSERT INTO gov_raw (title, content, publish_date, url, site_name, source, attachments) VALUES (?, ?, ?, ?, ?, ?, ?)",
            (r['title'], r.get('content', ''), r['date'], r['link'], r['site_name'], r['group'],
             json.dumps(r.get('attachments', []), ensure_ascii=False))
        )
        inserted += 1
    except sqlite3.IntegrityError:
        dupes += 1

conn.commit()

cursor.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE url LIKE 'https://www.hbqj.gov.cn/xxgk/xxgkml/szfxxgkml/gysyjs/hjbh/jsxmhjyxpjgs/%')")
cursor.execute("""
    INSERT INTO gov_search (rowid, title, content, site_name)
    SELECT id, title, content, site_name FROM gov_raw
    WHERE url LIKE 'https://www.hbqj.gov.cn/xxgk/xxgkml/szfxxgkml/gysyjs/hjbh/jsxmhjyxpjgs/%'
""")
conn.commit()

cursor.execute("SELECT COUNT(*) FROM gov_raw WHERE url LIKE 'https://www.hbqj.gov.cn/xxgk/xxgkml/szfxxgkml/gysyjs/hjbh/jsxmhjyxpjgs/%'")
print("Total in gov_raw: %d" % cursor.fetchone()[0])
cursor.execute("SELECT COUNT(*) FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE url LIKE 'https://www.hbqj.gov.cn/xxgk/xxgkml/szfxxgkml/gysyjs/hjbh/jsxmhjyxpjgs/%')")
print("Total in gov_search FTS: %d" % cursor.fetchone()[0])
conn.close()

# Add config
config_path = '/root/gov_crawler/daily_crawl_config.json'
with open(config_path, 'r', encoding='utf-8') as f:
    config = json.load(f)

for c in config['crawlers']:
    if 'hbqj_eia' in c.get('script', ''):
        print("Already exists: %s" % c['name'])
        sys.exit(0)

new_entry = {
    "name": "潜江市人民政府-建设项目环境影响评价公示",
    "script": "crawl_hbqj_eia.py",
    "incremental": True,
    "args": "",
    "sync_mode": "direct_server",
    "enabled": True,
    "group": "潜江市",
    "note": "hbqj.gov.cn 潜江市-建设项目环境影响评价公示, TRS CMS, index.html+index_n.html分页(25页), 400条, a[title]完整标题"
}

config['crawlers'].append(new_entry)
with open(config_path, 'w', encoding='utf-8') as f:
    json.dump(config, f, ensure_ascii=False, indent=2)

print("Added config: %s" % new_entry['name'])
print("Total crawlers: %d" % len(config['crawlers']))