#!/usr/bin/env python3
"""Import hbnq data into the gov DB"""
import json, sqlite3

jsonl_path = '/root/gov_crawler/output/hbnq_gsgg.jsonl'
results = []
with open(jsonl_path, 'r', encoding='utf-8') as f:
    for line in f:
        results.append(json.loads(line))

db_path = '/root/gov_crawler/search.db'
conn = sqlite3.connect(db_path)
cursor = conn.cursor()

cursor.execute("DELETE FROM gov_raw WHERE url LIKE 'https://www.hbnq.gov.cn/content/11049/%'")
print("Deleted %d existing records" % cursor.rowcount)

inserted = 0
for r in results:
    try:
        cursor.execute(
            "INSERT INTO gov_raw (title, content, publish_date, url, site_name, source, attachments) VALUES (?, ?, ?, ?, ?, ?, ?)",
            (r['title'], r.get('content', ''), r['date'], r['link'], r['site_name'], r['group'],
             json.dumps(r.get('attachments', []), ensure_ascii=False))
        )
        inserted += 1
    except sqlite3.IntegrityError:
        pass

conn.commit()

cursor.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE url LIKE 'https://www.hbnq.gov.cn/content/11049/%')")
cursor.execute("INSERT INTO gov_search (rowid, title, content, site_name) SELECT id, title, content, site_name FROM gov_raw WHERE url LIKE 'https://www.hbnq.gov.cn/content/11049/%'")
conn.commit()

cursor.execute("SELECT COUNT(*) FROM gov_raw WHERE url LIKE 'https://www.hbnq.gov.cn/content/11049/%'")
print("Total in gov_raw: %d" % cursor.fetchone()[0])
cursor.execute("SELECT COUNT(*) FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE url LIKE 'https://www.hbnq.gov.cn/content/11049/%')")
print("Total in gov_search FTS: %d" % cursor.fetchone()[0])

conn.close()
import json

config_path = '/root/gov_crawler/daily_crawl_config.json'
with open(config_path, 'r', encoding='utf-8') as f:
    config = json.load(f)

# Check if already exists
for c in config['crawlers']:
    if 'hbnq' in c.get('script', ''):
        print("Already exists in config: %s" % c['name'])
        exit(0)

new_entry = {
    "name": "内丘县人民政府-公告公示",
    "script": "crawl_hbnq_gsgg.py",
    "incremental": True,
    "args": "",
    "sync_mode": "direct_server",
    "enabled": True,
    "group": "内丘县",
    "note": "hbnq.gov.cn 内丘县人民政府-公告公示, 自定义CMS, channelList分页(10页), 200条, detail页meta ArticleTitle完整标题"
}

config['crawlers'].append(new_entry)
with open(config_path, 'w', encoding='utf-8') as f:
    json.dump(config, f, ensure_ascii=False, indent=2)

print("Added config: %s" % new_entry['name'])
print("Total crawlers: %d" % len(config['crawlers']))