#!/usr/bin/env python3
"""Import cqcs data into search DB"""
import json, sqlite3, os

jsonl_path = '/root/gov_crawler/output/cqcs_bmjz.jsonl'
if not os.path.exists(jsonl_path):
    print("JSONL not found yet!")
    exit(1)

with open(jsonl_path, 'r', encoding='utf-8') as f:
    results = [json.loads(line) for line in f if line.strip()]

print(f"Loaded {len(results)} records")

conn = sqlite3.connect('/mnt/data/search.db')
cursor = conn.cursor()

# Delete existing records from this site (URL prefix)
cursor.execute("DELETE FROM gov_raw WHERE page_url LIKE 'https://www.cqcs.gov.cn/zwxx_164/bmjz/%'")
deleted = cursor.rowcount
print(f"Deleted {deleted} existing records")

inserted = 0
for r in results:
    title = r['title']
    content = r.get('content', '')
    publish_date = r['date']
    site_name = r['site_name']
    source = r.get('group', '长寿区')
    attachments = json.dumps(r.get('attachments', []), ensure_ascii=False)
    page_url = r['link']

    try:
        cursor.execute(
            "INSERT INTO gov_raw (title, content, publish_date, page_url, site_name, source, attachments) VALUES (?, ?, ?, ?, ?, ?, ?)",
            (title, content, publish_date, page_url, site_name, source, attachments)
        )
        inserted += 1
    except sqlite3.IntegrityError:
        pass

conn.commit()
print(f"Inserted {inserted} records")

# Sync to gov_search FTS
cursor.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE page_url LIKE 'https://www.cqcs.gov.cn/zwxx_164/bmjz/%')")
cursor.execute("""
    INSERT INTO gov_search (rowid, site_name, title, summary)
    SELECT id, site_name, title, substr(content, 1, 800) as summary
    FROM gov_raw
    WHERE page_url LIKE 'https://www.cqcs.gov.cn/zwxx_164/bmjz/%'
""")
conn.commit()

# Verify
cnt = cursor.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url LIKE 'https://www.cqcs.gov.cn/zwxx_164/bmjz/%'").fetchone()[0]
fts_cnt = cursor.execute("SELECT COUNT(*) FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE page_url LIKE 'https://www.cqcs.gov.cn/zwxx_164/bmjz/%')").fetchone()[0]
print(f"gov_raw: {cnt}, gov_search: {fts_cnt}")

conn.close()
print("Import complete!")