#!/usr/bin/env python3
"""Run jzth crawler and import into DB"""
import crawl_jzth, json, re, sqlite3, sys
from io import StringIO

old_stdout = sys.stdout
sys.stdout = StringIO()

result = crawl_jzth.main()

output = sys.stdout.getvalue()
sys.stdout = old_stdout

json_match = re.search(r"JSON_BEGIN:(.+):JSON_END", output, re.DOTALL)
if json_match:
    data = json.loads(json_match.group(1))
    items = data["items"]
    print(f"PARSE OK: {len(items)} items")
    
    conn = sqlite3.connect("/mnt/data/search.db")
    c = conn.cursor()
    
    inserted = 0
    skipped = 0
    for item in items:
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],))
        if c.fetchone():
            skipped += 1
            continue
        
        title = item["title"]
        date_str = item["date"][:10]
        content = item.get("content") or ""
        
        c.execute(
            "INSERT INTO gov_raw (page_url, title, publish_date, content, site_name, category) VALUES (?, ?, ?, ?, ?, ?)",
            (item["url"], title, date_str, content, "jzth_tzgg", "辽宁")
        )
        inserted += 1
        
        if inserted % 20 == 0:
            conn.commit()
    
    conn.commit()
    
    c.execute("SELECT name FROM sqlite_master WHERE type=? AND name=?", ("table", "gov_raw_fts"))
    if c.fetchone():
        c.execute("INSERT INTO gov_raw_fts(gov_raw_fts) VALUES('rebuild')")
        conn.commit()
    
    conn.close()
    
    print(f"入库: {inserted}, 跳过(已存在): {skipped}")
else:
    print("JSON PARSE FAILED")