#!/usr/bin/env python3
"""
白银区人民政府 - 公示公告 爬虫
Playwright列表页 (AJAX分页) + curl详情页
"""
import asyncio, base64, json, os, re, sqlite3, subprocess, sys, time, urllib.request

SITE_NAME = "白银区人民政府"
BASE_URL = "https://www.baiyinqu.gov.cn"
LIST_URL = f"{BASE_URL}/xwzx/GSGG/index.html"
DB_PATH = os.path.expanduser("~/gov_crawler/baiyinqu_results.db")
SERVER_SSH = "root@1.94.217.116"
SERVER_SEARCH_DB = "/root/search.db"

def init_db():
    conn = sqlite3.connect(DB_PATH)
    conn.execute('''CREATE TABLE IF NOT EXISTS crawl_results (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT, url TEXT UNIQUE, content TEXT,
        publish_date TEXT, summary TEXT
    )''')
    conn.execute('DELETE FROM crawl_results')
    conn.commit()
    return conn

def esc(v):
    s = str(v or "")
    return s.replace("'", "''")

def sync_to_server():
    """将本地数据直接写入 search.db（服务器本地模式）"""
    print("\n📤 同步到 search.db...")

    conn = sqlite3.connect(DB_PATH)
    rows = conn.execute("SELECT title, url, content, publish_date, summary FROM crawl_results ORDER BY id").fetchall()
    conn.close()

    if not rows:
        print("  本地没有数据")
        return

    dst = sqlite3.connect("/root/search.db")
    dst.execute("PRAGMA journal_mode=WAL")

    site_name = "白银区人民政府"
    new_count = 0
    for r in rows:
        title, url, content, pub_date, summary = r
        try:
            dst.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(title, page_url, content, publish_date, summary, site_name, tags) "
                "VALUES (?,?,?,?,?,?,?)",
                (title, url, (content or "")[:8000], pub_date or "",
                 (summary or "")[:300], site_name, "")
            )
            if dst.total_changes > 0:
                new_count += 1
        except Exception as e:
            print(f"  Error: {e}")

    if new_count > 0:
        dst.commit()
        # Update FTS
        dst.execute(
            "INSERT INTO gov_search(rowid,title,site_name,summary) "
            "SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r "
            "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
            (site_name,))
        dst.commit()

    total = dst.execute(
        "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (site_name,)).fetchone()[0]
    dst.close()

    print(f"  OK {new_count}/{len(rows)} 条同步到 search.db (DB共{total}条)")


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--full', action='store_true')
    parser.add_argument('--test', type=int, default=0)
    parser.add_argument('--sync', action='store_true')
    args = parser.parse_args()
    
    if args.sync:
        sync_to_server()
    else:
        asyncio.run(crawl(test_limit=args.test))