#!/usr/bin/env python3
"""Test crawl_szwz_jshp.py first 3 pages"""
import sys
sys.path.insert(0, '/root/gov_crawler')
import crawl_szwz_jshp as c
import time

conn = c.sqlite3.connect(c.DB_PATH)
cur = conn.cursor()
cur.execute("DELETE FROM gov_raw WHERE site_name=? AND page_url LIKE '%jshp%'", (c.SITE_NAME,))
conn.commit()
print("Cleaned")

total = 0
for page in [1, 2, 3]:
    url = c.LIST_URL.format(page="") if page == 1 else c.LIST_URL.format(page="_" + str(page))
    soup = c.base.get_soup(url)
    items = soup.select("li.ewb-info-item")
    print(f"Page {page}: {len(items)} items")
    for item in items:
        a = item.find("a")
        link = a["href"].strip()
        link = c.base.resolve_url(url, soup, link)
        cur.execute("SELECT 1 FROM gov_raw WHERE page_url=?", (link,))
        if cur.fetchone():
            continue
        time.sleep(0.5)
        title, pub_date, content = c.base.extract_detail(link)
        if not title:
            title = a.get_text(strip=True)
        if not content or len(content) < 20:
            content = f"[{title}]({link})"
        cur.execute(
            "INSERT OR REPLACE INTO gov_raw (page_url, title, site_name, publish_date, content, date_rank, summary) VALUES (?,?,?,?,?,?,?)",
            (link, title, c.SITE_NAME, pub_date, content, pub_date.replace("-", "") if pub_date else "0", title[:200]),
        )
        total += 1
    time.sleep(0.5)

conn.commit()
conn.close()
print(f"Total added: {total}")