#!/usr/bin/env python3
"""Test crawl_tqx.py - first 3 pages"""
import sys
sys.path.insert(0, '/root/gov_crawler')
import crawl_tqx as c
import time
from bs4 import BeautifulSoup

conn = c.sqlite3.connect(c.DB_PATH)
cur = conn.cursor()
cur.execute("DELETE FROM gov_raw WHERE site_name=? AND page_url LIKE '%tqx.gov.cn%'", (c.SITE_NAME,))
conn.commit()
print("Cleaned old data")

total = 0
for page in [1, 2, 3]:
    url = c.LIST_URL.format(page=page)
    print(f"Page {page}...", end=" ", flush=True)
    r = c.session.get(url, timeout=20)
    soup = BeautifulSoup(r.text, "html.parser")
    cl = soup.find("ul", class_="content-list")
    if not cl:
        print("no list")
        break
    items = cl.find_all("li")
    print(f"{len(items)} items")
    for item in items:
        a = item.find("a")
        if not a:
            continue
        link = a.get("href", "").strip()
        if not link:
            continue
        if not link.startswith("http"):
            link = c.BASE_URL + link if link.startswith("/") else c.BASE_URL + "/" + link
        if "tqx.gov.cn" not in link:
            continue
        cur.execute("SELECT 1 FROM gov_raw WHERE page_url=?", (link,))
        if cur.fetchone():
            continue
        time.sleep(1.0)
        title, pub_date, content = c.extract_detail(link)
        if not title:
            title = a.get("title", "") or a.get_text(strip=True)
        if not content or len(content) < 20:
            content = f"[{title}]({link})"
        try:
            cur.execute(
                "INSERT OR REPLACE INTO gov_raw (page_url, title, site_name, publish_date, content, date_rank, summary) VALUES (?,?,?,?,?,?,?)",
                (link, title, c.SITE_NAME, pub_date, content, pub_date.replace("-", "") if pub_date else "0", title[:200]),
            )
            total += 1
        except Exception as e:
            print(f"  ERROR: {e}")
    time.sleep(1.0)

conn.commit()
conn.close()
print(f"\nTotal added: {total}")

# Verify
conn2 = c.sqlite3.connect(c.DB_PATH)
cur2 = conn2.cursor()
cur2.execute("SELECT COUNT(*), MIN(length(content)), MAX(length(content)) FROM gov_raw WHERE site_name=?", (c.SITE_NAME,))
cnt, minc, maxc = cur2.fetchone()
print(f"Verification: {cnt} records, content range: {minc}-{maxc} chars")
conn2.close()