#!/usr/bin/env python3
import sys, re, sqlite3, ssl, traceback
from datetime import datetime
from urllib.parse import urljoin
import requests

ssl._create_default_https_context = ssl._create_unverified_context
BASE = "https://kfq.ahsz.gov.cn"
COLUMN_URL = "https://kfq.ahsz.gov.cn/content/column/111328164"
SITE_NAME = "sz_kfq_gsgg"
DB_PATH = "/root/search.db"
headers = {"User-Agent": "Mozilla/5.0"}

def extract_div(html):
    m = re.search(r'(<div[^>]*class="[^"]*j-fontContent[^"]*"[^>]*>)', html)
    if not m: return ""
    c = html[m.start():]
    d = 0
    for i in range(len(c)):
        if c[i:i+4] == '<div': d += 1
        elif c[i:i+6] == '</div>':
            d -= 1
            if d == 0: return c[len(m.group(1)):i].strip()
    return ""

s = requests.Session()
s.headers.update(headers)

total_saved = 0
for pg in range(1, 6):
    try:
        r = s.get(f"{COLUMN_URL}?pageIndex={pg}&pageSize=20", timeout=20)
        h = r.text
    except:
        print(f"P{pg} list fail"); break
    
    items = re.findall(r'<a href="([^"]+)"[^>]*title="([^"]*)"[^>]*class="left">', h)
    dates = re.findall(r'<span[^>]*class="right date"[^>]*>([^<]+)</span>', h)
    print(f"P{pg}: {len(items)} items")
    
    for idx, (href, title) in enumerate(items):
        u = href if href.startswith('http') else BASE + '/' + href.lstrip('/')
        d = dates[idx].strip()[:10] if idx < len(dates) else ""
        print(f"  [{idx+1}/{len(items)}] {title[:45]}...", end=' ')
        
        try:
            r2 = s.get(u, timeout=10)
            content = extract_div(r2.text)
            conn = sqlite3.connect(DB_PATH, timeout=60)
            conn.execute("PRAGMA busy_timeout=30000")
            c = conn.cursor()
            if c.execute("SELECT id FROM gov_raw WHERE source_url=?", (u,)).fetchone():
                print("SKIP")
            else:
                ts = int(datetime.strptime(d, "%Y-%m-%d").timestamp()) if d else 0
                c.execute("INSERT INTO gov_raw VALUES (NULL,?,?,?,?,?,?,?,?)",
                          (SITE_NAME, u, u, title.strip(), d, ts, content[:200] if content else "", content))
                conn.commit()
                total_saved += 1
                print("OK")
            conn.close()
        except Exception as e:
            print(f"FAIL({e})")

print(f"\n=== Complete: {total_saved} records ===")