#!/usr/bin/env python3
import os
"""Crawl 白银高新区管委会 - 通知公告"""
import sys, re, os, requests
from datetime import datetime, timedelta
import sqlite3

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "白银高新区-通知公告"
BASE_URL = "https://www.baiyin.gov.cn"
API_URL = BASE_URL + "/api-gateway/jpaas-publish-server/front/page/build/unit"
API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "95a12467959f44d29b0148749a6e0daf",
    "tplSetId": "718c52a1506547739d141ce0ed891fd3",
    "pageType": "column",
    "tagId": "列表数据",
    "editType": "null",
    "pageId": "30a6064e0b3740f1ab77f5f04dc436bf",
}
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def get_list(page):
    """Fetch one page of list data from API"""
    try:
        params = dict(API_PARAMS, page=str(page))
        r = requests.get(API_URL, params=params, headers=HEADERS, timeout=20)
        data = r.json()
        html = data["data"]["html"]
        # Parse items
        items = re.findall(
            r'<a class="fl" href="([^"]+)" title="([^"]*)".*?<span class="fr">([^<]+)',
            html, re.DOTALL
        )
        # Get total count
        count_m = re.search(r'count="(\d+)"', html)
        total = int(count_m.group(1)) if count_m else len(items)
        return items, total
    except Exception as e:
        print("  API error page %d: %s" % (page, e), flush=True)
        return [], 0

def extract_content(html):
    """Extract content from div.content inside div.detail using depth counting"""
    idx = html.find('<div class="content">')
    if idx < 0:
        return ""
    start = idx + len('<div class="content">')
    depth = 1
    pos = start
    while pos < len(html) and depth > 0:
        if html[pos:pos+4] == '<div':
            depth += 1
            pos += 4
        elif html[pos:pos+6] == '</div>':
            depth -= 1
            pos += 6
        else:
            pos += 1
    content = html[start:pos-6]
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
    return content.strip()

def fetch_detail(url_path):
    """Fetch detail page and extract title, date, content"""
    url = BASE_URL + url_path
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        html = r.text
        title = ""
        t = re.search(r'<title>(.*?)</title>', html)
        if t:
            title = t.group(1).strip()[:200]
        content = extract_content(html)
        if not content or len(content.strip()) < 30:
            content = ""
        # Date from detail page timer
        date = ""
        pd = re.search(r'发布时间.*?(\d{4}-\d{2}-\d{2})', html)
        if pd:
            date = pd.group(1)
        return title, date, content, url
    except Exception as e:
        print("  Detail error: %s" % e, flush=True)
        return "", "", "", url

def main():
    print("Starting: %s" % SITE_NAME, flush=True)
    
    print("Phase 1: Getting list info...", flush=True)
    first_page, total = get_list(1)
    if total == 0:
        print("  No data found!", flush=True)
        return
    
    total_pages = (total + 14) // 15
    print("  Total items: %d, Pages: %d" % (total, total_pages), flush=True)
    
    print("Phase 2: Collecting list items...", flush=True)
    all_items = []
    for page in range(1, total_pages + 1):
        items, _ = get_list(page) if page > 1 else (first_page, total)
        if not items:
            break
        for url_path, title, date in items:
            if date and date < CUTOFF:
                continue
            # Clean title (remove HTML entities if any)
            clean_title = title.strip()[:200]
            all_items.append({"url_path": url_path, "title": clean_title, "date": date})
        if items and items[-1][2] < CUTOFF:
            break
        if page % 5 == 0:
            print("  Collected page %d/%d, items so far: %d" % (page, total_pages, len(all_items)), flush=True)
    
    print("  Total items within 3 years: %d" % len(all_items), flush=True)
    if not all_items:
        print("  Nothing to crawl!", flush=True)
        return
    
    print("Phase 3: Fetching details and saving...", flush=True)
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    c = conn.cursor()
    c.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT, content TEXT, summary TEXT,
        source_url TEXT UNIQUE, site_name TEXT,
        publish_date TEXT, category TEXT
    )""")
    
    saved = 0
    for i, item in enumerate(all_items):
        detail_url = BASE_URL + item["url_path"]
        # Quick check if exists
        existing = c.execute("SELECT 1 FROM gov_raw WHERE source_url = ?", (detail_url,)).fetchone()
        if existing:
            continue
        
        title, date, content, _ = fetch_detail(item["url_path"])
        title = title or item["title"]
        date = date or item["date"]
        if not content or not content.strip():
            content = ""
        
        try:
            c.execute("""INSERT OR REPLACE INTO gov_raw
                (title, content, summary, source_url, site_name, publish_date, category)
                VALUES (?,?,?,?,?,?,?)""",
                (title, content, title[:200], detail_url, SITE_NAME, date, ""))
            if c.rowcount > 0:
                saved += 1
        except Exception as e:
            print("  DB error: %s" % e, flush=True)
        
        if (i + 1) % 20 == 0:
            conn.commit()
            print("  Saved %d/%d items..." % (i+1, len(all_items)), flush=True)
    
    conn.commit()
    conn.close()
    print("  Total saved: %d" % saved, flush=True)
    
    # FTS
    print("Phase 4: Updating FTS...", flush=True)
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()
    try:
        c.execute("""INSERT OR REPLACE INTO gov_search (rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r WHERE r.site_name = ? AND r.content != ''
            AND NOT EXISTS (SELECT 1 FROM gov_search s WHERE s.rowid = r.id)""", (SITE_NAME,))
        conn.commit()
        print("  FTS updated", flush=True)
    except Exception as e:
        print("  FTS error: %s" % e, flush=True)
    conn.close()
    
    print("Done!", flush=True)

if __name__ == "__main__":
    main()