#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
爬虫：乌当区人民政府 - 通知公告
站点：https://www.gzwd.gov.cn/xwzx/tzgg/
CMS：TRS CMS
分页：index_N.html (0-indexed, N=0~232)，每页15条，共233页3486条
"""

import requests
import re
import sqlite3
import os
import time
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://www.gzwd.gov.cn/xwzx/tzgg/"
SITE_NAME = "乌当区-通知公告"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
}
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
TOTAL_PAGES = 233

def _get_db():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA synchronous=NORMAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn

def ensure_table(conn):
    conn.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT NOT NULL,
            content TEXT,
            source_url TEXT NOT NULL UNIQUE,
            publish_date TEXT,
            site_name TEXT,
            created_at TEXT DEFAULT (datetime('now','localtime'))
        )
    """)
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date)")
    # FTS 由 search.db 触发器 trg_gov_raw_fts_* 统一维护, 脚本不再自建 gov_fts 表
    conn.commit()

def row_exists(conn, source_url):
    cur = conn.execute("SELECT 1 FROM gov_raw WHERE source_url=?", (source_url,))
    return cur.fetchone() is not None

def insert_row(conn, title, content, source_url, publish_date):
    conn.execute(
        "INSERT OR IGNORE INTO gov_raw(title, content, source_url, publish_date, site_name) VALUES (?,?,?,?,?)",
        (title, content, source_url, publish_date, SITE_NAME)
    )
    if conn.total_changes > 0:
        return True
    return False

def extract_date_from_url(url):
    """从URL路径提取精确日期: t20260615 → 2026-06-15"""
    # 先尝试从文件名中提取完整日期 tYYYYMMDD
    m = re.search(r'/t(\d{8})_\d+\.html', url)
    if m:
        ds = m.group(1)
        return f"{ds[:4]}-{ds[4:6]}-{ds[6:8]}"
    # 备选：从年月份提取（精确到月）
    m = re.search(r'/(\d{6})/t\d+_\d+\.html', url)
    if m:
        y, mp = m.group(1)[:4], m.group(1)[4:6]
        return f"{y}-{mp}-01"
    return ""

def extract_content(soup):
    """从详情页提取正文 - TRS CMS"""
    for sel in ['.content', '#content', '.article-content', '.TRS_Editor', '.con_text', '#zoom']:
        div = soup.select_one(sel)
        if div:
            for img in div.find_all("img"):
                src = img.get("src", "")
                if src and not src.startswith("data:"):
                    img_url = urljoin(BASE_URL, src) if not src.startswith("http") else src
                    img.replace_with(f'<img src="{img_url}">')
                else:
                    img.decompose()
            for a_tag in div.find_all("a", href=True):
                h = a_tag["href"]
                if h and not h.startswith("http") and not h.startswith("#") and not h.startswith("javascript"):
                    a_tag["href"] = urljoin(BASE_URL, h)
            return str(div)
    return ""

def get_detail(url, session, retries=3):
    """获取详情页内容"""
    for attempt in range(retries):
        try:
            r = session.get(url, headers=HEADERS, timeout=60)
            r.encoding = 'utf-8'
            if r.status_code == 200:
                soup = BeautifulSoup(r.text, 'html.parser')
                content = extract_content(soup)
                return content
        except requests.RequestException as e:
            if attempt < retries - 1:
                time.sleep(2 ** attempt)
    return ""

def parse_list_page(url, session):
    """解析列表页，返回 [(title, detail_url, date), ...]"""
    results = []
    try:
        r = session.get(url, headers=HEADERS, timeout=60)
        r.encoding = 'utf-8'
        if r.status_code != 200:
            return results
    except Exception:
        return results

    soup = BeautifulSoup(r.text, 'html.parser')
    
    for a in soup.find_all('a', href=True):
        href = a['href']
        # TRS article URL pattern: /t20260615_90524558.html or full URL with year/month
        if not re.search(r'/\d{6}/t\d+_\d+\.html', href):
            continue
        title = a.get('title', '') or a.get_text(strip=True)
        if not title or len(title) < 5:
            continue
        
        detail_url = href if href.startswith('http') else urljoin(BASE_URL, href)
        date_str = extract_date_from_url(detail_url)
        
        results.append((title, detail_url, date_str))
    
    return results

def crawl():
    conn = _get_db()
    ensure_table(conn)
    
    session = requests.Session()
    session.headers.update(HEADERS)
    
    total_new = 0
    total_skip = 0
    
    for page in range(0, TOTAL_PAGES):
        if page == 0:
            url = BASE_URL
        else:
            url = f"{BASE_URL}index_{page}.html"
        
        articles = parse_list_page(url, session)
        if not articles:
            print(f"  第{page+1}页无文章，结束")
            break
        
        for title, detail_url, date_str in articles:
            if row_exists(conn, detail_url):
                total_skip += 1
                continue
            
            content = get_detail(detail_url, session)
            content_text = BeautifulSoup(content or '', 'html.parser').get_text(strip=True)
            if len(content_text) < 20:
                # TRS CMS detail page may have different content for some entries
                print(f"  ⚠ 内容过短: {title[:30]}... 跳过")
                total_skip += 1
                continue
            
            if insert_row(conn, title, content, detail_url, date_str):
                total_new += 1
                conn.commit()
                print(f"  ✓ [{total_new}] {title[:40]} ({date_str or '无日期'})")
            else:
                total_skip += 1
        
        print(f"  第{page+1}/{TOTAL_PAGES}页: {len(articles)}条, 累计新{total_new}条/跳过{total_skip}条")
        time.sleep(1)
    
    conn.close()
    print(f"\n✅ 采集完成！新增{total_new}条，跳过{total_skip}条，共{total_new+total_skip}条")

if __name__ == '__main__':
    crawl()