#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
爬虫：昆都仑区人民政府 - 通知公告
站点：https://www.kdl.gov.cn/xwdt/tzgg/
CMS：TRS CMS
分页：index_N.html (N=0~30)，每页10条，共31页~310条
"""

import requests
import re
import sqlite3
import os
import time
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://www.kdl.gov.cn/xwdt/tzgg/"
SITE_NAME = "昆都仑区-通知公告"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
}
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
TOTAL_PAGES = 31  # index_0 ~ index_30

def _get_db():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA synchronous=NORMAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn

def ensure_table(conn):
    conn.execute("CREATE TABLE IF NOT EXISTS gov_raw (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, content TEXT, source_url TEXT NOT NULL UNIQUE, publish_date TEXT, site_name TEXT, created_at TEXT DEFAULT (datetime('now','localtime')))")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date)")
    conn.commit()

def row_exists(conn, source_url):
    cur = conn.execute("SELECT 1 FROM gov_raw WHERE source_url=?", (source_url,))
    return cur.fetchone() is not None

def insert_row(conn, title, content, source_url, publish_date):
    conn.execute("INSERT OR IGNORE INTO gov_raw(title, content, source_url, publish_date, site_name) VALUES (?,?,?,?,?)",
                 (title, content, source_url, publish_date, SITE_NAME))
    if conn.total_changes > 0:
        return True
    return False

def extract_date_from_url(url):
    """从URL文件名提取日期: t20260624_930176.html → 2026-06-24"""
    m = re.search(r'/t(\d{8})_\d+\.html', url)
    if m:
        d = m.group(1)
        return f"{d[:4]}-{d[4:6]}-{d[6:8]}"
    return ""

def extract_content(soup):
    """从详情页提取正文"""
    div = soup.select_one("#content")
    if not div:
        div = soup.select_one(".content, .article-content, .TRS_Editor, #zoom, .detail")
    if div:
        for img in div.find_all("img"):
            src = img.get("src", "")
            if src and not src.startswith("data:"):
                img_url = urljoin(BASE_URL, src) if not src.startswith("http") else src
                img.replace_with(f'<img src="{img_url}">')
            else:
                img.decompose()
        for a in div.find_all("a", href=True):
            h = a["href"]
            if h and not h.startswith("http") and not h.startswith("#") and not h.startswith("javascript"):
                a["href"] = urljoin(BASE_URL, h)
        return str(div)
    return ""

def extract_title(soup):
    """从详情页提取真实标题（页面.title，比列表页更干净）"""
    for sel in ['.title', 'h1', '.h1', '.article-title', '.bt_title', '.detail-title']:
        el = soup.select_one(sel)
        if el:
            txt = el.get_text(strip=True)
            if 5 <= len(txt) <= 200:
                return txt
    return ""

def get_detail(url, session, retries=3):
    """获取详情页内容和真实标题"""
    for attempt in range(retries):
        try:
            r = session.get(url, headers=HEADERS, timeout=60)
            r.encoding = 'utf-8'
            if r.status_code == 200:
                soup = BeautifulSoup(r.text, 'html.parser')
                content = extract_content(soup)
                proper_title = extract_title(soup)
                return content, proper_title
        except requests.RequestException as e:
            if attempt < retries - 1:
                time.sleep(2 ** attempt)
    return "", ""

def parse_list_page(url, session):
    """解析列表页"""
    results = []
    try:
        r = session.get(url, headers=HEADERS, timeout=60)
        r.encoding = 'utf-8'
        if r.status_code != 200:
            return results
    except Exception:
        return results

    soup = BeautifulSoup(r.text, 'html.parser')
    
    for ul in soup.find_all('ul'):
        for li in ul.find_all('li'):
            a = li.find('a')
            if not a or not a.get('href'):
                continue
            href = a['href']
            if 't20' not in href:
                continue
            title = a.get_text(strip=True)
            if not title or len(title) < 5:
                continue
            # 清理标题：去掉前缀的 MM-DD
            title = re.sub(r'^\d{2}-\d{2}', '', title).strip()
            
            detail_url = href if href.startswith('http') else urljoin(BASE_URL, href)
            date_str = extract_date_from_url(detail_url)
            
            results.append((title, detail_url, date_str))
    
    return results

def crawl():
    conn = _get_db()
    ensure_table(conn)
    session = requests.Session()
    session.headers.update(HEADERS)
    total_new = 0
    total_skip = 0
    
    for page in range(TOTAL_PAGES):
        url = BASE_URL if page == 0 else f"{BASE_URL}index_{page}.html"
        articles = parse_list_page(url, session)
        if not articles:
            print(f"  第{page+1}页无文章，结束")
            break
        
        for title, detail_url, date_str in articles:
            if row_exists(conn, detail_url):
                total_skip += 1
                continue
            content, proper_title = get_detail(detail_url, session)
            # 优先使用详情页的干净标题
            final_title = proper_title if proper_title else title
            ct = BeautifulSoup(content or '', 'html.parser').get_text(strip=True)
            if len(ct) < 20:
                print(f"  ⚠ 内容过短: {final_title[:30]}... 跳过")
                total_skip += 1
                continue
            if insert_row(conn, final_title, content, detail_url, date_str):
                total_new += 1
                conn.commit()
                print(f"  ✓ [{total_new}] {final_title[:40]} ({date_str})")
            else:
                total_skip += 1
        
        print(f"  第{page+1}/{TOTAL_PAGES}页: {len(articles)}条, 累计新{total_new}条/跳过{total_skip}条")
        time.sleep(1)
    
    conn.close()
    print(f"\n✅ 完成！新增{total_new}条，跳过{total_skip}条")

if __name__ == '__main__':
    crawl()