#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
爬虫：阳新县人民政府 - 生态环境
站点：https://www.yx.gov.cn/zfxxgk/fdzdgknr/gysyjs/sthj_11098/
CMS：TRS CMS
分页：index_N.html (N=0~66)，共67页1330条
每页：~20条
"""

import requests
import re
import sqlite3
import os
import time
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://www.yx.gov.cn/zfxxgk/fdzdgknr/gysyjs/sthj_11098/"
SITE_NAME = "阳新县-生态环境"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
}
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
TOTAL_PAGES = 67

def _get_db():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA synchronous=NORMAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn

def ensure_table(conn):
    conn.execute("CREATE TABLE IF NOT EXISTS gov_raw (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, content TEXT, source_url TEXT NOT NULL UNIQUE, publish_date TEXT, site_name TEXT, created_at TEXT DEFAULT (datetime('now','localtime')))")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date)")
    conn.commit()

def row_exists(conn, source_url):
    cur = conn.execute("SELECT 1 FROM gov_raw WHERE source_url=?", (source_url,))
    return cur.fetchone() is not None

def insert_row(conn, title, content, source_url, publish_date):
    conn.execute("INSERT OR IGNORE INTO gov_raw(title, content, source_url, publish_date, site_name) VALUES (?,?,?,?,?)",
                 (title, content, source_url, publish_date, SITE_NAME))
    if conn.total_changes > 0:
        return True
    return False

def extract_date_from_url(url):
    m = re.search(r'/t(\d{8})_\d+\.html', url)
    if m:
        d = m.group(1)
        return f"{d[:4]}-{d[4:6]}-{d[6:8]}"
    return ""

def extract_content(soup):
    """从详情页提取正文 - 精确匹配 div.article-box"""
    for sel in ["div.article-box", "div.view.TRS_UEDITOR", "div.article_main"]:
        div = soup.select_one(sel)
        if div and len(div.get_text(strip=True)) > 50:
            for img in div.find_all("img"):
                src = img.get("src", "")
                if src and not src.startswith("data:"):
                    img_url = urljoin(BASE_URL, src) if not src.startswith("http") else src
                    img.replace_with(f'<img src="{img_url}">')
                else:
                    img.decompose()
            for a in div.find_all("a", href=True):
                h = a["href"]
                if h and not h.startswith("http") and not h.startswith("#") and not h.startswith("javascript"):
                    a["href"] = urljoin(BASE_URL, h)
            return str(div)
    return ""

def extract_proper_title(soup):
    for sel in ['h2', 'h1', '.h1', '.title', '.article-title', '.bt_title']:
        el = soup.select_one(sel)
        if el:
            txt = el.get_text(strip=True)
            if 5 <= len(txt) <= 200:
                return txt
    return ""

def get_detail(url, session, retries=3):
    for attempt in range(retries):
        try:
            r = session.get(url, headers=HEADERS, timeout=60)
            r.encoding = 'utf-8'
            if r.status_code == 200:
                soup = BeautifulSoup(r.text, 'html.parser')
                content = extract_content(soup)
                title = extract_proper_title(soup)
                return content, title
        except requests.RequestException as e:
            if attempt < retries - 1:
                time.sleep(2 ** attempt)
    return "", ""

def parse_list_page(url, session):
    results = []
    try:
        r = session.get(url, headers=HEADERS, timeout=60)
        r.encoding = 'utf-8'
        if r.status_code != 200:
            return results
    except Exception:
        return results

    soup = BeautifulSoup(r.text, 'html.parser')
    
    for a in soup.find_all('a', href=True):
        href = a['href']
        if '/t20' not in href:
            continue
        title = a.get_text(strip=True)
        if not title or len(title) < 5:
            continue
        
        detail_url = href if href.startswith('http') else urljoin(BASE_URL, href)
        date_str = extract_date_from_url(detail_url)
        results.append((title, detail_url, date_str))
    
    return results

def crawl():
    conn = _get_db()
    ensure_table(conn)
    session = requests.Session()
    session.headers.update(HEADERS)
    total_new = 0
    total_skip = 0
    
    for page in range(TOTAL_PAGES):
        if page == 0:
            url = BASE_URL
        else:
            url = f"{BASE_URL}index_{page}.html"
        
        articles = parse_list_page(url, session)
        if not articles:
            print(f"  第{page+1}页无文章，结束")
            break
        
        for list_title, detail_url, date_str in articles:
            if row_exists(conn, detail_url):
                total_skip += 1
                continue
            content, proper_title = get_detail(detail_url, session)
            final_title = proper_title if proper_title else list_title
            ct = BeautifulSoup(content or '', 'html.parser').get_text(strip=True)
            if len(ct) < 20:
                total_skip += 1
                continue
            if insert_row(conn, final_title, content, detail_url, date_str):
                total_new += 1
                conn.commit()
                print(f"  ✓ [{total_new}] {final_title[:40]} ({date_str})")
            else:
                total_skip += 1
        
        print(f"  第{page+1}/{TOTAL_PAGES}页: {len(articles)}条, 累计新{total_new}条/跳过{total_skip}条")
        time.sleep(1)
    
    conn.close()
    print(f"\n✅ 完成！新增{total_new}条，跳过{total_skip}条")

if __name__ == '__main__':
    crawl()