#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
爬虫：滕州市级索镇 - 通知公告
站点：http://www.tengzhou.gov.cn/zwgk/xxgkml/zj/jsz/
POST分页（TRS CMS），9页166条
"""

import requests, re, sqlite3, os, time, sys
from bs4 import BeautifulSoup
from urllib.parse import urljoin

POST_URL = "http://www.tengzhou.gov.cn/govsearch/searPageNewTengZhou.jsp"
BASE_URL = "http://www.tengzhou.gov.cn/zwgk/xxgkml/zj/jsz/"
SITE_NAME = "滕州市级索镇-通知公告"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

# 分页参数
SITEID = "189"
CLASSINFOID = "4162"
CHANNELID = "12630"
TOTAL_PAGES = 9
PAGE_SIZE = 20

def _get_db():
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA synchronous=NORMAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn

def ensure_table(conn):
    conn.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT NOT NULL,
        content TEXT,
        source_url TEXT NOT NULL UNIQUE,
        publish_date TEXT,
        site_name TEXT,
        created_at TEXT DEFAULT (datetime('now','localtime'))
    )""")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)")
    conn.execute("CREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date)")
    conn.commit()

def row_exists(conn, source_url):
    cur = conn.execute("SELECT 1 FROM gov_raw WHERE source_url=?", (source_url,))
    return cur.fetchone() is not None

def insert_row(conn, title, content, source_url, publish_date):
    conn.execute("""INSERT OR IGNORE INTO gov_raw(title, content, source_url, publish_date, site_name)
        VALUES (?,?,?,?,?)""",
        (title, content, source_url, publish_date, SITE_NAME))
    if conn.total_changes > 0:
        return True
    return False

def extract_date_from_url(url):
    m = re.search(r'/t(\d{8})_\d+\.html', url)
    if m:
        d = m.group(1)
        return f"{d[:4]}-{d[4:6]}-{d[6:8]}"
    return ""

def get_detail(url, session, retries=3):
    for attempt in range(retries):
        try:
            r = session.get(url, headers=HEADERS, timeout=60)
            r.encoding = 'utf-8'
            if r.status_code == 200:
                soup = BeautifulSoup(r.text, 'html.parser')
                # 标题
                h2 = soup.select_one("h2")
                title = h2.get_text(strip=True) if h2 else ""
                if not title or len(title) < 5:
                    for sel in ['h1', '.h1', '.title', '.article-title', '.bt_title']:
                        el = soup.select_one(sel)
                        if el and 5 <= len(el.get_text(strip=True)) <= 200:
                            title = el.get_text(strip=True)
                            break
                # 正文
                content_div = soup.select_one("#content")
                if not content_div:
                    content_div = soup.select_one(".content, .article-content, .TRS_Editor, #zoom")
                content = str(content_div) if content_div else ""
                return content, title
        except requests.RequestException as e:
            if attempt < retries - 1:
                time.sleep(2 ** attempt)
    return "", ""

def fetch_page(page_num, session):
    """获取指定页码的列表页"""
    data = {
        "page": str(page_num),
        "siteid": SITEID,
        "classinfoid": CLASSINFOID,
        "channelid": CHANNELID,
    }
    try:
        r = session.post(POST_URL, data=data, headers=HEADERS, timeout=30)
        r.encoding = 'utf-8'
        if r.status_code != 200:
            print(f"  POST失败: HTTP {r.status_code}")
            return []
        soup = BeautifulSoup(r.text, 'html.parser')
        articles = []
        for a in soup.find_all('a', href=True):
            href = a['href']
            if '/t20' in href and '.html' in href:
                title_attr = a.get('title', '') or a.get_text(strip=True)
                if title_attr and len(title_attr) > 5:
                    detail_url = href if href.startswith('http') else urljoin(BASE_URL, href)
                    date_str = extract_date_from_url(detail_url)
                    articles.append((title_attr, detail_url, date_str))
        return articles
    except requests.RequestException as e:
        print(f"  POST异常: {e}")
        return []

def crawl():
    conn = _get_db()
    ensure_table(conn)
    session = requests.Session()
    session.headers.update(HEADERS)

    total_new = 0
    total_skip = 0
    total_dup = 0

    for page in range(1, TOTAL_PAGES + 1):
        print(f"\n📄 第{page}/{TOTAL_PAGES}页...")
        articles = fetch_page(page, session)
        print(f"  获取到{len(articles)}条")

        if not articles:
            print("  空页，终止")
            break

        for title, url, date_str in articles:
            if row_exists(conn, url):
                total_dup += 1
                continue

            content, proper_title = get_detail(url, session)
            final_title = proper_title if proper_title else title

            # 验证内容
            ct = BeautifulSoup(content or '', 'html.parser').get_text(strip=True)
            if len(ct) < 20:
                print(f"  ⚠ 内容过短({len(ct)}): {final_title[:30]}")
                total_skip += 1
                continue

            if insert_row(conn, final_title, content, url, date_str):
                total_new += 1
                conn.commit()
                if total_new % 10 == 0:
                    print(f"  ✓ 已入库{total_new}条...")
            else:
                total_skip += 1

    conn.close()
    print(f"\n{'='*40}")
    print(f"✅ 完成！新增{total_new}条，重复{total_dup}条，跳过{total_skip}条")
    print(f"{'='*40}")

if __name__ == '__main__':
    crawl()