#!/usr/bin/env python3
"""石嘴山市人民政府 — 通知公告
https://www.shizuishan.gov.cn/xwzx/zwxx/tzgg/
TRS CMS, createPageHTML(38, 0, "index", "html") 分页
"""
import os, sys, re, time
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
BASE_URL = "https://www.shizuishan.gov.cn/xwzx/zwxx/tzgg/"
SITE_NAME = "石嘴山市人民政府-通知公告"
THRESHOLD = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
MAX_PAGES = 38

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
session = requests.Session()
session.headers.update(HEADERS)
import sqlite3

def store_item(title, date_str, content, page_url):
    try:
        conn = sqlite3.connect(DB_PATH, timeout=10)
        c = conn.cursor()
        summary = (content.strip()[:200] if content.strip() else '')
        date_rank = 0
        if date_str:
            try:
                date_rank = int(datetime.strptime(date_str[:10], "%Y-%m-%d").timestamp())
            except:
                pass
        c.execute("""INSERT OR IGNORE INTO gov_raw (title, publish_date, content, page_url, source_url, site_name, summary, date_rank)
                     VALUES (?,?,?,?,?,?,?,?)""",
                  (title.strip(), date_str, content.strip(), page_url.strip(), page_url.strip(), SITE_NAME, summary, date_rank))
        a = c.rowcount; conn.commit(); conn.close(); return a
    except Exception as e:
        print(f"[ERROR] 写入失败: {e}"); return 0

def parse_list_page(html, base_url):
    soup = BeautifulSoup(html, "html.parser")
    items = []
    ul = soup.find("ul", class_="newslist")
    if not ul: return items
    for li in ul.find_all("li"):
        a = li.find("a")
        if not a: continue
        href = a.get("href", "")
        title = a.get_text(strip=True)
        if not href or not title or len(title) < 5: continue
        full_url = urljoin(base_url, href)
        span = li.find("span")
        date_str = span.get_text(strip=True) if span else ""
        items.append((title, date_str, full_url))
    return items

def parse_detail_page(html, page_url):
    soup = BeautifulSoup(html, "html.parser")

    # Title
    title = ""
    tt = soup.find("title")
    if tt:
        title = tt.get_text(strip=True).replace("_石嘴山市人民政府", "").replace(" - 石嘴山市人民政府", "")
    if not title:
        at = soup.find("div", class_="article-title")
        if at: title = at.get_text(strip=True)

    # Date
    date_str = ""
    ds = soup.find("span", class_="date")
    if ds:
        m = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", ds.get_text())
        if m: date_str = m.group(1)

    # Content from TRS_UEDITOR or view class
    body_parts = []
    for cls_name in ["TRS_UEDITOR", "TRS_Editor"]:
        rc = soup.find("div", class_=re.compile(cls_name))
        if rc:
            for p in rc.find_all("p"):
                t = p.get_text(strip=True)
                if t and len(t) > 3: body_parts.append(t)
            for table in rc.find_all("table"):
                rows = []
                for tr in table.find_all("tr"):
                    cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])]
                    if cells: rows.append(" | ".join(cells))
                if rows: body_parts.append("表格:\n" + "\n".join(rows))
            break
    else:
        # Fallback: view class
        vw = soup.find("div", class_="view")
        if vw:
            for p in vw.find_all("p"):
                t = p.get_text(strip=True)
                if t and len(t) > 3: body_parts.append(t)
        else:
            for p in soup.find_all("p"):
                t = p.get_text(strip=True)
                if t and len(t) > 5: body_parts.append(t)

    seen = set()
    unique = []
    for part in body_parts:
        if part not in seen: seen.add(part); unique.append(part)
    content = "\n\n".join(unique)

    # Attachments
    atts = []
    for a in soup.find_all("a", href=True):
        h = a["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|caj)$", h, re.I):
            t = a.get_text(strip=True) or "附件"
            fu = h if h.startswith("http") else urljoin("https://www.shizuishan.gov.cn", h)
            link = f"[{t}]({fu})"
            if link not in atts: atts.append(link)

    if atts:
        if content:
            content += "\n\n---\n附件：\n" + "\n".join(atts)
        else:
            content = "附件：\n" + "\n".join(atts)

    return title, date_str, content

def main():
    print(f"[INFO] 开始爬取: {SITE_NAME}")
    print(f"[INFO] 阈值: {THRESHOLD}, 最大页数: {MAX_PAGES}")
    total_new, stop = 0, False
    for page_num in range(0, MAX_PAGES):
        if stop: break
        if page_num == 0:
            page_url = BASE_URL
        else:
            page_url = f"{BASE_URL}index_{page_num}.html"
        if page_num > 0:
            time.sleep(1.0)
        try:
            r = session.get(page_url, timeout=30)
            r.encoding = "utf-8"
        except Exception as e:
            print(f"[ERROR] 请求失败: {e}"); time.sleep(2); continue
        if r.status_code != 200:
            print(f"[PAGE] {page_num+1}: HTTP {r.status_code}, 停止"); break
        items = parse_list_page(r.text, page_url)
        if not items:
            print(f"[PAGE] {page_num+1}: 无数据, 停止"); break
        print(f"[PAGE] 第{page_num+1}页: {len(items)}条 [{items[0][1]}] {items[0][0][:35]}")
        for title, date_str, detail_url in items:
            if date_str and date_str < THRESHOLD:
                print(f"  [SKIP] {date_str} 超阈值: {title[:30]}")
                continue
            time.sleep(0.5)
            try:
                rd = session.get(detail_url, timeout=30)
                rd.encoding = "utf-8"
            except:
                print(f"  [!] 请求失败: {detail_url[:60]}"); continue
            if rd.status_code != 200: continue
            dt, dd, dc = parse_detail_page(rd.text, detail_url)
            ft = dt if dt else title
            fd = dd or date_str
            affected = store_item(ft, fd, dc, detail_url)
            if affected > 0:
                total_new += 1
                print(f"  [+] {ft[:35]}...({fd})")
            else:
                print(f"  [-] 已存在")
    print(f"\n=== 完成, 新增: {total_new} ===")

if __name__ == "__main__":
    main()