#!/usr/bin/env python3
"""克什克腾旗人民政府-通知公告 爬虫"""
import re, sys, sqlite3
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import os

SITE_NAME = "克什克腾旗-通知公告"
BASE = "http://www.kskt.gov.cn"
LIST = "/xwzx/tzgg/"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 5  # 新站点策略：最多5页

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": BASE + "/",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}


def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  ⚠️ 请求失败 {url}: {e}")
        return None


def parse_list(html):
    soup = BeautifulSoup(html, "lxml")
    items = []
    ul = soup.find("ul", id="lb")
    if not ul:
        return items
    for li in ul.find_all("li", class_="clearfix"):
        a = li.find("a")
        span = li.find("span")
        if not a:
            continue
        href = a.get("href", "")
        title = a.get("title") or a.get_text(strip=True)
        date = span.get_text(strip=True) if span else ""
        if href:
            full_url = urljoin(BASE + LIST, href)
            items.append({"title": title, "url": full_url, "date": date})
    return items


def parse_detail(html):
    soup = BeautifulSoup(html, "lxml")
    # 标题
    h1 = soup.find("h1") or soup.find("h2") or soup.find("title")
    title = h1.get_text(strip=True) if h1 else ""
    if "克什克腾旗人民政府" in title:
        title = title.split("_")[0].strip() if "_" in title else title

    # 日期 - meta或页面内
    date = ""
    for meta in soup.find_all("meta"):
        name = meta.get("name", "").lower()
        if name in ("publish_date", "pubdate", "articletitledate"):
            date = meta.get("content", "")
            break
    if not date:
        m = re.search(r"(\d{4}[-/]\d{1,2}[-/]\d{1,2})", str(soup))
        if m:
            date = m.group(1)

    # 正文
    content = ""
    for cls in ["article-content", "TRS_Editor", "news_content", "content", "zoom", "article"]:
        div = soup.find("div", class_=cls)
        if div:
            for tag in div(["script", "style", "iframe"]):
                tag.decompose()
            content = str(div)
            break
    if not content:
        # fallback: find div with most p tags
        best, best_count = None, 0
        for div in soup.find_all("div"):
            ps = div.find_all("p")
            if len(ps) > best_count:
                best, best_count = div, len(ps)
        if best:
            for tag in best(["script", "style", "iframe"]):
                tag.decompose()
            content = str(best)

    # 清理属性
    content = re.sub(r'\s*style="[^"]*"', "", content)
    content = re.sub(r'\s*class="[^"]*"', "", content)
    return title, date, content


def get_total_pages(html):
    soup = BeautifulSoup(html, "lxml")
    fenye = soup.find("div", class_="fenye")
    if fenye:
        nums = re.findall(r"/(\d+)", fenye.get_text())
        if nums:
            return int(nums[-1])
    return 1


def run(max_pages=None):
    if max_pages is None:
        max_pages = MAX_PAGES

    # 获取已索引URL
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    known = set(
        r[0] for r in db.execute(
            "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)
        ).fetchall()
    )
    db.close()

    all_items = []
    total_pages = None

    for page in range(1, max_pages + 1):
        if page == 1:
            url = BASE + LIST
        else:
            url = f"{BASE}{LIST}index_{page}.html"
        print(f"  📄 第 {page} 页...", end=" ")
        html = fetch(url)
        if not html:
            print("❌")
            break
        if total_pages is None:
            total_pages = get_total_pages(html)

        items = parse_list(html)
        new_items = []
        for item in items:
            if item["url"] not in known:
                new_items.append(item)
        print(f"✅ {len(items)} 条 (新增 {len(new_items)})")
        all_items.extend(new_items)

        if page >= total_pages:
            break

    if not all_items:
        print("  ⏭ 无新数据")
        return

    print(f"  🔍 获取 {len(all_items)} 条详情...")
    results = []
    for i, item in enumerate(all_items):
        html = fetch(item["url"])
        if not html:
            continue
        title, date_str, content = parse_detail(html)
        if not title:
            title = item["title"]
        if not date_str:
            date_str = item["date"]
        results.append({
            "site_name": SITE_NAME,
            "source_url": item["url"][:500],
            "page_url": item["url"],
            "title": (title or item["title"])[:500],
            "publish_date": date_str[:10] if date_str else "",
            "summary": (title or item["title"])[:500],
            "content": content,
            "status": "active",
            "category": "",
            "tags": "",
        })
        if (i + 1) % 10 == 0:
            print(f"    [{i+1}/{len(all_items)}]")

    # 入库
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in results:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                (
                    item["site_name"][:200], item["source_url"], item["page_url"],
                    item["title"], item["publish_date"], item["summary"],
                    item["content"], item["status"], item["category"], item["tags"],
                ),
            )
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1

    db.commit()
    db.execute(
        "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,),
    )
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")


if __name__ == "__main__":
    max_p = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else None
    run(max_p)