#!/usr/bin/env python3
"""永丰县人民政府-公共监管 爬虫 (CloudWAF + AJAX API)"""
import re, sys, sqlite3, json
import requests
from bs4 import BeautifulSoup
import os

SITE_NAME = "永丰县-公共监管"
BASE = "http://www.jxyongfeng.gov.cn"
LIST_PAGE = "/xxgk-list-ggjg1.html"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
MAX_PAGES = 5

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": BASE + "/",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}


def get_session():
    s = requests.Session()
    s.headers.update(HEADERS)
    # 先获取列表页建立session
    s.get(BASE + LIST_PAGE, timeout=20)
    return s


def fetch_list(session, page=1):
    """通过AJAX API获取列表."""
    data = {
        "ajax_type[]": ["5_xxgk", "165919", "5", "xxgk", "Y-m-d", "50", "20",
                         ["is_top DESC", "displayorder DESC", "inputtime DESC"], ""],
        "is_ds": "1",
    }
    try:
        r = session.post(f"{BASE}/api-ajax_list-{page}.html", data=data, timeout=30)
        return json.loads(r.text)
    except Exception as e:
        print(f"  ⚠️ API失败 page={page}: {e}")
        return None


def fetch_page(session, url):
    try:
        r = session.get(url, timeout=30)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  ⚠️ 请求失败 {url}: {e}")
        return None


def parse_detail(html):
    soup = BeautifulSoup(html, "lxml")
    # 标题
    title = ""
    h1 = soup.find("h1") or soup.find("h2")
    if h1:
        title = h1.get_text(strip=True)

    # 正文
    content = ""
    for cls in ["article-content", "content", "TRS_Editor", "zoom", "xxgk_content"]:
        div = soup.find("div", class_=cls)
        if div:
            for tag in div(["script", "style", "iframe"]):
                tag.decompose()
            content = str(div)
            break
    if not content:
        best, best_count = None, 0
        for div in soup.find_all("div"):
            ps = div.find_all("p")
            if len(ps) > best_count:
                best, best_count = div, len(ps)
        if best:
            for tag in best(["script", "style", "iframe"]):
                tag.decompose()
            content = str(best)

    content = re.sub(r'\s*style="[^"]*"', "", content)
    content = re.sub(r'\s*class="[^"]*"', "", content)
    return title, content


def run(max_pages=None):
    if max_pages is None:
        max_pages = MAX_PAGES

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    known = set(
        r[0] for r in db.execute(
            "SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)
        ).fetchall()
    )
    db.close()

    session = get_session()

    all_items = []
    for page in range(1, max_pages + 1):
        print(f"  📄 第 {page} 页...", end=" ")
        data = fetch_list(session, page)
        if not data or not data.get("data"):
            print("❌")
            break
        items = data["data"]
        new_items = []
        for item in items:
            url = BASE + "/" + item["url"] if not item["url"].startswith("http") else item["url"]
            if url not in known:
                new_items.append({
                    "title": item["title"],
                    "url": url,
                    "date": item["inputtime"][:10],
                    "summary": item.get("description", "")[:500],
                })
        print(f"✅ {len(items)} 条 (新增 {len(new_items)} / 总{data.get('total',0)})")
        if not new_items and len(items) < 20:
            break
        all_items.extend(new_items)
        if len(data["data"]) < 20:
            break

    if not all_items:
        print("  ⏭ 无新数据")
        return

    print(f"  🔍 获取 {len(all_items)} 条详情...")
    results = []
    for i, item in enumerate(all_items):
        html = fetch_page(session, item["url"])
        if not html:
            results.append({
                "site_name": SITE_NAME,
                "source_url": item["url"][:500],
                "page_url": item["url"],
                "title": item["title"][:500],
                "publish_date": item["date"],
                "summary": item["summary"],
                "content": item["summary"],
                "status": "active",
                "category": "",
                "tags": "",
            })
            continue

        title, content = parse_detail(html)
        if not title:
            title = item["title"]
        results.append({
            "site_name": SITE_NAME,
            "source_url": item["url"][:500],
            "page_url": item["url"],
            "title": title[:500],
            "publish_date": item["date"],
            "summary": item["summary"],
            "content": content or item["summary"],
            "status": "active",
            "category": "",
            "tags": "",
        })
        if (i + 1) % 10 == 0:
            print(f"    [{i+1}/{len(all_items)}]")

    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in results:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                (
                    item["site_name"][:200], item["source_url"], item["page_url"],
                    item["title"], item["publish_date"], item["summary"],
                    item["content"], item["status"], item["category"], item["tags"],
                ),
            )
            if db.total_changes > 0: ok += 1
            else: skip += 1
        except: skip += 1

    db.commit()
    db.execute(
        "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,),
    )
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")


if __name__ == "__main__":
    max_p = int(sys.argv[1]) if len(sys.argv) > 1 and sys.argv[1].isdigit() else None
    run(max_p)