#!/usr/bin/env python3
"""爬取连云港市人民政府 - 意见征集 (TrueCMS + Playwright)

列表: JSONP API → DOM渲染 → 从页面提取
  getftlist2(page) 函数触发分页
详情: /TrueCMS/visitorController/toViewTheme.do?id=UUID
  正文: div.art-main (id="menu")
  标题: div.art-title
  日期: div.art-menu 中的"发布日期"
"""

import sys, re, os, time, json
from datetime import datetime, timedelta
import sqlite3
from bs4 import BeautifulSoup
from playwright.sync_api import sync_playwright

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "连云港市-意见征集"
BASE_URL = "https://www.lyg.gov.cn"
LIST_URL = BASE_URL + "/zglygzfmhwz/yjzj/yjzjall.html"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
NUM_PER_PAGE = 18

USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"


def extract_list_items(page):
    """从当前页面的DOM中提取列表条目"""
    html = page.content()
    soup = BeautifulSoup(html, "html.parser")

    items = []
    init_div = soup.select_one("#initData")
    if not init_div:
        return [], 0

    for li in init_div.find_all("li", recursive=True):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"]
        if href.startswith("/"):
            href = BASE_URL + href
        title = a.get_text(strip=True)
        # 日期 (float:right的span)
        date_span = li.find("span", style=re.compile(r"float:right"))
        date = date_span.get_text(strip=True) if date_span else ""

        if title and date:
            items.append((href, title, date))

    return items, 0


def fetch_all_list_items(pw_page):
    """翻页获取所有列表条目"""
    print("📋 获取列表数据...", flush=True)

    for attempt in range(3):
        try:
            pw_page.goto(LIST_URL, wait_until="domcontentloaded", timeout=45000)
            time.sleep(4)
            break
        except Exception as e:
            print("  [retry %d] goto: %s" % (attempt + 1, e), flush=True)
            time.sleep(3)
    else:
        print("❌ 无法加载列表页")
        return []

    all_items = []
    page_num = 1

    while True:
        items, _ = extract_list_items(pw_page)
        if not items:
            print("  (空页, 停止)", flush=True)
            break

        last_date = items[-1][2]
        print("  第%d页: %d条, %s ~ %s" % (page_num, len(items),
              items[-1][2], items[0][2]), flush=True)

        if last_date < CUTOFF:
            items = [it for it in items if it[2] >= CUTOFF]
            all_items.extend(items)
            print("  ⏹️  已达3年边界, 停止翻页", flush=True)
            break

        all_items.extend(items)

        # 下一页
        page_num += 1
        try:
            pw_page.evaluate("getftlist2(%d)" % page_num)
            time.sleep(2)
        except Exception as e:
            print("  [翻页失败] page %d: %s" % (page_num, e), flush=True)
            break

    print("  共获取 %d 条\n" % len(all_items), flush=True)
    return all_items


def fetch_detail(pw_page, url):
    """获取详情页内容"""
    for attempt in range(2):
        try:
            pw_page.goto(url, wait_until="domcontentloaded", timeout=30000)
            time.sleep(2)
            break
        except Exception as e:
            print("  [retry] detail goto: %s" % e, flush=True)
            time.sleep(2)
    else:
        return {"title": "", "date": "", "content": ""}

    html = pw_page.content()
    soup = BeautifulSoup(html, "html.parser")

    title = ""
    t = soup.select_one("div.art-title")
    if t:
        title = t.get_text(strip=True)

    date = ""
    menu = soup.select_one("div.art-menu")
    if menu:
        m = re.search(r"(\d{4}-\d{2}-\d{2})", menu.get_text())
        if m:
            date = m.group(1)

    content = ""
    main = soup.select_one("div.art-main")
    if main:
        content = str(main)
        content = re.sub(r'<input[^>]*>', '', content)
        content = content.strip()

    return {"title": title, "date": date, "content": content}


def store(conn, item):
    """插入一条记录"""
    tags = "意见征集"
    try:
        conn.execute(
            """INSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date,
                summary, content, status, category, tags)
               VALUES (?, ?, ?, ?, ?, ?, ?, 'active', ?, ?)""",
            (
                SITE_NAME,
                BASE_URL,
                item["url"],
                item["title"],
                item["date"],
                (item["content"] or "")[:500],
                item["content"] or "",
                "意见征集",
                tags,
            ),
        )
        conn.commit()
        return conn.total_changes > 0
    except Exception as e:
        print("  [ERROR] store: %s" % e, flush=True)
        return False


def main():
    is_inc = "incremental" in sys.argv
    mode = "增量" if is_inc else "全量"
    print("=== %s模式: %s ===\n" % (mode, SITE_NAME))

    conn = sqlite3.connect(DB_PATH, timeout=60)

    pw_play = sync_playwright().start()
    browser = pw_play.chromium.launch(
        headless=True,
        args=["--disable-dev-shm-usage", "--no-sandbox",
              "--disable-gpu"]
    )
    context = browser.new_context(
        user_agent=USER_AGENT,
        viewport={"width": 1920, "height": 1080}
    )
    pw_page = context.new_page()

    try:
        all_items = fetch_all_list_items(pw_page)
        if not all_items:
            print("❌ 未获取到列表数据")
            return

        total_new = 0
        total_skip = 0
        for i, (url, title, date) in enumerate(all_items):
            print("  [%d/%d] %s | %s" % (i + 1, len(all_items), date, title[:50]), flush=True)

            detail = fetch_detail(pw_page, url)
            if not detail["content"]:
                print("    ⚠️  正文为空", flush=True)

            item_data = {
                "url": url,
                "title": detail["title"] or title,
                "date": detail["date"] or date,
                "content": detail["content"],
            }

            if store(conn, item_data):
                total_new += 1
                print("    ✅ 已入库", flush=True)
            else:
                total_skip += 1

            time.sleep(0.5)

        print("\n=== 完成: 新增 %d, 跳过 %d ===" % (total_new, total_skip))

    finally:
        browser.close()
        pw_play.stop()
        conn.close()


if __name__ == "__main__":
    main()