#!/usr/bin/env python3
"""
蓬莱区政府 - 建设项目环评 爬虫
JPAAS CMS, API分页
https://www.penglai.gov.cn/col/col30455/index.html
"""
import requests
import json
import re
import sys
import os
from datetime import datetime
from urllib.parse import quote

DB_PATH = "/root/search.db"
BASE_URL = "https://www.penglai.gov.cn"
LIST_URL = BASE_URL + "/api-gateway/jpaas-publish-server/front/page/build/unit"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": BASE_URL + "/col/col30455/index.html",
}
SITE_NAME = "蓬莱区建设项目环评"
SITE_URL = "https://www.penglai.gov.cn/col/col30455/index.html"
MAX_PAGES = 5  # 增量日跑只爬最近5页
BASE_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "152",
    "tplSetId": "awU93spIZ5G7CYnk12KGt",
    "pageType": "column",
    "tagId": "当前栏目列表",
    "editType": "null",
    "pageId": "30455",
}


def get_list_page(session, page_no, page_size=25):
    """获取列表页数据"""
    params = BASE_PARAMS.copy()
    params["paramJson"] = json.dumps({"pageNo": page_no, "pageSize": page_size}, ensure_ascii=False)
    r = session.get(LIST_URL, params=params, headers=HEADERS, timeout=30, verify=False)
    r.encoding = "utf-8"
    data = r.json()
    html = data.get("data", {}).get("html", "")
    # parse items
    items = re.findall(r'<li>(.*?)</li>', html, re.DOTALL)
    results = []
    for item in items:
        a = re.search(r'href="([^"]*)"', item)
        title_m = re.search(r'title="([^"]*)"', item)
        date_m = re.search(r'<span>([^<]*)</span>', item)
        if a and title_m:
            href = a.group(1)
            if not href.startswith("http"):
                href = BASE_URL + href
            results.append({
                "title": title_m.group(1).strip(),
                "url": href,
                "date": date_m.group(1).strip() if date_m else "",
            })
    return results


def get_detail(session, url):
    """获取详情页内容"""
    try:
        r = session.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        html = r.text
    except Exception as e:
        return {"content": "", "error": str(e)}

    # title
    title_m = re.search(r'<div class="main_title">(.*?)</div>', html, re.DOTALL)
    title = title_m.group(1).strip() if title_m else ""

    # content
    content_m = re.search(
        r'<div class="main_content">\s*<div id="zoom"[^>]*>(.*?)</div>\s*</div>',
        html, re.DOTALL
    )
    content = content_m.group(1).strip() if content_m else ""

    # clean up
    if content:
        content = re.sub(r'\s+style="[^"]*"', '', content)
        content = re.sub(r'\s+class="[^"]*"', '', content)
        content = content.strip()

    return {"title": title, "content": content, "error": ""}


def save_to_db(conn, items):
    """批量入库"""
    cur = conn.cursor()
    saved = 0
    for item in items:
        title = item.get("title", "")
        url = item.get("url", "")
        content = item.get("content", "")
        pub_date = item.get("date", "")
        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")

        if not content:
            continue

        # 去重检查
        cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (url,))
        if cur.fetchone():
            continue

        # 生成摘要
        summary = re.sub(r'<[^>]+>', '', content)[:200] if content else ""

        cur.execute(
            "INSERT OR IGNORE INTO gov_raw (title, page_url, content, summary, site_name, publish_date, source_url, category) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
            (title, url, content, summary, SITE_NAME, pub_date, url, "环评公示"),
        )

        # 也写入 FTS5 搜索表
        content_plain = re.sub(r'<[^>]+>', '', content).strip()
        cur.execute(
            "INSERT OR IGNORE INTO gov_search_v3 (title, content, source_url, publish_date, site_name) VALUES (?, ?, ?, ?, ?)",
            (title, content_plain, url, pub_date, SITE_NAME),
        )
        saved += 1
    conn.commit()
    return saved


def main():
    import sqlite3
    full_mode = "--full" in sys.argv
    max_pages = 999 if full_mode else MAX_PAGES

    conn = sqlite3.connect(DB_PATH)
    session = requests.Session()

    total_saved = 0
    total_skipped = 0
    total_pages = 0
    page_no = 1

    while page_no <= max_pages:
        print(f"Fetching page {page_no}...")
        items = get_list_page(session, page_no)

        if not items:
            print(f"  No items on page {page_no}, stopping")
            break

        print(f"  Got {len(items)} items")

        saved = 0
        skipped = 0
        for item in items:
            detail = get_detail(session, item["url"])
            if detail.get("error"):
                print(f"  ERROR: {detail['error']}")
                skipped += 1
                continue
            if not detail.get("content"):
                skipped += 1
                continue
            item["content"] = detail["content"]
            saved += 1

        db_saved = save_to_db(conn, [item for item in items if item.get("content")])
        total_saved += db_saved
        total_skipped += skipped
        total_pages += 1

        print(f"  Saved {db_saved} (skipped {skipped})")

        # 增量模式：如果已有已入库记录则停止
        if not full_mode:
            cur = conn.cursor()
            for item in items:
                cur.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
                if cur.fetchone():
                    print(f"  Found existing record, stopping (incremental)")
                    conn.close()
                    print(f"\nDone. Pages: {total_pages}, New: {total_saved}, Skipped: {total_skipped}")
                    return

        page_no += 1

    conn.close()
    print(f"\nDone. Pages: {total_pages}, New: {total_saved}, Skipped: {total_skipped}")


if __name__ == "__main__":
    main()