#!/usr/bin/env python3
"""北仑区-建设项目环境影响评价 爬虫


JPAAS API: /api-gateway/jpaas-publish-server/front/page/build/unit
"""
import requests
import re
import os
import json
from bs4 import BeautifulSoup
import warnings
import sys as _SYS
# 支持 --pages 参数
import argparse as _AP
_AP_PARSER = _AP.ArgumentParser()
_AP_PARSER.add_argument("--pages", type=int, default=0, help="限制页数")
_AP_ARGS, _ = _AP_PARSER.parse_known_args()
_MAX_PG = _AP_ARGS.pages if _AP_ARGS.pages > 0 else None
if _MAX_PG:
    print(f'[AutoPg] max_pages={_MAX_PG}')
# END AUTO PAGES
warnings.filterwarnings("ignore")

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "北仑区-建设项目环境影响评价"
BASE_URL = "https://www.bl.gov.cn"
API_URL = f"{BASE_URL}/api-gateway/jpaas-publish-server/front/page/build/unit"
API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "3499",
    "tplSetId": "iIEzq5wa84hjU6xuCoM3q",
    "pageType": "column",
    "tagId": "政务公开信息列表",
    "editType": "null",
    "pageId": "1229505487",
}
TOTAL_COUNT = 2199
PAGE_SIZE = 15
TOTAL_PAGES = (TOTAL_COUNT + PAGE_SIZE - 1) // PAGE_SIZE  # 147

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": f"{BASE_URL}/col/col1229505487/index.html",
}


def fetch_api(page_no):
    """调用JPAAS API获取列表页HTML"""
    params = {**API_PARAMS}
    params["paramJson"] = json.dumps({"pageNo": page_no, "pageSize": PAGE_SIZE}, ensure_ascii=False)
    try:
        r = requests.get(API_URL, params=params, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        data = r.json()
        if data.get("success") and data.get("data", {}).get("html"):
            return data["data"]["html"]
        return None
    except Exception as e:
        print(f"  ❌ API失败: {e}")
        return None


def fetch_detail(url):
    """获取详情页"""
    try:
        if not url.startswith("http"):
            url = BASE_URL + url
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        return None


def parse_list_from_html(html):
    """从API返回的HTML解析列表"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    # Items are in <li> directly inside the container
    for li in soup.find_all("li"):
        a = li.find("a")
        if not a:
            continue
        href = a.get("href", "")
        title = a.get("title", "") or a.get_text(strip=True)
        if not href or not title:
            continue
        detail_url = BASE_URL + href if href.startswith("/") else href
        # Date from <span>[2026-06-24]</span>
        span = li.find("span")
        date_str = span.get_text(strip=True).strip("[]") if span else ""
        items.append((title, detail_url, date_str))
    return items


def parse_detail(html):
    """解析详情页"""
    soup = BeautifulSoup(html, "html.parser")

    # Title from meta
    title = ""
    meta = soup.find("meta", attrs={"name": "ArticleTitle"})
    if meta and meta.get("content"):
        title = meta["content"]

    # Date from meta
    publish_date = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        m = re.search(r'(\d{4}-\d{2}-\d{2})', meta_date["content"])
        if m:
            publish_date = m.group(1)

    # Content from <div id="zoom">
    content = ""
    zoom_div = soup.find("div", id="zoom")
    if zoom_div:
        content = str(zoom_div)

    return title, content, publish_date


def crawl():
    conn = None
    try:
        import sqlite3
        conn = sqlite3.connect(SEARCH_DB, timeout=60)
        c = conn.cursor()
        c.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT,
            content TEXT,
            source_url TEXT UNIQUE,
            site_name TEXT,
            publish_date TEXT
        )''')

        total_new = 0
        total_skipped = 0

        for page in range(1, min(TOTAL_PAGES, _MAX_PG or TOTAL_PAGES)+1):
            html = fetch_api(page)
            if not html:
                print(f"第{page}/{TOTAL_PAGES}页: API无数据")
                continue

            items = parse_list_from_html(html)
            if not items:
                print(f"第{page}/{TOTAL_PAGES}页: 无数据，结束分页")
                break

            print(f"第{page}/{TOTAL_PAGES}页: {len(items)}条")

            for i, (title, detail_url, date_from_list) in enumerate(items, 1):
                c.execute("SELECT id FROM gov_raw WHERE source_url = ?", (detail_url,))
                if c.fetchone():
                    total_skipped += 1
                    continue

                detail_html = fetch_detail(detail_url)
                if not detail_html:
                    print(f"  [{i}/{len(items)}] ❌ 详情失败: {title[:30]}...")
                    continue

                _, content, publish_date = parse_detail(detail_html)
                if not publish_date:
                    publish_date = date_from_list

                try:
                    c.execute(
                        "INSERT OR IGNORE INTO gov_raw (title, content, source_url, site_name, publish_date) VALUES (?, ?, ?, ?, ?)",
                        (title, content, detail_url, SITE_NAME, publish_date or "")
                    )
                    conn.commit()
                    total_new += 1
                    content_len = len(content) if content else 0
                    status = " ⚠️ 内容过短" if content_len < 200 else ""
                    if total_new % 50 == 0:
                        print(f"  [page{page}] 已入库 {total_new} 条...")
                    # Print title for first item per page
                    if i == 1:
                        print(f"  [{i}/{len(items)}] ✅ {title[:35]}... ({publish_date}){status}")
                except Exception as e:
                    print(f"  [{i}/{len(items)}] ❌ 入库失败: {e}")

        print(f"\n爬取完成！新增: {total_new}, 跳过(已存在): {total_skipped}")
        c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
        total = c.fetchone()[0]
        print(f"  共 {total} 条")

    finally:
        if conn:
            conn.close()


if __name__ == "__main__":
    crawl()