#!/usr/bin/env python3
"""
crawl_xiaoshan.py — 杭州市萧山区人民政府-建设项目环境影响评价信息公示
CMS: JCMS（建果CMS）
列表: GET /api-gateway/jpaas-publish-server/front/page/build/unit
       关键参数 paramJson={"pageNo":N,"pageSize":"20"}
详情: 标准HTML，标题h2.article-titleH，正文div#zoom.article-content，日期span.article-time
WAF: OpenResty 简单频率限制（~1000/hr）

运行: python3 crawl_xiaoshan.py [--full]
  --full: 全量爬取（24页~467条）
  不加: 仅第1页（日跑增量）
"""

import sys, json, sqlite3, os, re, time, random
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed
import urllib.request
from urllib.parse import urlencode

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "杭州市萧山区人民政府-建设项目环境影响评价信息公示"
SOURCE = "杭州市萧山区人民政府"
BASE_URL = "https://www.xiaoshan.gov.cn"
LIST_API = "/api-gateway/jpaas-publish-server/front/page/build/unit"

BASE_PARAMS = {
    "webId": "2243",
    "pageId": "1229856904",
    "parseType": "bulidstatic",
    "pageType": "column",
    "tagId": "当前栏目列表1a",
    "tplSetId": "GazmxWZitZXDWHJedYwxo",
}

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": "https://www.xiaoshan.gov.cn/col/col1229856904/index.html",
}

MAX_PAGES = 24
NUM_THREADS = 8
THREE_YEARS_AGO = datetime.now() - timedelta(days=3 * 365)
THREE_YEARS_STR = THREE_YEARS_AGO.strftime("%Y-%m-%d")


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def fetch_list_page(page_no):
    """调用JCMS列表API（带paramJson分页参数），返回 [(url, title, date)]"""
    param_json = json.dumps({"pageNo": page_no, "pageSize": "20"}, ensure_ascii=False)
    params = dict(BASE_PARAMS, paramJson=param_json)
    url = BASE_URL + LIST_API + "?" + urlencode(params)

    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=25)
        data = json.loads(resp.read().decode())
    except Exception as e:
        log(f"第{page_no}页API请求失败: {e}")
        return []

    if not data.get("success"):
        log(f"第{page_no}页API返回失败: {data.get('message')}")
        return []

    html = data.get("data", {}).get("html", "")
    if not html:
        return []

    # 解析HTML: <a title="..." href="...">  <b class="zfxxgk_zdgkc_time">2026-01-16</b>
    pattern = re.compile(
        r'<a\s+title="([^"]*)"[^>]*href="([^"]*)"[^>]*>.*?</a>'
        r'.*?<b[^>]*class="zfxxgk_zdgkc_time"[^>]*>\s*(\d{4}-\d{2}-\d{2})\s*</b>',
        re.DOTALL,
    )
    articles = []
    for m in pattern.finditer(html):
        title = m.group(1).strip()
        href = m.group(2).strip()
        date = m.group(3).strip()

        if href.startswith("http"):
            full_url = href
        elif href.startswith("/"):
            full_url = BASE_URL + href
        else:
            full_url = BASE_URL + "/" + href

        articles.append((full_url, title, date))

    if not articles:
        log(f"第{page_no}页: 正则未匹配，HTML前300: {html[:300]}")
    return articles


def fetch_detail(url, list_title, date):
    """爬详情页 -> (url, full_title, date, content)"""
    req = urllib.request.Request(url, headers=HEADERS)
    time.sleep(random.uniform(0.2, 0.5))
    try:
        resp = urllib.request.urlopen(req, timeout=20)
        html = resp.read().decode(errors="replace")
    except Exception as e:
        log(f"详情页失败 {url[-50:]}: {e}")
        return (url, list_title, date, None)

    # 标题: h2.article-titleH
    full_title = list_title
    mt = re.search(r'<h2[^>]*class="article-titleH"[^>]*>(.*?)</h2>', html, re.DOTALL)
    if mt:
        full_title = mt.group(1).strip()

    # 正文: div#zoom.article-content
    content = None
    mc = re.search(
        r'<div[^>]*id="zoom"[^>]*class="article-content"[^>]*>(.*?)</div>\s*(?:<!--|<div|</div>)',
        html, re.DOTALL,
    )
    if mc:
        body = mc.group(1).strip()
        body = re.sub(r'\s+', ' ', body)  # 保留HTML标签
        content = body

    return (url, full_title, date, content)


def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0

    for url, title, date, content in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, SOURCE, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[-50:]}: {e}")

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    is_full = "--full" in sys.argv
    log(f"开始{'全量' if is_full else '增量'}爬取...")

    pages_to_crawl = range(1, MAX_PAGES + 1) if is_full else [1]

    # Stage 1: 列表页
    all_items = []
    for page_no in pages_to_crawl:
        articles = fetch_list_page(page_no)
        if not articles:
            log(f"第{page_no}页无数据，结束")
            break

        filtered = [(u, t, d) for u, t, d in articles if d >= THREE_YEARS_STR]
        log(f"第{page_no}页: {len(articles)}条（近3年: {len(filtered)}条）")
        all_items.extend(filtered)

        if len(filtered) == 0:
            break

        time.sleep(random.uniform(0.3, 0.6))

    if not all_items:
        log("无近3年数据，退出")
        return

    log(f"共 {len(all_items)} 条近3年数据，开始爬详情页...")

    # Stage 2: 并发详情
    detail_results = []
    with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
        futures = {executor.submit(fetch_detail, u, t, d): i for i, (u, t, d) in enumerate(all_items)}
        done, total = 0, len(futures)
        for f in as_completed(futures):
            done += 1
            r = f.result()
            if r:
                detail_results.append(r)
            if done % 20 == 0 or done == total:
                log(f"详情页进度: {done}/{total}")

    # Stage 3: 入库
    log(f"详情页完成，共 {len(detail_results)} 条")

    valid = []
    for url, title, date, content in detail_results:
        valid.append((url, title, date, content if content else ""))

    if valid:
        inserted, skipped = save_to_db(valid)
        log(f"入库完成: 新增 {inserted}, 跳过 {skipped}")
    else:
        log("无数据入库")


if __name__ == "__main__":
    main()