#!/usr/bin/env python3
"""
杭州市生态环境局 - 2026建设项目环境影响评价公示 爬虫
CMS: JCMS (Java CMS)
列表: API GET + paramJson 分页
详情: requests + <meta ArticleTitle> + <div class="pageFile-Content">
WAF: 阿里云 (需浏览器 UA)

运行: python3 crawl_hangzhou.py [--full]
  --full: 全量爬取 (所有46页)
  不加参数: 仅爬第1页 (增量模式，日跑用)
"""

import json, re, time, random, sys, sqlite3, os, urllib.request, urllib.parse, urllib.error
import html as htmlmod
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "杭州市生态环境局-2026建设项目环境影响评价公示"
SOURCE = "杭州市生态环境局"

BASE_URL = "https://epb.hangzhou.gov.cn"
API_URL = f"{BASE_URL}/api-gateway/jpaas-publish-server/front/page/build/unit"

API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "3249",
    "tplSetId": "pWtD8p2JcI0v1rw5TsyBq",
    "pageType": "column",
    "tagId": "当前栏目list",
    "editType": "null",
    "pageId": "S5UyzHsYHfVcPzifFrjDT",
}

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}

PAGE_SIZE = 20
MAX_PAGES = 46
REQUEST_DELAY = (0.3, 0.5)
NUM_THREADS = 5


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def fetch_list(page_no):
    params = dict(API_PARAMS)
    params["paramJson"] = json.dumps({"pageNo": page_no, "pageSize": PAGE_SIZE, "search": ""}, ensure_ascii=False)

    url = API_URL + "?" + "&".join(
        f"{k}={urllib.parse.quote(str(v))}" for k, v in params.items()
    )

    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=10)
        data = json.loads(resp.read().decode())
        if not data.get("success"):
            return []
        return data["data"]["html"]
    except Exception as e:
        log(f"列表API请求失败 (page {page_no}): {e}")
        return None


def parse_list(html_text):
    items = []
    pattern = re.compile(r"<li>(.*?)</li>", re.DOTALL)
    for li in pattern.findall(html_text):
        a_match = re.search(r'<a href="([^"]+)"[^>]*title="([^"]*)"', li)
        if not a_match:
            continue
        url_path = a_match.group(1)
        title = htmlmod.unescape(a_match.group(2).strip())
        if "/art/" not in url_path:
            continue
        date_match = re.search(r">(\d{4}-\d{2}-\d{2})<", li)
        date = date_match.group(1) if date_match else ""
        full_url = url_path if url_path.startswith("http") else BASE_URL + url_path
        items.append((full_url, title, date))
    return items


def fetch_detail(url, title, date):
    """爬取单篇详情 (用于线程池)"""
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=10)
        html = resp.read().decode()
    except Exception as e:
        return (url, title, date, None)

    # 提取标题
    final_title = title
    m = re.search(r'<meta[^>]*ArticleTitle["\s][^>]*content="([^"]*)"', html)
    if m:
        final_title = htmlmod.unescape(m.group(1).strip())

    # 提取正文 (.pageFile-Content)
    content = None
    m = re.search(r'<div class="pageFile-Content">(.*?)</div>\s*</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    else:
        # fallback: TRS_Editor
        for cls in ["TRS_Editor", "neirong"]:
            m = re.search(rf'<div[^>]*class="[^"]*{cls}[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
            if m:
                content = m.group(1).strip()
                break

    if content:
        content = re.sub(r'\s+', ' ', content).strip()

    return (url, final_title, date, content)


def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0

    for url, title, date, content in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, SOURCE, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[:50]}... {e}")

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    is_full = "--full" in sys.argv
    log(f"开始{'全量' if is_full else '增量'}爬取...")

    # 第一阶段: 收集列表
    all_items = []
    pages_to_crawl = range(1, MAX_PAGES + 1) if is_full else [1]

    for page_no in pages_to_crawl:
        time.sleep(random.uniform(*REQUEST_DELAY))
        html_text = fetch_list(page_no)
        if html_text is None:
            log(f"第 {page_no} 页跳过")
            continue
        items = parse_list(html_text)
        if not items:
            log(f"第 {page_no} 页为空，结束")
            break
        log(f"第 {page_no} 页: {len(items)} 条")
        all_items.extend(items)

    log(f"共获取 {len(all_items)} 条列表数据")

    if not all_items:
        log("没有数据，退出")
        return

    # 第二阶段: 多线程爬详情
    log(f"开始多线程爬取详情 ({NUM_THREADS}线程)...")
    results = []
    with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
        futures = {executor.submit(fetch_detail, url, t, d): (url, t, d) for url, t, d in all_items}
        done = 0
        total = len(futures)
        for f in as_completed(futures):
            done += 1
            try:
                results.append(f.result())
            except Exception as e:
                url, t, d = futures[f]
                results.append((url, t, d, None))
            if done % 50 == 0 or done == total:
                log(f"详情进度: {done}/{total}")

    # 第三阶段: 入库
    inserted, skipped = save_to_db(results)
    log(f"入库完成: 新增 {inserted}, 跳过 {skipped}")


if __name__ == "__main__":
    main()