#!/usr/bin/env python3
"""
木垒哈萨克自治县人民政府 - 生态环境局 - 生态环境
站点: https://www.mlx.gov.cn/p212/wrfzsthjj.html
API: GET /content/page?channelIds=7781&page=N&size=15&orderBy=27
数据量: 176条，全量爬取
正文: API响应的txt字段（已含完整HTML）
"""

import sys
import json
import time
import re
import os
import sqlite3
from datetime import datetime, timedelta
from urllib.parse import urljoin

import requests

BASE_URL = "https://www.mlx.gov.cn"
API_URL = f"{BASE_URL}/content/page"
CHANNEL_IDS = "7781"
PAGE_SIZE = 15
ORDER_BY = 27

SITE_NAME = "木垒县-生态环境"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept": "application/json, text/plain, */*",
    "Referer": f"{BASE_URL}/p212/wrfzsthjj.html",
}

# 仅保留最近3年的数据
CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")


def fetch_page(page: int) -> dict | None:
    """获取列表页API"""
    params = {
        "channelIds": CHANNEL_IDS,
        "page": page,
        "size": PAGE_SIZE,
        "orderBy": ORDER_BY,
    }
    try:
        resp = requests.get(API_URL, params=params, headers=HEADERS, timeout=30)
        resp.raise_for_status()
        return resp.json()
    except Exception as e:
        print(f"  [ERROR] 第{page}页请求失败: {e}", flush=True)
        return None


def extract_attachments(html_content: str, base: str = BASE_URL) -> str:
    """从HTML正文中提取附件链接，补全为可点击URL"""
    if not html_content:
        return ""

    def fix_url(m):
        href = m.group(1)
        if href.startswith("http"):
            return m.group(0)
        full_url = urljoin(base, href)
        return f'href="{full_url}"'

    html_content = re.sub(r'href="([^"]+)"', fix_url, html_content)
    html_content = re.sub(r'src="([^"]+)"', fix_url, html_content)
    return html_content


def get_summary(html_content: str) -> str:
    """从HTML正文提取纯文本摘要（前200字）"""
    if not html_content:
        return ""
    text = re.sub(r'<[^>]+>', '', html_content)
    text = re.sub(r'\s+', ' ', text).strip()
    return text[:200]


def crawl():
    """主爬取流程"""
    print(f"[INFO] 木垒县爬虫启动 | API: {API_URL}", flush=True)
    print(f"[INFO] 截止日期: {CUTOFF_DATE} (仅保留最近3年)", flush=True)

    data = fetch_page(1)
    if not data or data.get("code") != 200:
        print("[ERROR] 无法获取数据，退出", flush=True)
        return

    total_elements = data["data"]["totalElements"]
    total_pages = (total_elements + PAGE_SIZE - 1) // PAGE_SIZE
    print(f"[INFO] 共 {total_elements} 条，{total_pages} 页", flush=True)

    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()

    total_new = 0
    total_skip = 0
    total_old = 0
    threshold_date = CUTOFF_DATE

    for page in range(1, total_pages + 1):
        if page > 1:
            data = fetch_page(page)

        if not data or data.get("code") != 200:
            print(f"  [WARN] 第{page}页数据无效，跳过", flush=True)
            continue

        items = data["data"]["content"]
        if not items:
            print(f"  [INFO] 第{page}页无数据，结束", flush=True)
            break

        print(f"  [PAGE {page}/{total_pages}] {len(items)} 条", flush=True)

        page_all_before_cutoff = True
        for item in items:
            release_date = (item.get("releaseTime") or "")[:10]

            if release_date and release_date < threshold_date:
                total_skip += 1
                continue

            page_all_before_cutoff = False

            title = (item.get("title") or "").strip()
            page_url = (item.get("urlWhole") or "").strip()
            content_html = extract_attachments(item.get("txt") or "")
            summary = get_summary(content_html)

            if not title or not page_url:
                continue

            try:
                cur.execute(
                    "INSERT OR IGNORE INTO gov_raw (title, page_url, site_name, summary, content, publish_date) "
                    "VALUES (?, ?, ?, ?, ?, ?)",
                    (title, page_url, SITE_NAME, summary, content_html, release_date)
                )
                if cur.rowcount > 0:
                    total_new += 1
                else:
                    total_old += 1
            except Exception as e:
                print(f"    [ERROR] 入库失败: {title[:30]} - {e}", flush=True)

        if page_all_before_cutoff and page > 1:
            print(f"  [INFO] 第{page}页全部早于截止日期，提前结束", flush=True)
            break

        time.sleep(0.3)

    conn.commit()
    conn.close()

    print(f"\n[DONE] 木垒县爬取完成", flush=True)
    print(f"  新增: {total_new} 条", flush=True)
    print(f"  已存在: {total_old} 条", flush=True)
    print(f"  跳过(日期超限): {total_skip} 条", flush=True)


def incremental():
    """增量爬取：仅爬第一页"""
    print(f"[增量] 木垒县增量爬取启动", flush=True)

    data = fetch_page(1)
    if not data or data.get("code") != 200:
        print("[ERROR] 增量: API请求失败", flush=True)
        return

    items = data["data"]["content"]
    print(f"  最新 {len(items)} 条", flush=True)

    conn = sqlite3.connect(DB_PATH)
    cur = conn.cursor()

    new_count = 0
    for item in items:
        title = (item.get("title") or "").strip()
        page_url = (item.get("urlWhole") or "").strip()
        release_date = (item.get("releaseTime") or "")[:10]
        content_html = extract_attachments(item.get("txt") or "")
        summary = get_summary(content_html)

        if not title or not page_url:
            continue

        try:
            cur.execute(
                "INSERT OR IGNORE INTO gov_raw (title, page_url, site_name, summary, content, publish_date) "
                "VALUES (?, ?, ?, ?, ?, ?)",
                (title, page_url, SITE_NAME, summary, content_html, release_date)
            )
            if cur.rowcount > 0:
                new_count += 1
        except Exception as e:
            print(f"    [ERROR] 入库失败: {title[:30]} - {e}", flush=True)

    conn.commit()
    conn.close()
    print(f"  新增入库: {new_count} 条", flush=True)


if __name__ == "__main__":
    mode = sys.argv[1] if len(sys.argv) > 1 else "full"
    if mode == "incremental":
        incremental()
    else:
        crawl()