#!/usr/bin/env python3
"""
crawl_gaoan.py — 高安市人民政府·环境保护备案 + 行政许可
============================================================
CMS: 政府信息公开平台，列表为单文件 JSON（无分页）

列表：{codeName}-articleList.json  →  每次返回全部文章
详情：<div class="article-content"> 正文 | <meta name="ArticleTitle"> /
       <meta name="PubDate"> / <meta name="ContentSource">

两个栏目：
  1. hjbha  — 环境保护备案（50条，2024-08 ~ 2026-05）
  2. xzxkgh — 行政许可      （50条，2026-03 ~ 2026-06）

用法:
    python3 crawl_gaoan.py               # 全量
    python3 crawl_gaoan.py 1             # 增量（仅最新5条）
"""

import os
import re
import sys
import json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed

# ── DB ──────────────────────────────────────────────
DB_PATH = os.environ.get("GOV_DB_PATH", os.path.expanduser(
    "/mnt/data/search.db"
))

# ── Config ──────────────────────────────────────────
BASE_URL = "http://www.gaoan.gov.cn"
SITE_NAME = "高安市人民政府"

CHANNELS = [
    {
        "code": "hjbha",
        "name": "环境保护备案",
        "list_url": "/gasrmzf/hjbha/hjbha-articleList.json",
        "max_articles": 50,
    },
    {
        "code": "xzxkgh",
        "name": "行政许可",
        "list_url": "/gasrmzf/xzxkgh/xzxkgh-articleList.json",
        "max_articles": 50,
    },
]

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
                  "AppleWebKit/537.36 (KHTML, like Gecko) "
                  "Chrome/120.0.0.0 Safari/537.36",
}

MAX_WORKERS = 20  # concurrent detail fetchers


# ── helpers ─────────────────────────────────────────
def log(msg, prefix="gaoan"):
    print(f"[{prefix}] {msg}")


def fetch_article_list(channel):
    """Fetch the full article list JSON for a channel."""
    url = urljoin(BASE_URL, channel["list_url"])
    try:
        resp = requests.get(url, headers=HEADERS, timeout=30)
        resp.encoding = "utf-8"
        data = resp.json()
    except Exception as e:
        log(f"ERROR fetching list for {channel['code']}: {e}", channel["code"])
        return []

    items = []
    for art in data:
        title = art.get("title", "") or art.get("showTitle", "")
        pub_date = art.get("pubDate", "")
        urls_raw = art.get("urls", "{}")
        if isinstance(urls_raw, str):
            try:
                urls_obj = json.loads(urls_raw)
            except json.JSONDecodeError:
                urls_obj = {}
        else:
            urls_obj = urls_raw
        pc_url = urls_obj.get("pc", "")
        if not pc_url:
            continue
        full_url = urljoin(BASE_URL, pc_url)
        items.append((full_url, title, pub_date))

    return items


def fetch_detail(url):
    """Return (content_html, date_str, title) or (None, None, None)."""
    try:
        resp = requests.get(url, headers=HEADERS, timeout=30)
        resp.encoding = "utf-8"
        html = resp.text
    except Exception as e:
        log(f"  ERROR {url}: {e}")
        return None, None, None

    soup = BeautifulSoup(html, "html.parser")

    # Title: <meta name="ArticleTitle">
    meta_title = soup.find("meta", attrs={"name": "ArticleTitle"})
    title = meta_title["content"].strip() if meta_title and meta_title.get("content") else ""

    # Fallback: <div class="article-title">
    if not title:
        at = soup.find("div", class_=lambda c: c and "article-title" in (c if isinstance(c, str) else " ".join(c)))
        if at:
            title = at.get_text(strip=True)

    # Date: <meta name="PubDate">
    date_str = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        m = re.match(r"(\d{4}-\d{2}-\d{2})", meta_date["content"].strip())
        if m:
            date_str = m.group(1)

    # Content: <div class="article-content">
    content_div = soup.find("div", class_=lambda c: c and "article-content" in (c if isinstance(c, str) else " ".join(c)))
    content_html = str(content_div) if content_div else ""

    if not content_html:
        log(f"  WARNING: no content for {url}")
        return None, date_str, title

    return content_html, date_str, title


def fetch_detail_batch(urls):
    results = {}
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex:
        fut_map = {ex.submit(fetch_detail, url): url for url in urls}
        for fut in as_completed(fut_map):
            url = fut_map[fut]
            try:
                results[url] = fut.result()
            except Exception as e:
                results[url] = (None, None, None)
    return results


def save_to_db(items):
    """items: list of (url, title, date_str, content_html, category)"""
    import sqlite3
    if not items:
        return 0
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted = 0
    for url, title, date_str, content_html, category in items:
        try:
            c.execute(
                """INSERT OR REPLACE INTO gov_raw (site_name, page_url, title, publish_date, content, category, script_name) VALUES (?, ?, ?, ?, ?, ?, 'crawl_gaoan.py')""",
                (SITE_NAME, url, title, date_str, content_html, category),
            )
            inserted += 1
        except Exception as e:
            log(f"  DB error for {url}: {e}")
    conn.commit()
    conn.close()
    return inserted


def run_channel(channel, incremental):
    """Process one channel, return results list."""
    prefix = channel["code"]
    log(f"--- {channel['name']} ({channel['code']}) ---", prefix)

    articles = fetch_article_list(channel)
    log(f"List contains {len(articles)} articles total", prefix)

    if not articles:
        log("No articles found", prefix)
        return []

    three_years_ago = (datetime.now() - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
    log(f"3-year cutoff: {three_years_ago}", prefix)

    # Filter by date
    filtered = []
    for url, title, pub_date in articles:
        d = pub_date[:10] if pub_date else ""
        if d and d >= three_years_ago:
            filtered.append((url, title, d))
        elif not d:
            # No date, include it
            filtered.append((url, title, d))

    if incremental:
        # Only take the most recent 5 articles
        filtered = filtered[:5]
        log(f"Incremental: {len(filtered)} articles", prefix)
    else:
        log(f"After date filter: {len(filtered)} articles", prefix)

    # Phase 2: Fetch details
    results = []
    batch_size = MAX_WORKERS * 3
    for batch_start in range(0, len(filtered), batch_size):
        batch = filtered[batch_start:batch_start + batch_size]
        batch_urls = [u for u, _, _ in batch]
        log(f"  Details [{batch_start+1}-{batch_start+len(batch)}/{len(filtered)}]...", prefix)
        detail_map = fetch_detail_batch(batch_urls)
        for url, title, date_str in batch:
            content_html, detail_date, detail_title = detail_map.get(url, (None, None, None))
            final_title = detail_title or title
            final_date = detail_date or date_str
            category = channel["name"]
            results.append((url, final_title, final_date, content_html or "", category))

    return results


def main():
    incremental = len(sys.argv) > 1 and sys.argv[1] == "1"

    all_results = []
    for channel in CHANNELS:
        results = run_channel(channel, incremental)
        all_results.extend(results)

    # Save
    saved = save_to_db(all_results)

    log(f"\n{'=' * 50}")
    log(f"Done. Total: {len(all_results)} articles (saved: {saved})")

    # Per-channel summary
    for ch in CHANNELS:
        cat = ch["name"]
        ch_results = [r for r in all_results if r[4] == cat]
        dates = sorted([r[2] for r in ch_results if r[2]])
        date_range = f"{dates[0]} ~ {dates[-1]}" if dates else "N/A"
        log(f"  {cat}: {len(ch_results)} articles, {date_range}")

    # FTS rebuild
    try:
        import sqlite3
        conn = sqlite3.connect(DB_PATH, timeout=60)
        for ch in CHANNELS:
            cat = ch["name"]
            conn.execute("""
                INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
                SELECT rowid, title, site_name,
                       CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
                FROM gov_raw WHERE category = ?
            """, (cat,))
        conn.commit()
        conn.close()
        log("FTS index rebuilt")
    except Exception as e:
        log(f"FTS rebuild note: {e}")


if __name__ == "__main__":
    main()