#!/usr/bin/env python3
"""
韶关市南雄市水务局政府信息公开平台 - 爬虫
CMS: Laravel + Vue SPA (省级政府信息公开平台)
列表: /sgnxshuiwj/gkmlpt/api/all/{catId}?page=N&sid={siteId} (JSON, 100条/页)
详情: requests + <div class="article-content">
WAF: 无

运行: python3 crawl_nanxiong.py [--full]
  --full: 全量爬取 (所有3页，含日期过滤)
  不加: 仅爬第1页 (增量模式，日跑用)
"""

import sys, re, time, random, sqlite3, os, urllib.request, urllib.error, json
import html as htmlmod
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "韶关市南雄市水务局-政府信息公开"
SOURCE = "韶关市南雄市水务局"

BASE_URL = "https://www.gdnx.gov.cn"
API_URL = f"{BASE_URL}/sgnxshuiwj/gkmlpt/api/all/1653"
SITE_ID = "751222"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}

MAX_PAGES = 3
NUM_THREADS = 5
REQUEST_DELAY = (0.3, 0.5)
THREE_YEARS_AGO = (datetime.now() - timedelta(days=365*3)).timestamp()


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def fetch_list(page_no):
    url = f"{API_URL}?page={page_no}&sid={SITE_ID}"
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=15)
        data = json.loads(resp.read().decode())
        return data.get("articles", [])
    except Exception as e:
        log(f"列表页 {page_no} 请求失败: {e}")
        return None


def parse_articles(articles, is_full):
    """解析API返回的articles，过滤3年"""
    items = []
    for art in articles:
        pub_date = art.get("date") or art.get("first_publish_time") or 0
        if isinstance(pub_date, str):
            pub_date = int(pub_date)
        if pub_date > 1e12:  # 秒级时间戳
            pub_date = pub_date  # already seconds
        else:
            pub_date = pub_date  # assume seconds

        # 3年截点
        if is_full and pub_date and pub_date < THREE_YEARS_AGO:
            continue

        # 日期转 YYYY-MM-DD
        date_str = ""
        if pub_date:
            try:
                date_str = datetime.fromtimestamp(pub_date).strftime("%Y-%m-%d")
            except:
                date_str = str(pub_date)[:10]

        url_path = art.get("url", "")
        full_url = url_path if url_path.startswith("http") else BASE_URL + url_path
        title = htmlmod.unescape(art.get("title", "").strip())

        items.append((full_url, title, date_str, art.get("publisher", SOURCE)))
    return items


def fetch_detail(url, title, date, source):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=15)
        html_text = resp.read().decode()
    except Exception as e:
        return (url, title, date, None, source)

    # 内容: <div class="article-content">
    content = None
    m = re.search(r'<div class="article-content[^"]*"[^>]*>(.*?)</div>\s*<div', html_text, re.DOTALL)
    if m:
        content = m.group(1).strip()
    else:
        # simpler match
        m = re.search(r'<div class="article-content[^"]*"[^>]*>(.*?)</div>', html_text, re.DOTALL)
        if m:
            content = m.group(1).strip()

    # Extract title from page if available
    final_title = title
    m = re.search(r"<title>(.*?)</title>", html_text)
    if m:
        t = htmlmod.unescape(m.group(1).strip())
        # Remove site suffix
        t = re.sub(r'_全椒县人民政府|_南雄市.*', '', t).strip()
        if t:
            final_title = t

    if content:
        content = re.sub(r'\s+', ' ', content).strip()

    return (url, final_title, date, content, source)


def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0

    for url, title, date, content, source in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, source, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[:50]}... {e}")

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    is_full = "--full" in sys.argv
    log(f"开始{'全量' if is_full else '增量'}爬取...")

    all_items = []
    pages_to_crawl = range(1, MAX_PAGES + 1) if is_full else [1]

    for page_no in pages_to_crawl:
        time.sleep(random.uniform(*REQUEST_DELAY))
        articles = fetch_list(page_no)
        if articles is None:
            log(f"第 {page_no} 页跳过")
            continue
        if not articles:
            log(f"第 {page_no} 页为空，结束")
            break
        items = parse_articles(articles, is_full)
        log(f"第 {page_no} 页: {len(articles)} 条, 3年过滤后 {len(items)} 条")
        all_items.extend(items)

    log(f"共获取 {len(all_items)} 条列表数据")

    if not all_items:
        log("没有数据，退出")
        return

    # 第二阶段: 多线程详情
    log(f"开始多线程爬取详情 ({NUM_THREADS}线程)...")
    results = []
    with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
        futures = {executor.submit(fetch_detail, url, t, d, s): i for i, (url, t, d, s) in enumerate(all_items)}
        done = 0
        total = len(futures)
        for f in as_completed(futures):
            done += 1
            try:
                results.append(f.result())
            except Exception:
                idx = futures[f]
                url, t, d, s = all_items[idx]
                results.append((url, t, d, None, s))
            if done % 30 == 0 or done == total:
                log(f"详情进度: {done}/{total}")

    # 第三阶段: 入库
    inserted, skipped = save_to_db(results)
    log(f"入库完成: 新增 {inserted}, 跳过 {skipped}")


if __name__ == "__main__":
    main()