#!/usr/bin/env python3
"""
crawl_hetang.py — 株洲市荷塘区人民政府-通知公告
CMS: ZZCMS（株洲CMS）
列表: 静态HTML分页 /c11731/pages/{N}.html
详情: div.page_name 标题, div.page_time span 日期, div.page_text#page 正文
总: ~100+页, 15条/页 (近3年 ~91页 ~1365条)

运行: python3 crawl_hetang.py [--full]
"""

import sys, sqlite3, os, re, time, random
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed
import urllib.request

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "株洲市荷塘区人民政府-通知公告"
SOURCE = "株洲市荷塘区人民政府"
BASE_URL = "http://www.hetang.gov.cn"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}

MAX_PAGES = 95  # ~page 91 has last 3yr items
NUM_THREADS = 10
THREE_YEARS_AGO = datetime.now() - timedelta(days=3 * 365)
THREE_YEARS_STR = THREE_YEARS_AGO.strftime("%Y-%m-%d")


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def fetch_list_page(page_no):
    """抓取静态列表页，返回 [(url, title, date)]"""
    if page_no == 1:
        url = f"{BASE_URL}/c11731/index.html"
    else:
        url = f"{BASE_URL}/c11731/pages/{page_no}.html"

    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=20)
        html = resp.read().decode(errors="replace")
    except Exception as e:
        log(f"第{page_no}页请求失败: {e}")
        return []

    # 解析: <li><a href="..." title="...">title</a><span>YYYY-MM-DD</span></li>
    articles = []
    pattern = re.compile(
        r'<a\s+href="([^"]*)"[^>]*title="([^"]*)"[^>]*>.*?</a>\s*<span[^>]*>\s*(\d{4}-\d{2}-\d{2})\s*</span>',
        re.DOTALL,
    )
    for m in pattern.finditer(html):
        href = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()

        if href.startswith("http"):
            full_url = href
        elif href.startswith("/"):
            full_url = BASE_URL + href
        else:
            full_url = BASE_URL + "/" + href

        articles.append((full_url, title, date))

    return articles


def fetch_detail(url, list_title, date):
    """爬详情页 -> (url, full_title, date, content)"""
    req = urllib.request.Request(url, headers=HEADERS)
    time.sleep(random.uniform(0.15, 0.3))
    try:
        resp = urllib.request.urlopen(req, timeout=20)
        html = resp.read().decode(errors="replace")
    except Exception as e:
        log(f"详情页失败 {url[-50:]}: {e}")
        return (url, list_title, date, None)

    # 标题: div.page_name
    full_title = list_title
    mt = re.search(r'<div\s+class="page_name"[^>]*>(.*?)</div>', html, re.DOTALL)
    if mt:
        full_title = mt.group(1).strip()
    else:
        # fallback: <title> (strip suffix)
        mt2 = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
        if mt2:
            t = mt2.group(1).strip()
            full_title = re.sub(r'-.*$', '', t).strip()

    # 正文: div.page_text#page
    content = None
    mc = re.search(
        r'<div\s+class="page_text"\s+id="page"[^>]*>(.*?)</div>\s*(?:<!--|<script|<div)',
        html, re.DOTALL,
    )
    if mc:
        body = mc.group(1).strip()
        content = re.sub(r'\s+', ' ', body)

    return (url, full_title, date, content)


def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0

    for url, title, date, content in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, SOURCE, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[-50:]}: {e}")

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    is_full = "--full" in sys.argv
    log(f"开始{'全量' if is_full else '增量'}爬取...")

    pages_to_crawl = range(1, MAX_PAGES + 1) if is_full else [1]

    # Stage 1: 列表页
    all_items = []
    for page_no in pages_to_crawl:
        articles = fetch_list_page(page_no)
        if not articles:
            log(f"第{page_no}页无数据，结束")
            break

        filtered = [(u, t, d) for u, t, d in articles if d >= THREE_YEARS_STR]
        log(f"第{page_no}页: {len(articles)}条（近3年: {len(filtered)}条）")
        all_items.extend(filtered)

        if len(filtered) == 0:
            break

        time.sleep(random.uniform(0.2, 0.4))

    if not all_items:
        log("无近3年数据，退出")
        return

    log(f"共 {len(all_items)} 条近3年数据，开始爬详情页...")

    # Stage 2: 并发详情
    detail_results = []
    with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
        futures = {executor.submit(fetch_detail, u, t, d): i for i, (u, t, d) in enumerate(all_items)}
        done, total = 0, len(futures)
        for f in as_completed(futures):
            done += 1
            r = f.result()
            if r:
                detail_results.append(r)
            if done % 30 == 0 or done == total:
                log(f"详情页进度: {done}/{total}")

    log(f"详情页完成，共 {len(detail_results)} 条")

    valid = [(url, title, date, content if content else "") for url, title, date, content in detail_results]

    if valid:
        inserted, skipped = save_to_db(valid)
        log(f"入库完成: 新增 {inserted}, 跳过 {skipped}")
    else:
        log("无数据入库")


if __name__ == "__main__":
    main()