#!/usr/bin/env python3
"""
crawl_qianxi.py — 黔西市-公示公告
CMS: 自定义LayUI CMS（贵州多彩博虹科技）
列表: 静态分页 index.html (p1) / index_{N}.html (pN+1)
详情: h1标题, div.info-time span日期, div#content正文
总: 46页, ~14条/页 (近3年 ~9页 ~120条)

运行: python3 crawl_qianxi.py [--full]
"""

import sys, sqlite3, os, re, time, random
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed
import urllib.request

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "黔西市-公示公告"
SOURCE = "黔西市人民政府"
BASE_URL = "https://www.gzqianxi.gov.cn"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}

MAX_PAGES = 10  # page 9 is partially within 3 years
NUM_THREADS = 8
THREE_YEARS_AGO = datetime.now() - timedelta(days=3 * 365)
THREE_YEARS_STR = THREE_YEARS_AGO.strftime("%Y-%m-%d")


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def fetch_list_page(page_no):
    """0-indexed: page 0=index.html, page 1=index_1.html, etc."""
    if page_no == 0:
        url = f"{BASE_URL}/xxfb/gsgg/index.html"
    else:
        url = f"{BASE_URL}/xxfb/gsgg/index_{page_no}.html"

    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=20)
        html = resp.read().decode(errors="replace")
    except Exception as e:
        log(f"第{page_no+1}页请求失败: {e}")
        return []

    # 解析: <a target="_blank" href="...gsgg/..." title="..." alt="...">
    articles = []
    # Pattern: href with gsgg in URL and title attribute
    pattern = re.compile(
        r'href="([^"]*gsgg/[^"]*)"[^>]*title="([^"]*)"',
        re.DOTALL,
    )
    seen_urls = set()
    for m in pattern.finditer(html):
        href = m.group(1).strip()
        title = m.group(2).strip()

        if href.startswith("http"):
            full_url = href
        elif href.startswith("/"):
            full_url = BASE_URL + href
        else:
            full_url = BASE_URL + "/" + href

        if full_url in seen_urls:
            continue
        seen_urls.add(full_url)

        # Extract date from URL: /202606/t20260611_90516052.html
        date_match = re.search(r'/(\d{6})/t\d{8}_', full_url)
        if date_match:
            ym = date_match.group(1)
            date_str = f"{ym[:4]}-{ym[4:6]}-01"  # approximate month
        else:
            date_str = ""

        articles.append((full_url, title, date_str))

    if not articles:
        log(f"第{page_no+1}页: 未匹配到文章")
    return articles


def fetch_detail(url, list_title, month_date):
    """爬详情页 -> (url, full_title, date, content)"""
    req = urllib.request.Request(url, headers=HEADERS)
    time.sleep(random.uniform(0.15, 0.3))
    try:
        resp = urllib.request.urlopen(req, timeout=20)
        html = resp.read().decode(errors="replace")
    except Exception as e:
        log(f"详情页失败 {url[-50:]}: {e}")
        return (url, list_title, month_date, None)

    # 标题: h1 inside div.articleContent
    full_title = list_title
    mt = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
    if mt:
        full_title = mt.group(1).strip()

    # 日期: div.info-time > span -> "2026-06-11  18:34"
    date_str = month_date
    md = re.search(r'<div class="info-time">\s*<span>([^<]+)</span>', html, re.DOTALL)
    if md:
        raw_date = md.group(1).strip()
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', raw_date)
        if dm:
            date_str = dm.group(1)

    # 正文: div#content
    content = None
    mc = re.search(
        r'<div\s+id="content"[^>]*>(.*?)</div>\s*<!--endprint-->',
        html, re.DOTALL,
    )
    if mc:
        body = mc.group(1).strip()
        content = re.sub(r'\s+', ' ', body)

    return (url, full_title, date_str, content)


def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0

    for url, title, date, content in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, SOURCE, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[-50:]}: {e}")

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    is_full = "--full" in sys.argv
    log(f"开始{'全量' if is_full else '增量'}爬取...")

    pages_to_crawl = range(0, MAX_PAGES) if is_full else [0]

    all_items = []
    for page_no in pages_to_crawl:
        articles = fetch_list_page(page_no)
        if not articles:
            log(f"第{page_no+1}页无数据，结束")
            break

        log(f"第{page_no+1}页: {len(articles)}条")
        all_items.extend(articles)
        time.sleep(random.uniform(0.2, 0.4))

    if not all_items:
        log("无数据，退出")
        return

    log(f"共 {len(all_items)} 条，开始爬详情页...")

    detail_results = []
    with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
        futures = {executor.submit(fetch_detail, u, t, d): i for i, (u, t, d) in enumerate(all_items)}
        done, total = 0, len(futures)
        for f in as_completed(futures):
            done += 1
            r = f.result()
            if r:
                detail_results.append(r)
            if done % 20 == 0 or done == total:
                log(f"详情页进度: {done}/{total}")

    # Filter by actual date from detail page
    valid = []
    for url, title, date, content in detail_results:
        if date < THREE_YEARS_STR:
            continue
        valid.append((url, title, date, content if content else ""))

    log(f"详情页完成: {len(detail_results)}条，近3年: {len(valid)}条")

    if valid:
        inserted, skipped = save_to_db(valid)
        log(f"入库完成: 新增 {inserted}, 跳过 {skipped}")
    else:
        log("无数据入库")


if __name__ == "__main__":
    main()