#!/usr/bin/env python3
"""
crawl_yq.py — 阳泉市人民政府-通知公告
CMS: TRS WCM
列表: 静态分页 index_N.shtml (<ul class="main-listcon-items_box">)
详情: requests + <div class="article-body oflow-hd"> → TRS_UEDITOR
     标题从详情页 <h2> 提取完整标题（列表页title被截断含...）
WAF: 无

运行: python3 crawl_yq.py [--full]
  --full: 全量 (所有6页)
  不加: 仅第1页 (日跑增量)
"""

import sys, re, time, random, sqlite3, os, urllib.request, urllib.error
import html as htmlmod
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timedelta

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "阳泉市人民政府-通知公告"
SOURCE = "阳泉市人民政府"

BASE_URL = "https://www.yq.gov.cn"
LIST_BASE = f"{BASE_URL}/zwgk/gsgg/"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}

MAX_PAGES = 6
NUM_THREADS = 5
REQUEST_DELAY = (0.3, 0.5)

THREE_YEARS_AGO = datetime.now() - timedelta(days=3 * 365)


def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def fetch_list(page_no):
    if page_no == 0:
        url = f"{LIST_BASE}index.shtml"
    else:
        url = f"{LIST_BASE}index_{page_no}.shtml"
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=15)
        return resp.read().decode(errors="replace")
    except Exception as e:
        log(f"列表页 {page_no} 请求失败: {e}")
        return None


def parse_list(html_text):
    items = []
    m = re.search(r'<ul class="main-listcon-items_box"[^>]*>(.*?)</ul>\s*</div>', html_text, re.DOTALL)
    if not m:
        return items
    ul_content = m.group(1)

    for li in re.finditer(r'<li[^>]*class="atitle"[^>]*>(.*?)</li>', ul_content, re.DOTALL):
        li_text = li.group(1)
        a_m = re.search(r'<a\s+href="([^"]+)"[^>]*title="([^"]*)"', li_text)
        if not a_m:
            continue
        url_path = a_m.group(1)
        title = htmlmod.unescape(a_m.group(2).strip())

        date_m = re.search(r'<span[^>]*>(\d{4}-\d{2}-\d{2})</span>', li_text)
        if not date_m:
            continue
        pub_date = date_m.group(1)

        if url_path.startswith("./"):
            url_path = url_path[2:]
        full_url = f"{BASE_URL}/zwgk/gsgg/{url_path}" if not url_path.startswith("http") else url_path

        items.append((full_url, title, pub_date))
    return items


def extract_full_title(html):
    """从详情页提取完整标题（<h2> 或 <meta ArticleTitle>），去除 <br/> 换行"""
    # 优先 <meta name="ArticleTitle">
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]+)"', html, re.IGNORECASE)
    if m:
        return htmlmod.unescape(m.group(1).strip())
    # 其次 <h2>
    m = re.search(r'<h2>(.*?)</h2>', html, re.DOTALL)
    if m:
        t = m.group(1)
        t = re.sub(r'<br\s*/?>', '', t)  # 去 <br/> 换行
        t = re.sub(r'\s+', ' ', t).strip()
        return htmlmod.unescape(t)
    return None


def fetch_detail(url, list_title, date):
    """爬详情页，返回 (url, full_title, date, content)。
    full_title 优先使用详情页 <h2> 的完整标题，避免列表页截断。"""
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=15)
        html = resp.read().decode(errors="replace")
    except Exception as e:
        return (url, list_title, date, None)

    # 从详情页提取完整标题
    full_title = extract_full_title(html)
    if not full_title:
        full_title = list_title  # fallback 到列表页标题

    # 内容: <div class="article-body oflow-hd">
    content = None
    m = re.search(r'<div[^>]*class="article-body[^"]*oflow-hd"[^>]*>(.*?)</div>\s*(?:<!--|</div>|\n)', html, re.DOTALL)
    if m:
        body_html = m.group(1)
        m2 = re.search(r'<div[^>]*class="[^"]*TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>\s*(?:</div>|<!--)', body_html, re.DOTALL)
        if m2:
            content = m2.group(1).strip()
        else:
            content = body_html.strip()
    else:
        m = re.search(r'<div[^>]*class="[^"]*TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>\s*(?:</div>|<!--)', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
        else:
            for cls in ["TRS_Editor", "neirong", "content"]:
                m = re.search(rf'<div[^>]*class="[^"]*{cls}[^"]*"[^>]*>(.*?)</div>', html, re.DOTALL)
                if m:
                    content = m.group(1).strip()
                    break

    if content:
        content = re.sub(r'\s+', ' ', content).strip()
    else:
        content = ""

    return (url, full_title, date, content)


def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0

    for url, title, date, content in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, SOURCE, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[:50]}... {e}")

    conn.commit()
    conn.close()
    return inserted, skipped


def main():
    is_full = "--full" in sys.argv
    log(f"开始{'全量' if is_full else '增量'}爬取...")

    all_items = []
    pages_to_crawl = range(0, MAX_PAGES) if is_full else [0]

    for page_no in pages_to_crawl:
        time.sleep(random.uniform(*REQUEST_DELAY))
        html_text = fetch_list(page_no)
        if html_text is None:
            log(f"第 {page_no + 1} 页跳过")
            continue
        items = parse_list(html_text)
        if not items:
            log(f"第 {page_no + 1} 页为空，结束")
            break
        log(f"第 {page_no + 1} 页: {len(items)} 条")

        filtered = []
        for item in items:
            try:
                dt = datetime.strptime(item[2], "%Y-%m-%d")
                if dt >= THREE_YEARS_AGO:
                    filtered.append(item)
            except:
                filtered.append(item)
        all_items.extend(filtered)

        if len(items) < 15:
            break

    log(f"共获取 {len(all_items)} 条列表数据（近3年）")

    if not all_items:
        log("没有数据，退出")
        return

    log(f"开始多线程爬取详情 ({NUM_THREADS}线程)...")
    results = []
    with ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
        futures = {executor.submit(fetch_detail, url, t, d): i for i, (url, t, d) in enumerate(all_items)}
        done = 0
        total = len(futures)
        for f in as_completed(futures):
            done += 1
            try:
                results.append(f.result())
            except Exception:
                idx = futures[f]
                url, t, d = all_items[idx]
                results.append((url, t, d, None))
            if done % 50 == 0 or done == total:
                log(f"详情进度: {done}/{total}")

    inserted, skipped = save_to_db(results)
    log(f"入库完成: 新增 {inserted}, 跳过 {skipped}")


if __name__ == "__main__":
    main()