#!/usr/bin/env python3
"""雅安市生态环境局 - 公示公告 (sthjj.yaan.gov.cn)
CMS: 自建CMS (标签式内容发布)
列表: /xinwen/list/{uuid}.html + ?page=N (N=2~8, 共8页)
详情: /xinwen/show/{hash}.html, content=div.nr-rap > div.xqing-web-box

用法:
  python3 crawl_yaan_sthjj.py          # 全量8页
  python3 crawl_yaan_sthjj.py 1        # 增量1页
"""
import re, requests, sqlite3, os, sys, time, json
from datetime import datetime

BASE = "https://sthjj.yaan.gov.cn"
LIST_UUID = "f8afd4bc-28ae-407b-ac92-139d91beb15f"
SITE_NAME = "雅安市生态环境局-公示公告"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TOTAL_PAGES_ALL = 8


def log(msg):
    print(f"  {msg}", flush=True)


def http_get(url):
    try:
        resp = requests.get(url, headers=HEADERS, timeout=30)
        resp.encoding = "utf-8"
        if resp.status_code == 200 and len(resp.text) > 500:
            return resp.text
        log(f"[WARN] {url} → status={resp.status_code}, len={len(resp.text)}")
        return None
    except Exception as e:
        log(f"[ERROR] {url}: {e}")
        return None


def get_list_url(page):
    if page == 1:
        return f"{BASE}/xinwen/list/{LIST_UUID}.html"
    return f"{BASE}/xinwen/list/{LIST_UUID}.html?page={page}"


def parse_list(html):
    """从列表页提取 (title, url, date) 元组。title 优先取 title 属性。"""
    items = []
    # 找所有 target="_blank" 的 article links
    for m in re.finditer(
        r'<a\s+target="_blank"[^>]*title="([^"]*)"\s*href="([^"]*)"[^>]*>.*?</a>\s*<span>([^<]*)</span>',
        html, re.DOTALL
    ):
        title = m.group(1).strip()
        url = m.group(2).strip()
        date = m.group(3).strip()
        if title and url and date:
            if not url.startswith("http"):
                url = BASE + url
            items.append((title, url, date))
    return items


def extract_detail(url, title_from_list):
    """抓取详情页，提取正文HTML、附件"""
    html = http_get(url)
    if not html:
        return None

    # 标题：优先详情页 <title>
    title = title_from_list
    tm = re.search(r'<title>(.*?)\s*-\s*雅安市生态环境局</title>', html)
    if tm:
        t = tm.group(1).strip()
        if t:
            title = t

    # 日期
    date = ""
    dm = re.search(r'(\d{4}-\d{2}-\d{2})', html)
    if dm:
        date = dm.group(1)

    # 正文：div.nr-rap > div.xqing-web-box
    content = ""
    cm = re.search(
        r'<div[^>]*class="nr-rap"[^>]*>\s*<div[^>]*class="xqing-web-box"[^>]*>(.*?)</div>\s*</div>',
        html, re.DOTALL
    )
    if cm:
        raw = cm.group(1)
    else:
        # 备选：直接找 xqing-web-box
        cm2 = re.search(r'<div[^>]*class="xqing-web-box"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
        if cm2:
            raw = cm2.group(1)
        else:
            raw = ""

    if raw:
        # 清理脚本、样式
        raw = re.sub(r'<script[^>]*>.*?</script>', '', raw, flags=re.DOTALL)
        raw = re.sub(r'<link[^>]*>', '', raw, flags=re.DOTALL)
        raw = re.sub(r'<style[^>]*>.*?</style>', '', raw, flags=re.DOTALL)
        raw = raw.strip()
        if raw:
            content = raw

    # 附件：查找 PDF/DOC/DOCX/XLS/XLSX 等
    attachments = []
    for am in re.finditer(
        r'<a[^>]*href="([^"]*\.(pdf|doc|docx|xls|xlsx|zip|rar|7z))"[^>]*>([^<]+)</a>',
        html, re.I
    ):
        att_url = am.group(1)
        att_name = am.group(3).strip()
        if not att_name:
            att_name = f"附件{len(attachments)+1}"
        if att_url.startswith("/"):
            att_url = BASE + att_url
        if not att_url.startswith("http"):
            att_url = BASE + "/" + att_url.lstrip("/")
        attachments.append({"name": att_name, "url": att_url})

    # PDF空内容处理
    text_only = re.sub(r'<[^>]+>', ' ', content).strip()
    text_only = re.sub(r'\s+', ' ', text_only).strip()
    if not text_only and attachments:
        # 正文只有PDF附件，嵌入原文链接
        pdf_links = "\n".join(f"[{a['name']}]({a['url']})" for a in attachments)
        content = f'<p><a href="{url}">{title}</a></p>\n\nPDF附件：\n{pdf_links}'

    return {
        "title": title,
        "date": date,
        "content": content,
        "attachments": attachments,
    }


def crawl_page(page, incremental=False):
    """爬取单页列表，返回 (items, stored, skipped)"""
    list_url = get_list_url(page)
    log(f"列表页 {page}: {list_url}")
    html = http_get(list_url)
    if not html:
        return [], 0, 0

    items = parse_list(html)
    log(f"  发现 {len(items)} 篇文章")

    stored = 0
    skipped = 0
    enriched = []

    for i, (title, url, date) in enumerate(items):
        detail = extract_detail(url, title)
        if not detail:
            skipped += 1
            continue
        enriched.append({
            "title": detail["title"],
            "site_name": SITE_NAME,
            "page_url": url,
            "source_url": url,
            "publish_date": detail["date"] or date,
            "content": detail["content"],
            "summary": (re.sub(r'<[^>]+>', ' ', detail["content"]).strip() or detail["title"])[:500],
            "category": "公示公告",
            "attachments": json.dumps(detail["attachments"], ensure_ascii=False) if detail["attachments"] else "",
        })
        time.sleep(0.3)

    if enriched:
        conn = sqlite3.connect(DB_PATH, timeout=60)
        conn.execute("PRAGMA busy_timeout=10000")
        c = conn.cursor()
        for item in enriched:
            try:
                c.execute(
                    "INSERT OR IGNORE INTO gov_raw "
                    "(site_name, source_url, page_url, title, publish_date, content, summary, category, attachments)"
                    " VALUES (?,?,?,?,?,?,?,?,?)",
                    (item["site_name"], item["source_url"], item["page_url"],
                     item["title"], item["publish_date"], item["content"],
                     item["summary"], item["category"], item["attachments"])
                )
                if c.rowcount > 0:
                    stored += 1
                else:
                    skipped += 1
            except Exception as e:
                log(f"DB error: {e}")
                skipped += 1
        conn.commit()
        conn.close()

    return items, stored, skipped


def main():
    mode = "全量"
    max_pages = TOTAL_PAGES_ALL
    if len(sys.argv) > 1 and sys.argv[1] == "1":
        mode = "增量"
        max_pages = 1

    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME} ({mode})")
    print(f"   DB: {DB_PATH}")
    print(f"{'='*50}\n")

    total_stored = 0
    total_skipped = 0

    for page in range(1, max_pages + 1):
        items, stored, skipped = crawl_page(page)
        total_stored += stored
        total_skipped += skipped

    print(f"\n{'─'*30}")
    print(f"✅ 新增: {total_stored} | 跳过: {total_skipped}")
    print(f"📤 完成！共 {total_stored} 条新数据\n")


if __name__ == "__main__":
    main()