#!/usr/bin/env python3
"""
crawl_ynzk.py — 镇雄县-环境保护 (ynzk.gov.cn)
CMS: 镇雄县信息公开平台（自建）
列表: 静态HTML分页 /zfxxgk_zkx_sthjj/hjbh.html (首页) / .../index_2~5.html
      <dd class="even"><a ... href=".../artview/263/XXXXX.html" title="TITLE">TITLE</a><span>YYYY-MM-DD</span></dd>
详情: <meta ArticleTitle> / <meta PubDate> / <div class="xxgk-articleBox">正文HTML</div>
页数: 5页, ~15条/页, 共约75条
注意: SSL证书使用国密曲线，需Python自定义SSL context (curl的OpenSSL 3.x无法兼容)
"""

import ssl, urllib.request, re, sqlite3, json, time, random, os, sys
from datetime import datetime, timedelta
from urllib.parse import urljoin

DB_PATH = os.environ.get("SEARCH_DB", "/root/search.db")
SITE_NAME = "镇雄县-环境保护"
SOURCE = "镇雄县人民政府"
BASE_URL = "https://ynzk.gov.cn/zfxxgk_zkx_sthjj/hjbh.html"
LIST_BASE = "https://ynzk.gov.cn/zfxxgk_zkx_sthjj/hjbh"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
CUTOFF = (datetime.now() - timedelta(days=3 * 365)).strftime("%Y-%m-%d")
MAX_PAGES = 6

# SSL context to bypass SM2 EC curve issue
CTX = ssl.create_default_context()
CTX.minimum_version = ssl.TLSVersion.TLSv1_2
CTX.set_ciphers("AES128-GCM-SHA256:AES256-GCM-SHA384")
CTX.check_hostname = False
CTX.verify_mode = ssl.CERT_NONE

seen_urls = set()

def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)

def fetch(url):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=25, context=CTX)
        return resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        log(f"请求失败: {url[-60:]}: {e}")
        return None

def extract_list(html):
    items = []
    dds = re.findall(r'<dd[^>]*>(.*?)</dd>', html, re.DOTALL)
    for dd in dds:
        m = re.search(r'href="(/zfxxgk_zkx_sthjj/artview/\d+/\d+\.html)".*?title="([^"]*)"', dd, re.DOTALL)
        m2 = re.search(r'<span>(\d{4}-\d{2}-\d{2})</span>', dd)
        if m and m2:
            rel_url = m.group(1)
            title = m.group(2).strip()
            date_str = m2.group(1).strip()
            full_url = "https://ynzk.gov.cn" + rel_url if rel_url.startswith("/") else urljoin(LIST_BASE, rel_url)
            if full_url in seen_urls:
                continue
            seen_urls.add(full_url)
            items.append((full_url, title, date_str))
    return items

def extract_detail(html):
    title = ""
    content = ""
    pub_date = ""
    m = re.search(r'ArticleTitle"\s*content="([^"]+)"', html)
    if m:
        title = m.group(1).strip()
    m = re.search(r'PubDate"\s*content="([^"]+)"', html)
    if m:
        pub_date = m.group(1).strip()
    m = re.search(r'class="xxgk-articleBox">(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    return title, content, pub_date

def save_to_db(articles):
    conn = sqlite3.connect(DB_PATH, timeout=60)
    cursor = conn.cursor()
    inserted = 0
    skipped = 0
    for url, title, date, content in articles:
        try:
            cursor.execute(
                """INSERT OR IGNORE INTO gov_raw
                   (site_name, title, page_url, publish_date, source_url, content)
                   VALUES (?, ?, ?, ?, ?, ?)""",
                (SITE_NAME, title, url, date, SOURCE, content or ""),
            )
            if cursor.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            log(f"入库失败: {url[-50:]}: {e}")
    conn.commit()
    conn.close()
    return inserted, skipped

def main():
    pages = [BASE_URL] + [f"{LIST_BASE}/index_{i}.html" for i in range(2, MAX_PAGES + 1)]
    total_articles = []

    for page_url in pages:
        log(f"取列表: {page_url}")
        html = fetch(page_url)
        if not html:
            continue
        items = extract_list(html)
        if not items:
            log(f"  无内容，停止分页")
            break
        log(f"  共{len(items)}条")

        for url, title, date_str in items:
            if date_str < CUTOFF:
                log(f"  跳过(旧): {date_str} {title[:40]}")
                continue

            time.sleep(random.uniform(0.3, 0.8))
            detail_html = fetch(url)
            if not detail_html:
                continue
            d_title, content, d_date = extract_detail(detail_html)
            if not d_title:
                d_title = title
            if not d_date:
                d_date = date_str
            if not content:
                log(f"  无正文: {d_title[:40]}")
                continue

            total_articles.append((url, d_title, d_date, content))

        time.sleep(0.3)

    inserted, skipped = save_to_db(total_articles)
    log(f"完成！新增{inserted}条，跳过{skipped}条（总计{len(total_articles)}条）")

if __name__ == "__main__":
    main()