#!/usr/bin/env python3
"""
河间市-征集调查 (hejian.gov.cn) 爬虫
====================================
列表: /hejian/c100280/listDisplaySon.shtml (createPageHTML 分页)
详情: /hejian/c100280/YYYYMM/uuid.shtml
正文: div.articlecon
CMS: createPageHTML('page_tag',总页数,当前页,'listDisplaySon','shtml',总数)
注意: 第2页 URL 为 listDisplaySon_2.shtml (不是 _1)
"""

import sys, os, re, time, json
from datetime import datetime, timezone, timedelta
import requests
import urllib3
import socket

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "河间市-征集调查"
BASE_URL = "https://hejian.gov.cn"
LIST_PATH = "/hejian/c100280/listDisplaySon.shtml"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
MAX_PAGES = 3  # Crawl all pages (total ~2)

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36",
}

# ── DNS workaround: hejian.gov.cn resolves via Chinese DNS ──
HEJIAN_IP = "111.62.46.38"
_orig_getaddrinfo = socket.getaddrinfo
def _patched_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
    if 'hejian.gov.cn' in host:
        return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (HEJIAN_IP, port))]
    return _orig_getaddrinfo(host, port, family, type, proto, flags)
socket.getaddrinfo = _patched_getaddrinfo


def fetch_page(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  ❌ 请求失败: {e}")
        return None


def parse_list(html):
    """解析列表页，返回 [(title, url, date_str), ...]"""
    items = []
    # <li><span class="date">YYYY-MM-DD</span><a href="...">title</a></li>
    pattern = re.compile(
        r'<li>.*?<span class="date">(\d{4}-\d{2}-\d{2})</span>.*?<a href="([^"]+)"[^>]*>(.*?)</a>',
        re.DOTALL
    )
    for m in pattern.finditer(html):
        date_str = m.group(1).strip()
        href = m.group(2).strip()
        title = re.sub(r"<[^>]+>", "", m.group(3)).strip()
        if not title or not href:
            continue
        # Ensure full URL
        if href.startswith("/"):
            full_url = BASE_URL + href
        elif not href.startswith("http"):
            full_url = BASE_URL + "/" + href
        else:
            full_url = href
        items.append((title, full_url, date_str))
    return items


def get_pagination_info(html):
    """获取总页数，从 createPageHTML('page_tag', N, ...) 提取"""
    m = re.search(r"createPageHTML\s*\(\s*['\"]page_tag['\"]\s*,\s*(\d+)", html)
    if m:
        return int(m.group(1))
    return 1


def fetch_detail(url):
    """获取详情页：返回 (title, content_html, date_str)"""
    html = fetch_page(url)
    if not html:
        return None, None, None

    result = {}

    # 标题: <h1>xxx</h1>
    m = re.search(r"<h1[^>]*>(.*?)</h1>", html, re.DOTALL)
    if m:
        result["title"] = re.sub(r"<[^>]+>", "", m.group(1)).strip()

    # 日期: <meta name="PubDate" content="YYYY-MM-DD ...">
    m = re.search(r'<meta name="PubDate"\s+content="(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        result["publish_date"] = m.group(1)

    # 正文: <div class="articlecon">...</div>
    m = re.search(r'<div class="articlecon">(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
        # 移除尾部分享/按钮区
        content = re.sub(r'<div class="share">.*?</div>', '', content, flags=re.DOTALL)
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL | re.I)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL | re.I)
        result["content"] = content.strip()

    return (
        result.get("title"),
        result.get("content"),
        result.get("publish_date"),
    )


def build_page_url(page_num):
    """构建第N页的URL（从1开始）"""
    if page_num == 1:
        return BASE_URL + LIST_PATH
    else:
        # 注意: 第2页是 listDisplaySon_2.shtml (不是 _1)
        return f"{BASE_URL}/hejian/c100280/listDisplaySon_{page_num}.shtml"


def main():
    max_pages = 5  # crawl all available pages
    if len(sys.argv) > 1 and sys.argv[1].isdigit():
        max_pages = int(sys.argv[1])

    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"{'='*50}")

    # ── Step 1: Determine total pages from page 1 ──
    print(f"  📄 获取总页数...", end=" ", flush=True)
    html_p1 = fetch_page(build_page_url(1))
    if not html_p1:
        print("❌ 首页获取失败")
        return
    total_pages = get_pagination_info(html_p1)
    print(f"共 {total_pages} 页")
    max_pages = min(max_pages, total_pages)

    # ── Step 2: Collect all list items ──
    all_items = []
    for pg in range(1, max_pages + 1):
        url = build_page_url(pg)
        print(f"  📄 第{pg}页...", end=" ", flush=True)
        html = fetch_page(url) if pg > 1 else html_p1
        if not html:
            print("❌")
            break
        items = parse_list(html)
        if not items:
            print("0 条")
            break
        print(f"✅ {len(items)} 条")
        seen = {item[1] for item in all_items}
        for item in items:
            if item[1] not in seen:
                all_items.append(item)
                seen.add(item[1])
        time.sleep(0.3)

    print(f"\n📊 列表总计: {len(all_items)} 条")

    # ── Step 3: Fetch details ──
    results = []
    cutoff = THREE_YEARS_AGO

    for i, (title, url, date_str) in enumerate(all_items):
        if date_str and date_str < cutoff:
            print(f"  [{i+1}/{len(all_items)}] ⏭ 过期 ({date_str})")
            continue

        print(f"  [{i+1}/{len(all_items)}] {title[:50]}...", end=" ", flush=True)
        dt, content, dt2 = fetch_detail(url)
        if not dt:
            dt = title
        final_date = dt2 or date_str

        if not content or len(content) < 30:
            print("⚠ 无正文")
            # Still save with empty content if title exists
            content = ""

        summary = re.sub(r"<[^>]+>", " ", content).strip()
        summary = re.sub(r"\s+", " ", summary)[:300]

        results.append({
            "site_name": SITE_NAME,
            "title": dt,
            "url": url,
            "content": content,
            "pub_date": final_date,
            "summary": summary,
            "category": "政府公告",
            "tags": SITE_NAME,
        })
        print("✅")
        time.sleep(0.2)

        # Batch push every 10
        if len(results) >= 10:
            print(f"  📤 推送 {len(results)} 条至服务器...")
            push_to_searchdb(results, "hejian_zjdc")
            results = []

    if results:
        print(f"  📤 推送 {len(results)} 条至服务器...")
        push_to_searchdb(results, "hejian_zjdc")

    print(f"\n{'='*50}")
    print(f"✅ 完成!")
    print(f"{'='*50}")


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time() - t0:.1f}s\n")