#!/usr/bin/env python3
"""
海西州生态环境局-工作动态 (sthjj.haixi.gov.cn/xwdt/gzdt.htm)
政府网站标准列表：<div class="news_list"> 含拆分日期
详情页：<div class="v_news_content"> 正文
分页：gzdt/142.htm ... gzdt/1.htm（倒序编号）
"""
import sys, os, re, time, html as html_mod
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "海西州生态环境局-工作动态"
BASE_URL = "https://sthjj.haixi.gov.cn"
LIST_PATH = "/xwdt/gzdt.htm"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url):
    for _ in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=20)
            r.encoding = "utf-8"
            if r.text.strip():
                return r.text
        except:
            pass
        time.sleep(2)
    return None

def parse_list(html):
    """提取 (title, url, date_str)"""
    items = []
    for li in re.findall(r'<li>(.*?)</li>', html, re.DOTALL):
        # 日期：拆分格式 <p>2026-06</p><span>08</span>
        date_m = re.search(r'<p>(\d{4}-\d{2})</p>\s*<span>(\d{2})</span>', li)
        if not date_m:
            continue
        date_str = f"{date_m.group(1)}-{date_m.group(2)}"
        # 链接
        a_m = re.search(r'href="([^"]+)"[^>]*>([^<]+)</a>', li)
        if not a_m:
            continue
        href = html_mod.unescape(a_m.group(1)).strip()
        title = html_mod.unescape(a_m.group(2)).strip()
        if title:
            items.append((title, href, date_str))
    return items

def fetch_content(url):
    """从详情页提取 <div class="v_news_content"> 正文"""
    html = fetch(url)
    if not html:
        return ""
    m = re.search(r'<div class="v_news_content">(.*?)</div>', html, re.DOTALL)
    if m:
        return m.group(1).strip()
    return ""

def get_total_page_id(html):
    """从分页导航获取最大 page_id"""
    # 查找页码链接: gzdt/(\d+).htm
    nums = re.findall(r'href="gzdt/(\d+)\.htm"', html)
    max_n = 1
    for n in nums:
        ni = int(n)
        if ni > max_n:
            max_n = ni
    return max_n

def build_page_url(page_id):
    """构建分页 URL"""
    if page_id == 0:
        return BASE_URL + LIST_PATH
    return f"{BASE_URL}/xwdt/gzdt/{page_id}.htm"

def main(incremental=False, limit=None):
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")

    html = fetch(build_page_url(0))
    if not html:
        print("❌ 首页获取失败")
        return

    total_page_id = get_total_page_id(html)
    print(f"📊 最大 page_id: {total_page_id}")

    # 生成页面列表: page 0 (首页) 然后从 total_page_id 递减
    page_ids = [0] + list(range(total_page_id, 0, -1))

    all_items, seen = [], set()
    stop = False

    for page_id in page_ids:
        if stop:
            break
        url = build_page_url(page_id)
        html = fetch(url)
        if not html:
            continue

        items = parse_list(html)
        if not items:
            continue

        # 日期过滤
        page_dates = [d for _, _, d in items]
        if page_dates:
            oldest = min(page_dates)
            if oldest < THREE_YEARS_AGO:
                print(f"  ⏹ 第{page_id}页最早 {oldest}，停止")
                break
            print(f"  📄 第{page_id}页 ({min(page_dates)}~{max(page_dates)})")

        for title, url_path, list_date in items:
            if list_date and list_date < THREE_YEARS_AGO:
                continue

            # 构造完整 URL
            if url_path.startswith('/'):
                full_url = BASE_URL + url_path
            elif url_path.startswith('http'):
                full_url = url_path
            else:
                full_url = BASE_URL + "/" + url_path.lstrip('./')

            if full_url in seen:
                continue
            seen.add(full_url)

            print(f"  [{len(all_items)+1}] {title[:40]}...", end=" ", flush=True)
            content = fetch_content(full_url)

            summary = re.sub(r"<[^>]+>", " ", content).strip()[:300]
            summary = re.sub(r"\s+", " ", summary) if summary else title
            print(f"✅ ({len(content)} chars)")

            all_items.append({
                "site_name": SITE_NAME,
                "title": title,
                "url": full_url,
                "content": content,
                "pub_date": list_date,
                "summary": summary,
                "tags": SITE_NAME,
            })

            time.sleep(0.3)
            if limit and len(all_items) >= limit:
                stop = True
                break

    if all_items:
        push_to_searchdb(all_items, "haixi_gzdt")
        print(f"\n✅ 完成! 共 {len(all_items)} 条")
    else:
        print("\n⏭ 无新数据")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--incremental", action="store_true")
    parser.add_argument("--limit", type=int)
    args = parser.parse_args()
    incremental = args.incremental or (len(sys.argv) > 1 and sys.argv[1] == "1")
    t0 = time.time()
    main(incremental=incremental, limit=args.limit)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")