#!/usr/bin/env python3
"""
常熟经济技术开发区 - 信息公示 (cedz.org)
列表: /part-xinxifabu.html (SSR，分页page_N.html)
详情: /item-{hash}.html (SSR，正文在原始HTML中)
注：详情页是SSR渲染，正文直接在HTML中可提取
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "常熟经济技术开发区-信息公示"
BASE_URL = "https://www.cedz.org"
LIST_URL = "https://www.cedz.org/part-xinxifabu.html"
HEADERS = {"User-Agent": "Mozilla/5.0"}
MAX_PAGES = 5
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def parse_list(html):
    """<li class="work-activity-list-item"><a href="..."><div class="...text...">title</div><div>date</div></a></li>"""
    items = []
    for m in re.finditer(r'<li[^>]*class="work-activity-list-item[^"]*"[^>]*>.*?<a[^>]*href="([^"]+)"[^>]*>.*?<div[^>]*class="[^"]*text[^"]*"[^>]*>(.*?)</div>.*?(?:(\d{4}-\d{2}-\d{2})|(\d{4}/\d{2}/\d{2}))', html, re.DOTALL):
        href = m.group(1)
        title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        date = (m.group(3) or m.group(4) or "").strip()
        if not href.startswith('http'):
            href = BASE_URL + href
        if title and date:
            items.append((title, href, date))
    return items

def fetch_detail(url):
    """提取详情页SSR正文"""
    html = fetch(url)
    if not html:
        return ""
    # 正文在 <div class="py-6 space-y-6"> 中
    m = re.search(r'class="py-6 space-y-6">(.*?)</section>', html, re.DOTALL)
    if not m:
        return ""
    content = '<div class="py-6 space-y-6">' + m.group(1) + '</section>'
    # 清理多余的 style 属性
    content = re.sub(r' style="[^"]*"', '', content)
    # 清理多余的空行
    content = re.sub(r'>\s+<', '><', content)
    return content

def parse_pagination(html):
    """获取总页数"""
    pages = re.findall(r'page=(\d+)', html)
    if pages:
        return max(int(p) for p in pages)
    # 取页面上的最大页码数字
    nums = re.findall(r'>(\d+)</a>', html)
    nums = [int(n) for n in nums if 1 <= int(n) <= 999]
    return max(nums) if nums else 1

def run(args=None):
    incremental = False
    if args and '1' in args:
        incremental = True
    print(f"爬取: {SITE_NAME}")
    results = []
    
    # 先爬第一页获取总页数和数据
    html = fetch(LIST_URL)
    if not html:
        print("  ❌ 无法获取列表页")
        return
    
    total_pages = parse_pagination(html)
    print(f"  总页数: {total_pages}")
    max_pages = 1 if incremental else min(MAX_PAGES, total_pages)
    
    for page in range(1, max_pages + 1):
        if page == 1:
            page_html = html
        else:
            page_url = f"{LIST_URL}?page={page}"
            page_html = fetch(page_url)
            if not page_html or len(page_html) < 500:
                break
        
        items = parse_list(page_html)
        if not items:
            if page > 1: break
            print(f"  第{page}页: 0 条")
            continue
        print(f"  第{page}页: {len(items)} 条")
        
        for i, (title, url, date) in enumerate(items):
            print(f"  [{i+1}/{len(items)}] {title[:40]}...", end=" ", flush=True)
            if date < CUTOFF:
                print("⏭ 超时范围")
                continue
            
            content = fetch_detail(url)
            if not content or len(content) < 50:
                print("⚠ 无正文")
                continue
            
            summary = re.sub(r'<[^>]+>', '', content)[:200].strip()
            results.append({
                "site_name": SITE_NAME, "title": title,
                "url": url, "content": content,
                "summary": summary,
                "pub_date": date,
                "category": "环评公示", "tags": "常熟经开区",
            })
            print(f"✅ ({len(content)}字)")
            time.sleep(0.3)
        
        if incremental:
            break
    
    if results:
        print(f"\n  入库 {len(results)} 条")
        push_to_searchdb(results, "cedz_gs")
    print("完成")

if __name__ == "__main__":
    run(sys.argv[1] if len(sys.argv)>1 else None)