#!/usr/bin/env python3
"""
crawl_jiangmen_jsxm.py — 江门市-建设项目环境影响评价
====================================================
URL: https://www.jiangmen.gov.cn/bmpd/jmssthjj/zdlyxxgk/jsxmhjyxpjxx/
CMS: 江门市政府门户网站群, 静态分页 index_N.html
"""

import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests
import urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "江门市-建设项目环评"
BASE_URL = "https://www.jiangmen.gov.cn"
LIST_PATH = "/bmpd/jmssthjj/zdlyxxgk/jsxmhjyxpjxx"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
MAX_PAGES = 5
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None


def parse_list(html):
    """解析列表页，提取标题、URL、日期"""
    items = []
    # <li><a href="http://.../content/post_NNNNNN.html" title="..." target="_blank">标题</a><span>YYYY-MM-DD</span></li>
    pattern = r'<li><a[^>]*href="(http[s]?://[^"]+/content/post_\d+\.html)"[^>]*>(.*?)</a><span>(\d{4}-\d{1,2}-\d{1,2})</span></li>'
    for m in re.finditer(pattern, html, re.DOTALL):
        url = m.group(1).strip()
        title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        date_str = m.group(3).strip()
        items.append((title, url, date_str))
    return items


def extract_nested_content(html, tag_start):
    """从tag_start位置开始，找到匹配的闭合标签，支持嵌套div/section"""
    depth = 0
    i = tag_start
    n = len(html)
    while i < n:
        # Skip HTML comments
        if html[i:i+4] == '<!--':
            end = html.find('-->', i+4)
            if end > i:
                i = end + 3
            else:
                i += 1
            continue
        # Check closing tags
        if html[i:i+6] == '</div>' or html[i:i+8] == '</section>':
            if depth == 0:
                return html[tag_start:i]
            depth -= 1
            i += 6 if html[i:i+6] == '</div>' else 8
            continue
        # Check opening tags
        if i < n and html[i] == '<':
            next_chars = html[i+1:i+6]
            if next_chars in ('div', 'div '):
                depth += 1
                i += 4
                continue
            if next_chars[:4] in ('sect',) and html[i+1:i+8] == 'section':
                depth += 1
                i += 8
                continue
        i += 1
    return html[tag_start:]  # fallback


def fetch_detail(url):
    """获取详情页标题、正文、日期"""
    html = fetch(url)
    if not html:
        return None, None, None

    result = {}

    # 标题: <h1>xxx</h1>（跳过隐藏的 fzsj 日期 h1）
    m = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
    if m:
        title = re.sub(r"<[^>]+>", "", m.group(1)).strip()
        # 如果标题是纯日期格式（YYYY-MM-DD），说明取到了隐藏的 fzsj h1，找下一个
        if re.match(r'^\d{4}-\d{2}-\d{2}$', title):
            html_after = html[m.end():]
            m2 = re.search(r'<h1[^>]*>(.*?)</h1>', html_after, re.DOTALL)
            if m2:
                title = re.sub(r"<[^>]+>", "", m2.group(1)).strip()
        result["title"] = title

    # 日期: <meta name="PubDate" content="YYYY-MM-DD ...">
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        result["publish_date"] = m.group(1)

    # 正文: div.content#zhengwen > div#zoomcon (自动检测多个候选容器)
    # 先清除 script/style 块，避免 JS 字符串内的 </div> 干扰深度计数
    clean_html = re.sub(r"<script[^>]*>.*?</script>", "", html, flags=re.DOTALL | re.I)
    clean_html = re.sub(r"<style[^>]*>.*?</style>", "", clean_html, flags=re.DOTALL | re.I)

    # 优先 #zoomcon（正文内容容器）
    m = re.search(r'<div[^>]*id="zoomcon"[^>]*>', clean_html, re.DOTALL)
    if m:
        content = extract_nested_content(clean_html, m.end())
        content = content.strip()
        if len(content) > 200:
            result["content"] = content
    else:
        # 兜底: 尝试 div.content (id=zhengwen)
        m = re.search(r'<div[^>]*class="content"[^>]*id="zhengwen"[^>]*>', html, re.DOTALL)
        if m:
            content = extract_nested_content(html, m.end())
            content = content.strip()
            if len(content) > 200:
                content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL | re.I)
                content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL | re.I)
                result["content"] = content
            else:
                # 进一步在 content 内找 #zoomcon
                m2 = re.search(r'<div[^>]*id="zoomcon"[^>]*>', content, re.DOTALL)
                if m2:
                    zoom_content = extract_nested_content(content, m2.end())
                    zoom_content = zoom_content.strip()
                    if len(zoom_content) > 100:
                        zoom_content = re.sub(r"<script[^>]*>.*?</script>", "", zoom_content, flags=re.DOTALL | re.I)
                        zoom_content = re.sub(r"<style[^>]*>.*?</style>", "", zoom_content, flags=re.DOTALL | re.I)
                        result["content"] = zoom_content

    return result.get("title"), result.get("content"), result.get("publish_date")


def main():
    print(f"\n{'='*50}")
    print(f"🏠 {SITE_NAME}")
    print(f"{'='*50}")

    all_list_items = []
    for page in range(1, MAX_PAGES + 1):
        if page == 1:
            url = f"{BASE_URL}{LIST_PATH}/"
        else:
            url = f"{BASE_URL}{LIST_PATH}/index_{page}.html"
        print(f"\n📄 第 {page} 页...", end=" ", flush=True)
        html = fetch(url)
        if not html:
            print("❌ 无返回")
            break
        items = parse_list(html)
        if not items:
            print("0 条")
            break
        print(f"✅ {len(items)} 条")
        all_list_items.extend(items)

    print(f"\n📊 列表总计: {len(all_list_items)} 条")

    all_items = []
    seen_urls = set()
    for i, (title, url, date_str) in enumerate(all_list_items):
        if "项目" not in title:
            continue
        if url in seen_urls:
            continue
        seen_urls.add(url)

        print(f"  [{i+1}/{len(all_list_items)}] {title[:50]}...", end=" ", flush=True)
        detail_title, content, detail_date = fetch_detail(url)

        # 合并标题（详情页标题优先）
        final_title = detail_title if detail_title else title
        # 日期：详情页 meta 优先，次之列表页日期
        final_date = detail_date if detail_date else date_str

        # 归一化日期
        if final_date:
            m = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', str(final_date))
            if m:
                y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
                final_date = f"{y:04d}-{mo:02d}-{d:02d}"
            else:
                final_date = ""

        summary = re.sub(r"<[^>]+>", " ", content or "").strip()
        summary = re.sub(r"\s+", " ", summary)[:300]

        all_items.append({
            "site_name": SITE_NAME,
            "title": final_title,
            "url": url,
            "content": content or "",
            "pub_date": final_date,
            "summary": summary,
            "tags": SITE_NAME,
        })
        print(f"✅")
        time.sleep(0.3)

    if not all_items:
        print("⏭️ 无数据")
        return

    push_to_searchdb(all_items, "jiangmen_jsxm")
    print(f"\n✅ 完成! 共 {len(all_items)} 条")


if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")