#!/usr/bin/env python3
"""
赣州经开区-环境保护 (gzjkq.ganzhou.gov.cn)
列表: /jkqxxgk/c109966/xxgk_list.shtml
分页: index_N.shtml 模式
详情: /jkqxxgk/c109966/YYYYMM/{hash}.shtml
正文: div.xxgk-content 或 .xxgk-container
日期: <meta name="PubDate">
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "赣州经开区-环境保护"
BASE_URL = "https://gzjkq.ganzhou.gov.cn"
LIST_PATH = "/jkqxxgk/c109966/xxgk_list.shtml"
LIST_DIR = "/jkqxxgk/c109966"
MAX_PAGES = 5
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def parse_list(html):
    """
    提取列表条目: (title, url, date_str)
    
    常见 gov 列表模式:
      <li><a href="..." title="标题">标题</a><span>YYYY-MM-DD</span></li>
      <li><span>YYYY-MM-DD</span><a href="...">标题</a></li>
      <div class="..."><a href="...">标题</a><span>YYYY-MM-DD</span></div>
    """
    items = []
    
    # 模式1: <li> 中有 <a title="..."> 和 <span> 日期
    pattern1 = r'<li[^>]*>.*?<a[^>]*href="([^"]+)"[^>]*title="([^"]*)"[^>]*>.*?</a>.*?<span[^>]*>(\d{4}-\d{2}-\d{2})</span>.*?</li>'
    for m in re.finditer(pattern1, html, re.DOTALL):
        href, title, date_str = m.group(1), m.group(2).strip(), m.group(3)
        if title and href:
            items.append((title, href, date_str))
    
    # 模式2: <li> 中 <span>日期</span> 在 <a> 之前
    if not items:
        pattern2 = r'<li[^>]*>.*?<span[^>]*>(\d{4}-\d{2}-\d{2})</span>\s*<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>.*?</li>'
        for m in re.finditer(pattern2, html, re.DOTALL):
            date_str, href, title = m.group(1), m.group(2), re.sub(r'<[^>]+>', '', m.group(3)).strip()
            if title and href:
                items.append((title, href, date_str))
    
    # 模式3: 通用 <a href="...">title</a> 在列表容器中，日期在附近
    if not items:
        pattern3 = r'<a[^>]*href="(/jkqxxgk/c109966/\d{6}/[a-f0-9]+\.shtml)"[^>]*>(.*?)</a>'
        for m in re.finditer(pattern3, html, re.DOTALL):
            href = m.group(1)
            title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
            if not title or len(title) < 4:
                continue
            # 尝试从附近提取日期
            end_pos = m.end()
            nearby = html[end_pos:end_pos+500]
            date_m = re.search(r'(\d{4}-\d{2}-\d{2})', nearby)
            date_str = date_m.group(1) if date_m else ""
            items.append((title, href, date_str))
    
    # 规范化 URL
    result = []
    for title, href, date_str in items:
        if not href.startswith('http'):
            href = BASE_URL + href
        result.append((title, href, date_str))
    return result

def get_page_url(page):
    """构建分页 URL"""
    if page == 1:
        return BASE_URL + LIST_PATH
    # 尝试 index_N.shtml 模式 (最常见)
    return f"{BASE_URL}{LIST_DIR}/index_{page}.shtml"

def fetch_detail(url):
    """获取详情页标题、正文、发布日期"""
    html = fetch(url)
    if not html:
        return None, None, None
    
    result = {}
    
    # 标题: meta ArticleTitle 或 h1
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]+)"', html)
    if m:
        result["title"] = m.group(1).strip()
    if not result.get("title"):
        m = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
        if m:
            result["title"] = re.sub(r"<[^>]+>", "", m.group(1)).strip()
    if not result.get("title"):
        m = re.search(r"<title>(.*?)</title>", html, re.DOTALL)
        if m:
            result["title"] = m.group(1).strip()
    
    # 日期: meta PubDate 优先
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        result["publish_date"] = m.group(1)
    if not result.get("publish_date"):
        # 发布日期：YYYY-MM-DD
        m = re.search(r"发布日期[：:]\s*(\d{4}-\d{1,2}-\d{1,2})", html)
        if m:
            result["publish_date"] = m.group(1)
    if not result.get("publish_date"):
        # 发布时间：YYYY-MM-DD
        m = re.search(r"发布时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})", html)
        if m:
            result["publish_date"] = m.group(1)
    if not result.get("publish_date"):
        # 通用日期提取
        m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
        if m:
            result["publish_date"] = m.group(1)
    
    # 正文: div.xxgk-content 或 .xxgk-container
    content = None
    
    # 方法1: div.xxgk-content
    m = re.search(r'<div[^>]*class="[^"]*xxgk-content[^"]*"[^>]*>(.*?)</div>\s*(?:<div|<!--|$)', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    
    # 方法2: 在 xxgk-container 内
    if not content or len(content) < 50:
        m = re.search(r'class="[^"]*xxgk-container[^"]*"[^>]*>(.*?)</div>\s*(?:</div>|<!--|$)', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
            # 如果 xxgk-container 只是外层包装，尝试找内部正文
            inner = re.search(r'<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>', content, re.DOTALL)
            if inner and len(inner.group(1)) > 50:
                content = inner.group(1).strip()
    
    # 方法3: div#Zoom (常见 gov 正文容器)
    if not content or len(content) < 50:
        m = re.search(r'<div[^>]*id="Zoom"[^>]*>(.*?)</div>\s*(?:<div|<!--|$)', html, re.DOTALL)
        if m:
            content = m.group(1).strip()
    
    # 方法4: 通用内容选择器
    if not content or len(content) < 50:
        for cls in ['xxgk_content', 'article-content', 'article_content', 'detail-content', 'main-content', 'content']:
            m = re.search(rf'<div[^>]*class="[^"]*{cls}[^"]*"[^>]*>(.*?)</div>\s*(?:<div|<!--|$)', html, re.DOTALL)
            if m and len(m.group(1)) > 100:
                content = m.group(1).strip()
                break
    
    # 清洗
    if content:
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL | re.I)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL | re.I)
        result["content"] = content
    
    return result.get("title"), result.get("content"), result.get("publish_date")

def main(incremental=False):
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")
    if incremental:
        print("  🔄 增量模式：仅爬第1页")
    
    all_list = []
    for page in range(1, MAX_PAGES + 1):
        url = get_page_url(page)
        print(f"\n📄 第 {page} 页...", end=" ", flush=True)
        html = fetch(url)
        if not html:
            print("❌")
            if page == 1:
                print("  ⚠ 首页失败，停止")
            break
        
        items = parse_list(html)
        if not items:
            print("0 条")
            if page > 1:
                break
            continue
        
        print(f"✅ {len(items)} 条")
        all_list.extend(items)
        
        if incremental:
            break
    
    print(f"\n📊 列表总计: {len(all_list)} 条")
    
    all_items, seen = [], set()
    for i, (title, url, list_date) in enumerate(all_list):
        # 只取标题含"项目"的
        if "项目" not in title:
            continue
        if url in seen:
            continue
        seen.add(url)
        
        print(f"  [{i+1}/{len(all_list)}] {title[:50]}...", end=" ", flush=True)
        dt, content, date = fetch_detail(url)
        if dt:
            title = dt
        final_date = date or list_date
        
        summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
        summary = re.sub(r"\s+", " ", summary)
        
        all_items.append({
            "site_name": SITE_NAME,
            "title": title,
            "url": url,
            "content": content or "",
            "pub_date": final_date,
            "summary": summary,
            "tags": SITE_NAME,
        })
        print("✅")
        time.sleep(0.3)
    
    if all_items:
        push_to_searchdb(all_items, "ganzhou_jkq")
    else:
        print("  ⏭ 无数据，跳过推送")
    
    print(f"\n✅ 完成! 共 {len(all_items)} 条")

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description=f"爬虫: {SITE_NAME}")
    parser.add_argument("--incremental", action="store_true", help="增量模式：仅爬第1页")
    parser.add_argument("--limit", type=int, help="测试：只处理N条详情")
    args = parser.parse_args()
    
    t0 = time.time()
    # 也支持旧的命令行参数方式：python3 script.py 1 表示增量
    incremental = args.incremental or (len(sys.argv) > 1 and sys.argv[1] == '1')
    main(incremental=incremental)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")