#!/usr/bin/env python3
"""
crawl_jianan_tzgg.py - 许昌市建安区-通知公告
CMS: EWB (电子政务WebBuilder)
列表: /jrja/001006/subpage.html (第1页), /jrja/001006/{N}.html (第N页)
详情: /jrja/001006/YYYYMMDD/{uuid}.html
内容: div.ewb-text-copy > p, table, a[附件]
标题: <a title="..."> (列表页完整)
"""
import requests
import re
import sys
import os

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "许昌市建安区通知公告"
BASE_URL = "http://www.jianan.gov.cn"
LIST_URL = f"{BASE_URL}/jrja/001006/subpage.html"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}


def fetch(url, encoding="utf-8"):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = encoding
        if r.status_code != 200:
            print(f"  WARN: HTTP {r.status_code} for {url}")
            return None
        return r.text
    except Exception as e:
        print(f"  ERROR: {e}")
        return None


def parse_list_page(html):
    """解析列表页，返回条目列表"""
    items = []
    # 匹配 li.ewb-info-item
    blocks = re.findall(
        r'<li class="ewb-info-item[^"]*">\s*<div class="ewb-info-forus">\s*<a href="([^"]+)" title="([^"]*)"[^>]*>.*?</a>\s*</div>\s*<span class="ewb-date">([^<]+)</span>\s*</li>',
        html, re.DOTALL
    )
    
    for href, title, date_str in blocks:
        url = href
        if not url.startswith("http"):
            url = BASE_URL + url
        
        items.append({
            "url": url,
            "title": title.strip(),
            "date": date_str.strip()
        })
    
    return items


def parse_detail_page(html):
    """解析详情页，返回正文"""
    body = ""
    
    # 正文容器 div.ewb-text-copy
    m = re.search(r'<div class="ewb-text-copy">(.*?)</div>\s*</div>', html, re.DOTALL)
    if not m:
        m = re.search(r'<div class="ewb-text-copy">(.*?)</div>', html, re.DOTALL)
    
    if m:
        content_html = m.group(1).strip()
        body = html_to_text(content_html)
    
    return body


def html_to_text(html):
    """HTML转纯文本，保留段落、表格、附件"""
    text = html
    
    # 附件链接保留
    text = re.sub(
        r'<a[^>]*href="([^"]+\.(?:pdf|doc|docx|xls|xlsx))"[^>]*>([^<]+)</a>',
        r'[附件：\2](\1)',
        text, flags=re.IGNORECASE
    )
    
    # 表格转Markdown管道表
    text = re.sub(r'<table[^>]*>.*?<caption[^>]*>(.*?)</caption>.*?</table>', r'\n【表格：\1】\n', text, flags=re.DOTALL|re.IGNORECASE)
    
    # <br> → \n
    text = re.sub(r'<br\s*/?>', '\n', text)
    # </p>, </div>, </li>, </tr> → \n
    for tag in ['</p>', '</div>', '</li>', '</tr>', '</h1>', '</h2>', '</h3>', '</h4>', '</h5>']:
        text = re.sub(re.escape(tag), '\n', text, flags=re.IGNORECASE)
    # 开头标签 → \n
    text = re.sub(r'<(?:p|div|li|tr|h[1-5])[^>]*>', '\n', text, flags=re.IGNORECASE)
    
    # 移除剩余标签
    text = re.sub(r'<[^>]+>', '', text)
    
    # 实体
    text = text.replace('&nbsp;', ' ').replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"').replace('&ldquo;', '“').replace('&rdquo;', '”').replace('&mdash;', '—')
    
    # 清理多余空格
    text = re.sub(r'[ \t]+\n', '\n', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    
    return text.strip()


def main():
    import argparse
    parser = argparse.ArgumentParser(description=f"{SITE_NAME}爬虫")
    parser.add_argument("--pages", type=int, default=5, help="爬取页数")
    args = parser.parse_args()
    
    print(f"=== {SITE_NAME} ===")
    
    all_items = []
    for page in range(1, args.pages + 1):
        if page == 1:
            url = LIST_URL
        else:
            url = f"{BASE_URL}/jrja/001006/{page}.html"
        
        print(f"  列表页 {page}/{args.pages}: {url}")
        html = fetch(url)
        if not html:
            continue
        
        items = parse_list_page(html)
        print(f"    → {len(items)}条")
        all_items.extend(items)
    
    print(f"\n共获取 {len(all_items)} 条列表项")
    
    results = []
    for i, item in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] {item['title'][:50]}...")
        
        dtl_html = fetch(item["url"])
        if not dtl_html:
            continue
        
        body = parse_detail_page(dtl_html)
        
        results.append({
            "title": item["title"],
            "url": item["url"],
            "source_url": item["url"],
            "content": body,
            "pub_date": item["date"],
            "site_name": SITE_NAME,
        })
        
        print(f"    正文: {len(body)}字")
    
    if not results:
        print("未获取到数据")
        return
    
    print(f"\n入库 {len(results)} 条...")
    push_to_searchdb(results, SITE_NAME)
    print("完成！")


if __name__ == "__main__":
    main()