#!/usr/bin/env python3
"""
crawl_kzzq_sthj.py - 科尔沁左翼中旗-生态环境
CMS: 自定义 (内蒙古旗县级 CMS)
列表: /zwgk/sthj1/ (index.html, 0-indexed: index_{N}.html)
详情: /zwgk/sthj1/YYYYMM/tYYYYMMDD_ID.html
内容: div.lis_list_part2_content > p, table
标题: <meta ArticleTitle> (完整)
日期: <meta PubDate>
"""
import requests
import re
import sys
import os

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "科左中旗生态环境"
BASE_URL = "http://www.kzzq.gov.cn"
LIST_DIR = "/zwgk/sthj1"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}


def fetch(url, encoding="utf-8"):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = encoding
        if r.status_code != 200:
            print(f"  WARN: HTTP {r.status_code} for {url}")
            return None
        return r.text
    except Exception as e:
        print(f"  ERROR: {e}")
        return None


def parse_list_page(html):
    """解析列表页，返回(URL, date)列表"""
    items = []
    # 匹配 <li><a href="./202607/...">TITLE</a> <span>DATE</span></li>
    blocks = re.findall(
        r'<li><a href="\./([^"]+)">[^<]*</a>\s*<span>([^<]+)</span></li>',
        html
    )
    
    for rel_path, date_str in blocks:
        url = f"{BASE_URL}{LIST_DIR}/{rel_path}"
        items.append({
            "url": url,
            "date": date_str.strip()
        })
    
    return items


def parse_detail_page(html):
    """解析详情页，返回标题、日期、正文"""
    # 标题
    title = ""
    m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]+)"', html, re.IGNORECASE)
    if m:
        title = m.group(1).strip()
    
    if not title:
        m = re.search(r'<title>([^<]+)</title>', html, re.IGNORECASE)
        if m:
            title = m.group(1).strip()
    
    # 日期
    date = ""
    m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]+)"', html, re.IGNORECASE)
    if m:
        date = m.group(1)[:10]
    
    # 正文
    body = ""
    m = re.search(r'<div class="lis_list_part2_content">(.*?)</div>\s*</div>', html, re.DOTALL)
    if not m:
        m = re.search(r'<div class="lis_list_part2_content">(.*?)</div>', html, re.DOTALL)
    
    if m:
        content_html = m.group(1).strip()
        body = html_to_text(content_html)
    
    return title, date, body


def html_to_text(html):
    """HTML转纯文本"""
    text = html
    
    # 附件链接保留
    text = re.sub(
        r'<a[^>]*href="([^"]+\.(?:pdf|doc|docx|xls|xlsx|zip|rar))"[^>]*>([^<]+)</a>',
        r'[附件：\2](\1)',
        text, flags=re.IGNORECASE
    )
    
    # 图片
    text = re.sub(r'<img[^>]*src="([^"]+)"[^>]*alt="([^"]*)"[^>]*>', r'![\2](\1)', text)
    text = re.sub(r'<img[^>]*src="([^"]+)"[^>]*>', r'![](\1)', text)
    
    # 换行标签
    text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
    for tag in ['</p>', '</div>', '</li>', '</tr>', '</h1>', '</h2>', '</h3>', '</h4>', '</h5>', '</h6>', '</table>']:
        text = re.sub(re.escape(tag), '\n', text, flags=re.IGNORECASE)
    for tag in ['<p', '<div', '<li', '<tr', '<h1', '<h2', '<h3', '<h4', '<h5', '<h6']:
        text = re.sub(rf'{re.escape(tag)}[^>]*>', '\n', text, flags=re.IGNORECASE)
    
    # 移除剩余标签
    text = re.sub(r'<[^>]+>', '', text)
    
    # 实体
    text = text.replace('&nbsp;', ' ').replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"')
    text = text.replace('&ldquo;', '“').replace('&rdquo;', '”').replace('&mdash;', '—').replace('&hellip;', '…')
    
    text = re.sub(r'[ \t]+\n', '\n', text)
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()


def main():
    import argparse
    parser = argparse.ArgumentParser(description=f"{SITE_NAME}爬虫")
    parser.add_argument("--pages", type=int, default=5, help="爬取页数")
    args = parser.parse_args()
    
    print(f"=== {SITE_NAME} ===")
    
    all_items = []
    for page in range(1, args.pages + 1):
        if page == 1:
            url = f"{BASE_URL}{LIST_DIR}/"
        else:
            # 0-indexed: page 2 = index_1.html
            url = f"{BASE_URL}{LIST_DIR}/index_{page-1}.html"
        
        print(f"  列表页 {page}/{args.pages}: {url}")
        html = fetch(url)
        if not html:
            continue
        
        items = parse_list_page(html)
        print(f"    → {len(items)}条")
        all_items.extend(items)
    
    print(f"\n共获取 {len(all_items)} 条列表项")
    
    results = []
    for i, item in enumerate(all_items):
        print(f"  [{i+1}/{len(all_items)}] ", end="")
        
        dtl_html = fetch(item["url"])
        if not dtl_html:
            print("  FAIL (fetch)")
            continue
        
        title, date, body = parse_detail_page(dtl_html)
        if not title:
            print(f"  (no title)")
            continue
        
        print(f"{title[:50]}...")
        
        results.append({
            "title": title,
            "url": item["url"],
            "source_url": item["url"],
            "content": body,
            "pub_date": date or item["date"],
            "site_name": SITE_NAME,
        })
        
        print(f"    正文: {len(body)}字 | 日期: {date or item['date']}")
    
    if not results:
        print("未获取到数据")
        return
    
    print(f"\n入库 {len(results)} 条...")
    push_to_searchdb(results, SITE_NAME)
    print("完成！")


if __name__ == "__main__":
    main()