#!/usr/bin/env python3
"""昭通市人民政府-通知公告 爬虫
URL: http://www.zt.gov.cn/lanmu/xwzx/15.html
分页: 15_N.html (N=2..6)
列表: colu-cell > colu-name > a (title by text), colu-desc > a (summary)
详情: deta-bd (正文, 含p标签逻辑段落)
"""

import re
import json
import time
import hashlib
import requests
from datetime import datetime
from bs4 import BeautifulSoup

BASE_URL = "https://www.zt.gov.cn"
LIST_URL = f"{BASE_URL}/lanmu/xwzx/15.html"
SITE_NAME = "昭通市人民政府-通知公告"
TOTAL_PAGES = 6

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}

session = requests.Session()
session.headers.update(HEADERS)


def get_list_url(page):
    if page <= 1:
        return LIST_URL
    return f"{BASE_URL}/lanmu/xwzx/15_{page}.html"


def parse_list_page(html):
    """Parse list page, return list of (url, title, date_str)"""
    items = []
    # colu-cell items
    cell_pattern = re.compile(r'<div class="colu-cell[^"]* wow fadeInUp">(.*?)</div>\s*</div>\s*</div>', re.DOTALL)
    for m in cell_pattern.finditer(html):
        cell = m.group(1)
        # Title from colu-name > a (text, not title attr)
        a_m = re.search(r'<div class="colu-name">\s*<a\s+href="([^"]*)"[^>]*>(.*?)</a>', cell, re.DOTALL)
        if not a_m:
            continue
        href = a_m.group(1).strip()
        title = re.sub(r'<[^>]+>', '', a_m.group(2)).strip()
        # Date - look for YYYY-MM-DD pattern
        date_m = re.search(r'(\d{4}-\d{2}-\d{2})', cell)
        date_str = date_m.group(1).strip() if date_m else ""
        items.append((href, title, date_str))
    return items


def parse_detail(html, url):
    """Parse detail page - extract content from deta-bd"""
    soup = BeautifulSoup(html, "html.parser")
    
    # Title from meta[name=ArticleTitle]
    title_tag = soup.find("meta", attrs={"name": "ArticleTitle"})
    full_title = title_tag.get("content", "").strip() if title_tag else ""
    
    # Date from meta[name=PubDate]
    pub_date = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        pub_date = meta_date["content"].strip()[:10]
    
    # Content from deta-bd
    content_div = soup.find("div", class_="deta-bd")
    
    content_html = ""
    attachments = []
    
    if content_div:
        # Attachments from all page links
        for a_tag in soup.find_all("a"):
            href = a_tag.get("href", "")
            a_text = a_tag.get_text(strip=True)
            if re.search(r'\.(doc|docx|pdf|xls|xlsx|zip|rar)$', href, re.I):
                if not href.startswith("http"):
                    if href.startswith("//"):
                        href = f"https:{href}"
                    elif href.startswith("/"):
                        href = f"{BASE_URL}{href}"
                    else:
                        href = f"{BASE_URL}/{href}"
                if len(a_text) > 2:
                    attachments.append({"title": a_text, "url": href})
                    a_tag.replace_with(f"[{a_text}]({href})")
        
        # Handle tables
        for table in content_div.find_all("table"):
            table_html = str(table)
            table.replace_with(f"\n\n[TABLE]\n{table_html}\n[/TABLE]\n")
        
        # Get paragraphs from <p> tags
        content_parts = []
        for child in content_div.children:
            tag_name = getattr(child, "name", None)
            if tag_name == "p":
                text = child.get_text(" ", strip=True).strip()
                if text:
                    content_parts.append(text)
            elif tag_name == "div":
                text = child.get_text(" ", strip=True).strip()
                if text:
                    content_parts.append(text)
            elif child.name is None:
                text = str(child).strip()
                if text:
                    content_parts.append(text)
        
        if not content_parts:
            content_html = content_div.get_text(" ", strip=True)
        else:
            content_html = "\n\n".join(content_parts)
        
        # Restore tables
        content_html = re.sub(
            r'\[TABLE\]\n(.*?)\n\[/TABLE\]',
            lambda m: f"\n\n{m.group(1)}\n\n",
            content_html, flags=re.DOTALL,
        )
        
        # Filter metadata lines
        metadata_patterns = [
            r"^责任编辑[：:]", r"^初审[：:]", r"^复审[：:]", r"^终审[：:]",
            r"^\[纠错\]", r"^【纠错】", r"^扫一扫在手机打开当前页",
        ]
        lines = content_html.split("\n")
        filtered_lines = []
        for line in lines:
            stripped = line.strip()
            skip = False
            for pat in metadata_patterns:
                if re.match(pat, stripped):
                    skip = True
                    break
            if not skip:
                filtered_lines.append(line)
        content_html = "\n".join(filtered_lines).strip()
        
        # Empty content fallback: use summary from page
        if len(content_html.strip()) < 20:
            # Try summary from meta
            desc_tag = soup.find("meta", attrs={"name": "description"})
            if desc_tag and desc_tag.get("content"):
                content_html = desc_tag["content"].strip()
    
    return {
        "title": full_title,
        "content": content_html,
        "publish_date": pub_date,
        "attachments": json.dumps(attachments, ensure_ascii=False),
        "source_url": url,
    }


def crawl_pages(max_pages, page_start=1, page_end=None):
    if page_end is None:
        page_end = max_pages
    
    results = []
    total_listed = 0
    
    for page in range(page_start, page_end + 1):
        url = get_list_url(page)
        print(f"[列表] 第{page}页: {url}", flush=True)
        
        try:
            resp = session.get(url, timeout=30)
            resp.encoding = "utf-8"
            if resp.status_code != 200:
                print(f"  -> HTTP {resp.status_code}, 跳过", flush=True)
                continue
        except Exception as e:
            print(f"  -> 请求失败: {e}", flush=True)
            time.sleep(2)
            continue
        
        items = parse_list_page(resp.text)
        print(f"  -> 解析到 {len(items)} 条", flush=True)
        total_listed += len(items)
        
        if not items:
            break
        
        for idx, (item_url, list_title, date_str) in enumerate(items, 1):
            print(f"  [{idx}/{len(items)}] {list_title[:60]}...", flush=True)
            try:
                dresp = session.get(item_url, timeout=30)
                dresp.encoding = "utf-8"
                if dresp.status_code != 200:
                    print(f"    -> HTTP {dresp.status_code}", flush=True)
                    continue
            except Exception as e:
                print(f"    -> 详情页请求失败: {e}", flush=True)
                time.sleep(1)
                continue
            
            detail = parse_detail(dresp.text, item_url)
            if detail["title"]:
                list_title = detail["title"]
            detail["list_title"] = list_title
            detail["list_date"] = date_str
            results.append(detail)
            time.sleep(0.3)
        
        time.sleep(1)
    
    print(f"\n总计: 列表 {total_listed} 条, 详情 {len(results)} 条", flush=True)
    return results


def output_jsonl(results, output_path):
    count = 0
    with open(output_path, "w", encoding="utf-8") as f:
        for r in results:
            item_id = hashlib.md5(r["source_url"].encode()).hexdigest()
            record = {
                "id": item_id,
                "title": r["title"] or r["list_title"],
                "site_name": SITE_NAME,
                "source_url": r["source_url"],
                "publish_date": r["publish_date"] or r["list_date"],
                "content": r["content"],
                "attachments": r["attachments"],
                "created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                "updated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
            }
            f.write(json.dumps(record, ensure_ascii=False) + "\n")
            count += 1
    print(f"输出 {count} 条到 {output_path}", flush=True)
    return count


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description=SITE_NAME)
    parser.add_argument("--pages", type=int, default=None, help="页数限制")
    parser.add_argument("--output", type=str, default=None, help="输出文件路径")
    args = parser.parse_args()
    
    max_pages = args.pages or TOTAL_PAGES
    out_path = args.output or f"output/{SITE_NAME.replace('-','_').replace(' ','')}.jsonl"
    
    results = crawl_pages(max_pages)
    if results:
        output_jsonl(results, out_path)
    else:
        print("无结果", flush=True)