#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
crawl_kmfgw_pzjg.py - 昆明市发改委-批准结果信息
CMS: 昆明市政府信息公开平台 (ZCMS)
列表: /zfxxgklm/fdzdgknr/zdlyxxgk/zdjsxmpzhssly/pzjgxx/index.shtml (page1)
      /.../pzjgxx/index_{N}.shtml (page N)
详情: /c/YYYY-MM-DD/ID.shtml
内容: div.activity > p, table
标题: div.content (完整)
日期: div.activity-info
"""
import requests
import re
import sys
import os

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
import urllib.parse

SITE_NAME = "昆明市发改委-批准结果信息"
BASE_URL = "https://fgw.km.gov.cn"
LIST_DIR = "/zfxxgklm/fdzdgknr/zdlyxxgk/zdjsxmpzhssly/pzjgxx"
LIST_PAGE1 = f"{BASE_URL}{LIST_DIR}/index.shtml"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}

# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def parse_list_page(html):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')
    items = []
    for item_div in soup.select('div.right.fr div.data-table-item div.item-url'):
        p_title = item_div.select_one('p.w659 a')
        p_date = item_div.select_one('p.w80 a')
        if not p_title:
            continue
        href = p_title.get('href', '')
        title = p_title.get_text(strip=True)
        date_str = ''
        if p_date:
            date_str = p_date.get('title', '') or p_date.get_text(strip=True)
            # normalize date
            m = re.match(r'(\d{4})[.\-](\d{1,2})[.\-](\d{1,2})', date_str)
            if m:
                date_str = f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"
        if not href or not title:
            continue
        if href.startswith('/'):
            full_url = BASE_URL + href
        elif href.startswith('http'):
            full_url = href
        else:
            full_url = f"{BASE_URL}{LIST_DIR}/{href}"
        items.append({
            'title': title.strip(),
            'url': full_url,
            'date': date_str
        })
    return items

def parse_detail_page(html):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')
    
    # 标题
    title = ''
    content_div = soup.select_one('div.content')
    if content_div:
        title = content_div.get_text(strip=True)
        # title is first text before "发布时间"
        m = re.match(r'^(.+?)发布时间', title)
        if m:
            title = m.group(1).strip()
    
    # 日期
    date_str = ''
    info_div = soup.select_one('div.activity-info')
    if info_div:
        m = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', info_div.get_text())
        if m:
            date_str = m.group(1)
    
    # 正文
    body_parts = []
    attachments = []
    activity_div = soup.select_one('div.activity')
    if activity_div:
        for el in activity_div.children:
            if el.name == 'p':
                p_text = body_text(el)
                if p_text:
                    # Check attachments in p tags
                    for a in el.find_all('a'):
                        href = a.get('href', '')
                        if re.search(r'\.(docx?|pdf|xlsx?|rar|zip)$', href, re.I):
                            attach_name = a.get_text(strip=True) or os.path.basename(href)
                            if href.startswith('/'):
                                full_url = BASE_URL + href
                            elif href.startswith('http'):
                                full_url = href
                            else:
                                full_url = f"{BASE_URL}/{href.lstrip('/')}"
                            attachments.append({'name': attach_name, 'url': full_url})
                    body_parts.append(p_text)
            elif el.name == 'table':
                md = html_table_to_html(el)
                if md:
                    body_parts.append(md)
            elif el.name == 'div' and el.find('table'):
                for table in el.find_all('table'):
                    md = html_table_to_html(table)
                    if md:
                        body_parts.append(md)
    
    if not attachments and activity_div:
        for a in activity_div.find_all('a'):
            href = a.get('href', '')
            if re.search(r'\.(docx?|pdf|xlsx?|rar|zip)$', href, re.I):
                attach_name = a.get_text(strip=True) or os.path.basename(href)
                if href.startswith('/'):
                    full_url = BASE_URL + href
                elif href.startswith('http'):
                    full_url = href
                else:
                    full_url = f"{BASE_URL}/{href.lstrip('/')}"
                attachments.append({'name': attach_name, 'url': full_url})
    
    body = '\n\n'.join(body_parts)
    if attachments:
        if body:
            body += '\n\n'
        body += '\n'.join(['[附件：{}]({})'.format(a['name'], a['url']) for a in attachments])
    
    attach_str = ','.join(a['url'] for a in attachments)
    
    return title, date_str, body, attach_str

def main():
    import argparse
    parser = argparse.ArgumentParser(description='昆明市发改委-批准结果信息爬虫')
    parser.add_argument('--pages', type=int, default=5, help='爬取页数')
    args = parser.parse_args()
    
    all_list_items = []
    for page in range(1, args.pages + 1):
        if page == 1:
            url = LIST_PAGE1
        else:
            url = f"{BASE_URL}{LIST_DIR}/index_{page}.shtml"
        print(f"  列表页 {page}/{args.pages}: {url}")
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            r.encoding = 'utf-8'
        except Exception as e:
            print(f"    FAIL: {e}")
            continue
        items = parse_list_page(r.text)
        if not items:
            print(f"    → 0条，停止")
            break
        print(f"    → {len(items)}条")
        all_list_items.extend(items)
    
    print(f"\n共获取 {len(all_list_items)} 条列表项")
    
    results = []
    for i, item in enumerate(all_list_items):
        print(f"  [{i+1}/{len(all_list_items)}] ", end="")
        try:
            r = requests.get(item['url'], headers=HEADERS, timeout=30)
            r.encoding = 'utf-8'
        except Exception as e:
            print(f"  FAIL (fetch): {e}")
            continue
        
        title, date, body, attach = parse_detail_page(r.text)
        
        if not title:
            title = item['title']
        
        if not body or len(body.strip()) < 5:
            print(f"  跳过(空内容): {title[:30]}")
            continue
        
        print(f"{title[:50]}...")
        
        results.append({
            "title": title,
            "url": item['url'],
            "source_url": item['url'],
            "content": body,
            "pub_date": date or item['date'],
            "site_name": SITE_NAME,
            "attachments": attach
        })
        print(f"    正文: {len(body)}字 | 日期: {date or item['date']}")
    
    if not results:
        print("未获取到数据")
        return
    
    print(f"\n入库 {len(results)} 条...")
    push_to_searchdb(results, SITE_NAME)
    print(f"完成！共入库 {len(results)} 条")

if __name__ == '__main__':
    main()