#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
富民县人民政府-公示公告 爬虫
CMS: ZCMS 2.5.28063 (昆明信息港技术支持)
列表: /zfxxgk/fdzdgknr/gsgg/ (index.shtml), index_N.shtml (N=2..104)
详情: /c/YYYY-MM-DD/NNNNNNN.shtml
注意: DNS不可达, 需 IP 116.52.6.87 + Host header 绕过
"""

import sys, os, re, time, json
from datetime import datetime, timezone, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

import requests
from bs4 import BeautifulSoup

# === 自动页数控制 ===
_MAX_PG = None
for i, a in enumerate(sys.argv):
    if a == '--pages' and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break
if _MAX_PG is not None:
    print("[AutoPg] max_pages=" + str(_MAX_PG))

SITE_NAME = "富民县-公示公告"
BASE_IP = "116.52.6.87"
DOMAIN = "www.kmfm.gov.cn"
BASE_URL = f"http://{BASE_IP}"
LIST_PATH = "/zfxxgk/fdzdgknr/gsgg"
TOTAL_PAGES = 104
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Host": DOMAIN,
}


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)
def fetch(url, timeout=20):
    """通过 IP + Host header 抓取"""
    for attempt in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=timeout)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            if attempt < 2:
                time.sleep(2)
            else:
                print(f"  [WARN] 抓取失败 {url}: {e}")
                return None


def make_list_url(page):
    """构建列表页URL"""
    if page == 1:
        path = LIST_PATH + "/"
    else:
        path = f"{LIST_PATH}/index_{page}.shtml"
    return BASE_URL + path


def parse_list_page(html):
    """解析列表页, 返回 [(title, url, date), ...]"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    
    for item_div in soup.find_all("div", class_="data-table-item"):
        url_div = item_div.find("div", class_="item-url")
        if not url_div:
            continue
        
        # 标题链接
        title_a = url_div.find("p", class_="w659")
        if not title_a:
            continue
        a_tag = title_a.find("a", href=True)
        if not a_tag:
            continue
        
        href = a_tag["href"].strip()
        title = a_tag.get_text(" ", strip=True)
        title = re.sub(r'\s+', ' ', title).strip()
        
        if not href.startswith("http"):
            href = "http://" + DOMAIN + href
        
        if len(title) < 4:
            continue
        
        # 日期
        date = ""
        date_p = url_div.find("p", class_="w80")
        if date_p:
            date_a = date_p.find("a")
            if date_a and date_a.get("title"):
                m = re.search(r'(\d{4}-\d{2}-\d{2})', date_a["title"])
                if m:
                    date = m.group(1)
        
        items.append((title, href, date))
    
    return items


def _clean_span_spaces(text):
    """清洗 span 碎片导致的间距问题"""
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=[\u4e00-\u9fff])', '', text)  # 中<->中
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=\d)', '', text)              # 中<->数字
    text = re.sub(r'(?<=\d)[ \t\u3000]+(?=[\u4e00-\u9fff])', '', text)              # 数字<->中
    text = re.sub(r'\s+([，。、；：？！)】」》])', r'\1', text)               # 标点前
    text = re.sub(r'([（【「《])\s+', r'\1', text)                          # 标点后
    text = re.sub(r'(?<=\d)\s+(?=\d)', '', text)                           # 数字内
    return re.sub(r'\s+', ' ', text).strip()

def _make_url(src):
    """将相对URL转为绝对URL"""
    if src.startswith("http"):
        return src
    if src.startswith("/"):
        return "http://" + DOMAIN + src
    return "http://" + DOMAIN + "/" + src


def _is_attachment_link(href):
    """判断是否附件链接"""
    h = href.lower().split("?")[0]
    return any(h.endswith(e) for e in ('.pdf', '.doc', '.docx', '.xls', '.xlsx', '.zip', '.rar', '.txt', '.wps', '.et'))


def _is_filetype_icon(src):
    """判断是否文件类型图标"""
    return "/images/filetype/" in src or "/filetype/" in src


def extract_content(activity_div, page_url):
    """从 activity div 中提取正文内容"""
    parts = []
    
    for child in activity_div.children:
        name = getattr(child, 'name', None)
        
        if name == 'table':
            tbl_html = html_table_to_html(child, page_url)
            if tbl_html:
                parts.append(tbl_html)
        elif name == 'p':
            # 跳过表格内的 <p>
            if child.find_parent("table"):
                continue
            
            # 先检查是否有附件链接或有意义的图片
            has_attachment = False
            attach_texts = []
            meaningful_images = []
            
            for a_tag in child.find_all("a", href=True):
                if _is_attachment_link(a_tag["href"]):
                    has_attachment = True
                    a_text = a_tag.get_text(" ", strip=True)
                    full_href = _make_url(a_tag["href"])
                    attach_texts.append(f"[{a_text}]({full_href})")
            
            for img in child.find_all("img"):
                src = img.get("src", "")
                if src and not _is_filetype_icon(src):
                    alt = img.get("alt", "")
                    meaningful_images.append(f"![{alt}]({_make_url(src)})")
            
            # 如果有附件链接, 只输出附件链接, 不输出重复文本
            if has_attachment:
                for at in attach_texts:
                    parts.append(at + "\n\n")
                continue
            
            # 输出有意义的图片
            for mi in meaningful_images:
                parts.append(mi + "\n\n")
            
            # 正文文本 (无附件)
            txt = child.get_text(" ", strip=True)
            txt = _clean_span_spaces(txt)
            txt = re.sub(r'&#\d+;', '', txt)
            if txt and len(txt) > 3:
                parts.append(txt)
                parts.append("\n\n")
        
        elif name is None:
            t = str(child).strip()
            if t and len(t) > 10:
                t = re.sub(r'&#\d+;', '', t)
                t = re.sub(r'<[^>]+>', '', t)
                parts.append(t.strip())
                parts.append("\n\n")
    
    result = ''.join(parts).strip()
    result = re.sub(r'\n{4,}', '\n\n', result)
    return result


def parse_detail(url):
    """解析详情页"""
    html = fetch(url)
    if not html:
        return None, None, None
    
    soup = BeautifulSoup(html, "html.parser")
    
    # 标题: meta ArticleTitle 优先
    title = ""
    meta = soup.find("meta", attrs={"name": "ArticleTitle"})
    if meta:
        title = meta.get("content", "").strip()
    if not title:
        meta_p = soup.find("meta", attrs={"property": "ArticleTitle"})
        if meta_p:
            title = meta_p.get("content", "").strip()
    if not title:
        title_tag = soup.find("title")
        if title_tag:
            t = title_tag.get_text(strip=True)
            title = re.sub(r'_公示公告_富民县人民政府.*', '', t).strip()
    
    # 清洗HTML实体
    title = re.sub(r'&#\d+;', '', title)
    
    # 日期: meta PubDate
    date = ""
    meta_d = soup.find("meta", attrs={"name": "PubDate"})
    if meta_d:
        d = meta_d.get("content", "")
        m = re.search(r'(\d{4}-\d{2}-\d{2})', d)
        if m:
            date = m.group(1)
    if not date:
        activity_info = soup.find("div", class_="activity-info")
        if activity_info:
            m = re.search(r'(\d{4}-\d{2}-\d{2})', activity_info.get_text())
            if m:
                date = m.group(1)
    
    # 正文
    content = ""
    content_div = soup.find("div", class_="content")
    if content_div:
        activity = content_div.find("div", class_="activity")
        if activity:
            content = extract_content(activity, url)
    
    if not content:
        return title, "", date
    
    return title, content, date


def main():
    # 页数限制
    pages_to_fetch = min(TOTAL_PAGES, _MAX_PG) if _MAX_PG else TOTAL_PAGES
    
    print(f"=== {SITE_NAME} ===")
    print(f"  总页数: {TOTAL_PAGES}, 本次抓取: {pages_to_fetch} 页")
    
    all_items = []
    for page in range(1, pages_to_fetch + 1):
        list_url = make_list_url(page)
        print(f"  第 {page}/{pages_to_fetch} 页: {list_url}")
        
        html = fetch(list_url)
        if not html:
            print(f"    无法获取, 停止翻页")
            break
        
        items = parse_list_page(html)
        if not items:
            print(f"    空列表, 停止翻页")
            break
        
        print(f"    找到 {len(items)} 条")
        all_items.extend(items)
        time.sleep(0.3)
    
    print(f"\n  列表总计: {len(all_items)} 条\n")
    
    if not all_items:
        print("  无数据")
        return
    
    # 日期过滤
    recent = [i for i in all_items if i[2] >= CUTOFF] if all_items[0][2] else all_items
    print(f"  近3年: {len(recent)} 条 (过滤掉 {len(all_items)-len(recent)})")
    
    # 抓取详情页
    results = []
    for idx, (title, url, date) in enumerate(recent):
        print(f"  [{idx+1}/{len(recent)}] {title[:40]}...", end=" ", flush=True)
        
        dt, content, dd = parse_detail(url)
        final_title = dt or title
        final_date = dd or date
        
        if not content or len(content.strip()) < 10:
            print("⚠ 无正文")
            # 有标题也要入库吗? 跳过
            continue
        
        summary = content[:200].strip()
        results.append({
            "site_name": SITE_NAME,
            "title": final_title,
            "url": url,
            "content": content,
            "summary": summary,
            "pub_date": final_date,
        })
        print(f"✅ ({len(content)}字)")
        
        if (idx + 1) % 20 == 0:
            print(f"  进度: {idx+1}/{len(recent)}, 已入库: {len(results)}")
    
    if results:
        print(f"\n  入库 {len(results)} 条")
        push_to_searchdb(results, "kmfm_gsgg")
    else:
        print("  无新数据入库")
    
    print("完成")


if __name__ == "__main__":
    main()