#!/usr/bin/env python3
"""
沂水县人民政府 — 公告公示栏目爬虫 (经济开发区)
URL: http://www.yishui.gov.cn/xzwgk1/bm_xz_jd_xxgk/zssydw/xjjkfq/fdzdgknr1/gggs.htm
CMS: VSB9 (Visual SiteBuilder 9)
需设置 r.encoding = 'utf-8' 解决无charset导致乱码问题
"""

import requests
from bs4 import BeautifulSoup
import re
import sys
import os
import argparse
from datetime import datetime
from urllib.parse import urljoin

# ─── 配置 ─────────────────────────────────────────────
BASE_URL = "http://www.yishui.gov.cn"
LIST_URL_TEMPLATE = "http://www.yishui.gov.cn/xzwgk1/bm_xz_jd_xxgk/zssydw/xjjkfq/fdzdgknr1/gggs{path}"
SITE_NAME = "沂水县人民政府"
COLUMN_NAME = "公告公示"
GROUP = "山东省"
INDUSTRY = "政府公告"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb


# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def safe_fetch(url):
    """带编码处理的请求"""
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        if r.status_code == 200:
            r.encoding = 'utf-8'  # VSB9 不返回charset，强制UTF-8
            return r.text
        return None
    except Exception as e:
        print(f"    ❌ 请求失败: {e}", flush=True)
        return None


def fetch_list_page(url: str) -> tuple:
    """
    获取列表页，返回 (items, next_page_url)
    items: [(title, full_url, date_str), ...]
    next_page_url: 下一页URL或None
    """
    html = safe_fetch(url)
    if not html:
        return [], None

    soup = BeautifulSoup(html, "html.parser")
    items = []

    # VSB9列表: <li> 内含 <a> + 日期文本
    for li in soup.find_all("li"):
        a_tag = li.find("a", href=True)
        if not a_tag:
            continue
        href = a_tag["href"].strip()
        title = a_tag.get("title", "").strip() or a_tag.get_text(strip=True)
        if not title or "/info/35725/" not in href:
            continue

        # 完整URL - 用urljoin解析相对路径
        full_url = urljoin(url, href)

        # 日期：li中的日期文本
        date_str = ""
        li_text = body_text(li)
        # 去掉标题文字，看剩余部分是否有日期
        m = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", li_text)
        if m:
            date_str = m.group(1)

        items.append((title, full_url, date_str))

    # 找下一页
    next_url = None
    for a in soup.find_all("a", href=True):
        txt = a.get_text(strip=True)
        href = a["href"].strip()
        if txt in ("下页", "下一页", "下一页>", "»") and "htm" in href:
            if href.startswith("http"):
                next_url = href
            elif href.startswith("../"):
                next_url = "http://www.yishui.gov.cn/xzwgk1/bm_xz_jd_xxgk/zssydw/xjjkfq/fdzdgknr1/" + href.lstrip("../")
            elif not href.startswith("/"):
                next_url = "http://www.yishui.gov.cn/xzwgk1/bm_xz_jd_xxgk/zssydw/xjjkfq/fdzdgknr1/" + href
            else:
                next_url = BASE_URL + href
            break

    print(f"  📄 {os.path.basename(url) or 'gggs.htm'}: {len(items)}条, 下一页={'有' if next_url else '无'}", flush=True)
    return items, next_url


def extract_content_with_tables(div):
    """递归提取正文，保留表格HTML"""
    content_parts = []

    def walk_element(el):
        for child in el.children:
            if child.name == "table":
                content_parts.append(str(child))
            elif child.name and child.name not in ("script", "style"):
                if child.find_all("table"):
                    walk_element(child)
                else:
                    sep = "" if child.name == "p" else " "
                    txt = child.get_text(sep, strip=True)
                    if txt:
                        content_parts.append(txt)
    walk_element(div)
    return "\n".join(content_parts) if content_parts else div.get_text(" ", strip=True)


def fetch_detail(url: str) -> tuple:
    """获取详情页，返回 (title, content, publish_date)"""
    html = safe_fetch(url)
    if html is None:
        return None, "", ""

    soup = BeautifulSoup(html, "html.parser")

    # 标题
    title = ""
    meta_title = soup.find("meta", attrs={"name": "ArticleTitle"})
    if meta_title and meta_title.get("content"):
        title = meta_title["content"].strip()
    if not title:
        t = soup.find("title")
        if t:
            title = t.get_text(strip=True)

    # 日期
    pub_date = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        pub_date = meta_date["content"].strip()

    # 正文: VSB9 的 vsb_content（id动态后缀）
    content_div = soup.find("div", id=lambda x: x and x.startswith("vsb_content"))
    if not content_div:
        content_div = soup.find("div", class_="newscontent_s")

    content = extract_content_with_tables(content_div) if content_div else ""

    return title, content, pub_date


def crawl(pages: int = 5, push: bool = True):
    """主爬虫"""
    first_url = "http://www.yishui.gov.cn/xzwgk1/bm_xz_jd_xxgk/zssydw/xjjkfq/fdzdgknr1/gggs.htm"

    print(f"🔍 {SITE_NAME} - {COLUMN_NAME}", flush=True)
    print(f"📊 本次爬取 {pages} 页" if pages > 0 else "📊 爬取全部页", flush=True)

    all_items = []
    current_url = first_url
    page_no = 0

    while current_url and (pages <= 0 or page_no < pages):
        page_no += 1
        items, next_url = fetch_list_page(current_url)
        if not items and page_no == 1:
            print(f"  ❌ 首页无数据", flush=True)
            break
        all_items.extend(items)
        current_url = next_url

        if not current_url:
            break

    print(f"\n📋 列表共 {len(all_items)} 条", flush=True)
    print(f"{'='*60}", flush=True)

    push_items = []
    fail_count = 0

    for idx, (title, url, date_str) in enumerate(all_items, 1):
        print(f"\n[{idx}/{len(all_items)}] {title[:60]}", flush=True)
        print(f"    📎 {url}", flush=True)
        if date_str:
            print(f"    📅 {date_str}", flush=True)

        detail_title, content, pub_date = fetch_detail(url)
        if detail_title is None:
            print(f"    ❌ 详情获取失败", flush=True)
            fail_count += 1
            continue

        final_title = detail_title or title
        final_date = pub_date or date_str

        content_preview = content[:80].replace('\n', ' ')
        print(f"    ✅ 正文 {len(content)} 字: {content_preview}...", flush=True)

        push_items.append({
            "title": final_title,
            "url": url,
            "source_url": url,
            "pub_date": final_date,
            "content": content,
            "summary": content[:500] if content else final_title,
            "site_name": SITE_NAME,
            "group_name": GROUP,
            "industry": INDUSTRY,
            "category": COLUMN_NAME,
        })

    if push and push_items:
        print(f"\n{'='*60}", flush=True)
        print(f"📤 正在推送 {len(push_items)} 条到 search.db...", flush=True)
        push_to_searchdb(push_items, batch_label=f"{SITE_NAME}-{COLUMN_NAME}")
    elif push:
        print(f"\n⚠️ 无数据可推送", flush=True)

    print(f"\n{'='*60}", flush=True)
    print(f"✅ 完成: 共{len(push_items)}条, 失败{fail_count}", flush=True)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=f"{SITE_NAME} - {COLUMN_NAME} 爬虫")
    parser.add_argument("--pages", type=int, default=5, help="爬取页数 (0=全部)")
    parser.add_argument("--no-push", action="store_true", help="不推送到search.db")
    args = parser.parse_args()

    crawl(pages=args.pages, push=not args.no_push)