#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
仙桃文明网-公示公告 爬虫
站点: http://www.cnxiantao.com/gg/
TRS 静态页: createPageHTML(19, 0, "index", "html")
列表: <li><span class="rt">2026-07-21</span><a href="./202607/t20260721_480830.html">标题</a></li>
分页: index.html / index_1.html ~ index_18.html
详情容器: div.neroncont (TRS 生成, 含 p.Custom_UnionStyle 嵌套 span)
标题: div.con > div.p1 > h2; 日期: 发布时间：YYYY年MM月DD日
"""
import os, sys, re, time, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb, normalize_pub_date

SITE_NAME = "仙桃文明网-公示公告"
CATEGORY = "公示公告"
LIST_URL = "http://www.cnxiantao.com/gg/"
PAGE_URL = "http://www.cnxiantao.com/gg/index_{}.html"
LINK_RE = re.compile(
    r'<li>\s*<span class="rt">([^<]+)</span>\s*'
    r'<a href="([^"]+)"[^>]*>([^<]{4,120})</a>',
    re.S
)
CONTAINERS = ["div.neroncont", ".TRS_UEDITOR", "div.content", "div.view", "div.article"]
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
HEADERS = {"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"}


def clean_title(title):
    """标题清洗: 剥 &middot;/&nbsp;/零宽/装饰符/栏目后缀"""
    import html as html_mod
    title = html_mod.unescape(title or "")
    title = re.sub(r"&middot;|&nbsp;", "", title)
    title = re.sub(r"[\u200b\u200c\u200d\ufeff]", "", title)
    title = re.sub(r"^[•·●\-—\s]+", "", title)
    title = re.sub(r"[—\-–]\s*(公示公告|通知公告|仙桃文明网)\s*$", "", title)
    title = re.sub(r"\s+", " ", title)
    return title.strip()


def fetch(url, session):
    try:
        r = session.get(url, timeout=20)
        r.encoding = "utf-8"
        return r.text if r.status_code == 200 else ""
    except Exception:
        return ""


def parse_list(html, list_url):
    items = []
    for m in LINK_RE.finditer(html):
        date_str, href, title = m.group(1).strip(), m.group(2).strip(), m.group(3).strip()
        if not title or len(title) < 4:
            continue
        if any(x in title for x in ["首页", "邮箱", "搜索", "无障碍", "地图"]):
            continue
        # 只保留站内 TRS 文章链接 (./202607/t20260721_480830.html), 过滤微信外链等
        if not re.search(r"t20\d{6}_\d+\.html?$", href):
            continue
        url = urljoin(list_url, href)
        date = normalize_pub_date(date_str)
        items.append({"title": clean_title(title), "url": url, "date": date})
    return items


def extract_detail(html, page_url, used_container):
    """提取标题/日期/正文(保留 p HTML)/附件, 记录命中的容器名"""
    soup = BeautifulSoup(html, "html.parser")

    # 标题: div.con > div.p1 > h2 优先, 回退 ArticleTitle meta / title 标签
    title = ""
    h2 = soup.select_one("div.con div.p1 h2") or soup.find("h2")
    if h2:
        title = clean_title(h2.get_text(strip=True))
    if not title:
        m = re.search(r'<meta[^>]*name=["\']ArticleTitle["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
        if m:
            title = clean_title(m.group(1))
    if not title:
        t = soup.find("title")
        if t:
            title = clean_title(t.get_text(strip=True).split("——")[0])

    # 日期: 发布时间：YYYY年MM月DD日 优先
    date = ""
    m = re.search(r"发布时间[：:]\s*([0-9]{4})年([0-9]{1,2})月([0-9]{1,2})日", html)
    if m:
        date = f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
    if not date:
        m = re.search(r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
        if m:
            date = normalize_pub_date(m.group(1))

    # 正文容器
    body_el = None
    for sel in CONTAINERS:
        el = soup.select_one(sel)
        if el and el.get_text(strip=True):
            body_el = el
            used_container.append(sel)
            break
    if body_el is None:
        return title, date, "", []

    # 附件: 容器内 a[href] 匹配扩展名或 download
    attachments = []
    for a in body_el.find_all("a", href=True):
        href = a["href"].strip()
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$", href, re.I) or "download" in href.lower():
            full = urljoin(page_url, href)
            name = a.get_text(strip=True) or href.split("/")[-1]
            attachments.append({"name": name, "url": full})

    # 正文: 保留 p/table/img HTML, 附件链接 urljoin 绝对化
    parts = []
    for el in body_el.find_all(["p", "table", "img"]):
        if el.name == "p":
            # 跳过表格内 p（表格会整体保留）和含表格的 p（避免表格文字重复）
            if el.find_parent("table") or el.find("table") or el.find("p"):
                continue
            for a in el.find_all("a", href=True):
                href = a["href"].strip()
                if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z)$", href, re.I) or "download" in href.lower():
                    a["href"] = urljoin(page_url, href)
            txt = el.get_text(" ", strip=True)
            txt = re.sub(r"\s+", " ", txt)
            if txt:
                parts.append(f"<p>{txt}</p>")
        elif el.name == "table":
            parts.append(str(el))
        elif el.name == "img":
            src = el.get("src", "")
            if src:
                full = urljoin(page_url, src)
                alt = el.get("alt", "")
                parts.append(f'<p><img src="{full}" alt="{alt}"></p>')

    for att in attachments:
        parts.append(f'<p><a href="{att["url"]}">{att["name"]}</a></p>')

    content = "\n".join(parts)
    # 提取文本过短(<50)且容器无表格时, fallback 容器整体文本
    # (兼容正文直接是 div/裸文本而非 p 的页面; 有表格页面保留表格 HTML 不 fallback)
    extracted_text = re.sub(r"<[^>]+>", "", content)
    extracted_text = re.sub(r"\s+", "", extracted_text)
    if (not content or len(extracted_text) < 50) and not body_el.find("table"):
        txt = body_el.get_text(" ", strip=True)
        if txt:
            content = f"<p>{txt}</p>"

    return title, date, content, attachments


def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return None


def main():
    max_pages = parse_pages_arg()
    if max_pages is None:
        max_pages = 19
    print(f"[{SITE_NAME}] max_pages={max_pages}")
    session = requests.Session()
    session.headers.update(HEADERS)

    all_items, seen = [], set()
    for pg in range(1, max_pages + 1):
        url = LIST_URL if pg == 1 else PAGE_URL.format(pg)
        html = fetch(url, session)
        if not html or len(html) < 200:
            print(f"  Page {pg}: 空/失败, 停止")
            break
        items = parse_list(html, LIST_URL)
        new_items = [x for x in items if x["url"] not in seen]
        for x in new_items:
            seen.add(x["url"])
        all_items.extend(new_items)
        print(f"  Page {pg}: {len(items)}条(新{len(new_items)}) 累计{len(all_items)}")
        if not new_items:
            break
        time.sleep(0.5)

    print(f"\n共 {len(all_items)} 条, 抓详情...")
    used_containers = []
    results = []
    for i, item in enumerate(all_items):
        html = fetch(item["url"], session)
        if not html:
            results.append({"site_name": SITE_NAME, "title": item["title"], "pub_date": item["date"],
                            "content": "", "source_url": item["url"], "url": item["url"],
                            "category": CATEGORY, "attachments": ""})
            continue
        title, date, content, attachments = extract_detail(html, item["url"], used_containers)
        if not title:
            title = item["title"]
        if not date:
            date = item["date"]
        results.append({"site_name": SITE_NAME, "title": title, "pub_date": date, "content": content,
                        "source_url": item["url"], "url": item["url"], "category": CATEGORY,
                        "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else ""})
        if (i + 1) % 10 == 0:
            print(f"  [{i+1}/{len(all_items)}] {title[:40]}")
        time.sleep(0.3)

    push_to_searchdb(results, batch_label=SITE_NAME)

    # 统计
    empty = sum(1 for r in results if not r["content"].strip())
    with_p = sum(1 for r in results if "<p>" in r["content"])
    att = sum(1 for r in results if r.get("attachments"))
    dates = [r["pub_date"] for r in results if r["pub_date"]]
    print(f"\n统计: 共{len(results)}条, 空正文{empty}, 含<p>正文{with_p}, 带附件{att}条")
    print(f"详情容器命中: {sorted(set(used_containers)) if used_containers else '未命中'}")
    print(f"日期范围: {min(dates) if dates else '-'} ~ {max(dates) if dates else '-'}")


if __name__ == "__main__":
    main()