#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
鹤山市生态环境局-通知公告 爬虫
站点: https://www.heshan.gov.cn/zwgk/xxgk/hsstyz/gzdt/tzgg/
NFCMS 静态分页: createPageHTML(25,0,"index","html") -> index.html / index_N.html
列表链接: .../content/post_3533019.html (注意匹配 post_ 后数字)
详情容器: div.view.TRS_UEDITOR (class="view TRS_UEDITOR trs_paper_default trs_web")
正文保留 <p> HTML, 附件 urljoin 绝对化, 标题清洗。
用法: python3 crawl_heshan_tzgg.py [--pages=N]
"""
import os, sys, re, time, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

SITE_NAME = "鹤山市生态环境局-通知公告"
CATEGORY = "通知公告"
LIST_URL = "https://www.heshan.gov.cn/zwgk/xxgk/hsstyz/gzdt/tzgg/"
BASE_LIST = "https://www.heshan.gov.cn/zwgk/xxgk/hsstyz/gzdt/tzgg/index_{}.html"

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
HEADERS = {"User-Agent": UA, "Accept": "text/html,application/xhtml+xml", "Accept-Language": "zh-CN,zh;q=0.9"}

# 链接正则: content/post_ 后跟数字, 可选 .html 后缀
LINK_PAT = re.compile(r'href="([^"]*content/post_(\d+)[^"]*)"[^>]*>\s*([^<]{4,120})</a>', re.I)
ATTACH_EXT = re.compile(r'\.(pdf|doc|docx|xls|xlsx|ppt|pptx|zip|rar|wps|et|dps|ofd)$', re.I)

CONTAINERS = ["div.view.TRS_UEDITOR", "div.view", "div.content", "div.article"]


def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return None


def clean_title(title):
    import html as html_mod
    title = html_mod.unescape(title or "")
    title = re.sub(r"&middot;|&nbsp;|\u200b|\ufeff", "", title)
    title = re.sub(r"\s+", " ", title)
    title = re.sub(r"^[•·\-–—]\s*", "", title)
    return title.strip()


def fetch(url, session):
    try:
        r = session.get(url, timeout=25)
        r.encoding = "utf-8"
        return r.text if r.status_code == 200 else ""
    except Exception as e:
        print(f"    [WARN] fetch fail {url}: {e}")
        return ""


def parse_list(html, list_url):
    """解析列表页 -> [(title, url, date)]"""
    items = []
    for m in LINK_PAT.finditer(html):
        href, post_id, title = m.group(1), m.group(2), m.group(3).strip()
        title = clean_title(title)
        if not title or len(title) < 4:
            continue
        if any(x in title for x in ["首页", "邮箱", "搜索", "无障碍", "地图", "网站地图"]):
            continue
        url = urljoin(list_url, href)
        items.append({"title": title, "url": url, "post_id": post_id, "date": ""})
    return items


def extract_detail(html, page_url):
    """提取 标题/日期/正文(保留 p HTML, 附件绝对化)/附件列表"""
    soup = BeautifulSoup(html, "html.parser")

    # 标题: meta ArticleTitle > div.title > title 标签
    title = ""
    m = re.search(r'<meta[^>]*name=["\']ArticleTitle["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
    if m:
        title = clean_title(m.group(1))
    if not title:
        t = soup.select_one("div.title")
        if t:
            title = clean_title(t.get_text(strip=True))
    if not title:
        m = re.search(r'<meta[^>]*content=["\']([^"\']*)["\'][^>]*name=["\']ArticleTitle["\']', html, re.I)
        if m:
            title = clean_title(m.group(1))
    if not title:
        t = soup.find("title")
        if t:
            title = clean_title(t.get_text(strip=True).split("_")[0])

    # 日期: meta PubDate > span.time > 页面正则
    date = ""
    m = re.search(r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
    if m:
        date = m.group(1)
    if not date:
        m = re.search(r'<meta[^>]*content=["\']([^"\']*)["\'][^>]*name=["\']PubDate["\']', html, re.I)
        if m:
            date = m.group(1)
    if not date:
        t = soup.select_one("span.time")
        if t:
            date = t.get_text(strip=True)
    if not date:
        m = re.search(r'(20\d{2}-\d{1,2}-\d{1,2})', html)
        if m:
            date = m.group(1)

    # 正文容器
    body_el = None
    for sel in CONTAINERS:
        body_el = soup.select_one(sel)
        if body_el and body_el.get_text(strip=True):
            break
    if body_el is None:
        return title, date, "", []

    # 附件: 容器内 + 页面 #fujian / .m-attachment 区域
    attachments = []
    seen_att = set()
    for a in soup.select("#fujian a[href], .m-attachment a[href]") + body_el.find_all("a", href=True):
        href = (a.get("href") or "").strip()
        if not href or href.startswith("javascript"):
            continue
        if ATTACH_EXT.search(href.lower()) or "download" in href.lower():
            full = urljoin(page_url, href)
            if full in seen_att:
                continue
            seen_att.add(full)
            name = clean_title(a.get_text(strip=True)) or href.split("/")[-1]
            attachments.append({"name": name, "url": full})

    # 正文: 保留 <p> 的 inner HTML, a/img 绝对化; table 原样; 无 p 时兜底纯文本
    parts = []
    for el in body_el.find_all(["p", "table", "img"]):
        if el.name == "p":
            if el.find_parent("table") or el.find("table") or el.find("p"):
                continue
            for a in el.find_all("a", href=True):
                h = (a.get("href") or "").strip()
                if h and not h.startswith("javascript"):
                    a["href"] = urljoin(page_url, h)
            for img in el.find_all("img"):
                src = img.get("src") or img.get("data-src") or ""
                if src:
                    img["src"] = urljoin(page_url, src)
            txt = el.get_text(" ", strip=True)
            if txt:
                parts.append(str(el))
        elif el.name == "table":
            parts.append(str(el))
        elif el.name == "img":
            src = el.get("src") or el.get("data-src") or ""
            if src:
                full = urljoin(page_url, src)
                alt = el.get("alt", "")
                parts.append(f'<p><img src="{full}" alt="{alt}"></p>')

    for att in attachments:
        parts.append(f'<p><a href="{att["url"]}">{att["name"]}</a></p>')

    content = "\n".join(parts)
    # 提取文本过短(<50)且容器无表格时, fallback 容器整体文本
    extracted_text = re.sub(r"<[^>]+>", "", content)
    extracted_text = re.sub(r"\s+", "", extracted_text)
    if (not content or len(extracted_text) < 50) and not body_el.find("table"):
        txt = body_el.get_text(" ", strip=True)
        if txt:
            content = f"<p>{txt}</p>"

    return title, date, content, attachments


def main():
    max_pages = parse_pages_arg() or 30
    print(f"[HeshanTZGG] SITE={SITE_NAME} max_pages={max_pages}")
    session = requests.Session()
    session.headers.update(HEADERS)

    all_items = []
    seen = set()
    for pg in range(1, max_pages + 1):
        url = LIST_URL if pg == 1 else BASE_LIST.format(pg - 1)
        html = fetch(url, session)
        if not html or len(html) < 200:
            print(f"  Page {pg}: 空/失败({url}), 停止")
            break
        items = parse_list(html, LIST_URL)
        new_items = [x for x in items if x["url"] not in seen]
        for x in new_items:
            seen.add(x["url"])
        all_items.extend(new_items)
        print(f"  Page {pg}: {len(items)}条(新{len(new_items)}) 累计{len(all_items)}")
        if not new_items:
            print("  无新链接, 停止翻页")
            break
        time.sleep(0.4)

    print(f"\n共 {len(all_items)} 条, 抓详情...")
    results = []
    empty_body = 0
    with_p = 0
    att_total = 0
    for i, item in enumerate(all_items):
        html = fetch(item["url"], session)
        if not html:
            results.append({"site_name": SITE_NAME, "title": item["title"], "pub_date": item["date"],
                            "content": "", "source_url": item["url"], "url": item["url"],
                            "category": CATEGORY, "attachments": ""})
            empty_body += 1
            continue
        title, date, content, attachments = extract_detail(html, item["url"])
        if not title:
            title = item["title"]
        if not date:
            date = item["date"]
        if not content.strip():
            empty_body += 1
        if re.search(r"<p[\s>]", content):
            with_p += 1
        att_total += len(attachments)
        results.append({"site_name": SITE_NAME, "title": title, "pub_date": date, "content": content,
                        "source_url": item["url"], "url": item["url"], "category": CATEGORY,
                        "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else ""})
        if (i + 1) % 10 == 0:
            print(f"  [{i+1}/{len(all_items)}] {title[:40]}")
        time.sleep(0.3)

    push_to_searchdb(results, batch_label=SITE_NAME)
    dates = [r["pub_date"] for r in results if r["pub_date"]]
    print(f"\n站点: {SITE_NAME} 采集: {len(results)} 空正文: {empty_body} 含<p>: {with_p} 附件: {att_total}")
    print(f"日期范围: {min(dates) if dates else '-'} ~ {max(dates) if dates else '-'}")


if __name__ == "__main__":
    main()