#!/usr/bin/env python3
"""
TRS 静态页组批量爬虫: 莆田/襄阳 公示公告
列表: createPageHTML index_N.shtml 静态分页
链接格式: ./YYYYMM/tYYYYMMDD_ID.shtml|htm (相对路径)
详情: TRS_UEDITOR / content / view 容器
"""
import os, sys, re, time, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
HEADERS = {"User-Agent": UA, "Accept": "text/html,application/xhtml+xml", "Accept-Language": "zh-CN,zh;q=0.9"}

# SITES 配置: key -> (site_name, list_url, 链接正则, 分页格式, 详情容器)
SITES = {
    "putian": {
        "name": "莆田市生态环境局-公示公告",
        "list": "https://sthjj.putian.gov.cn/xxgk/gsgg/",
        "link_pat": r'href="([^"]*(?:t20\d{6}_\d+\.s?html?))"[^>]*>([^<]{4,80})</a>',
        "page_url": "https://sthjj.putian.gov.cn/xxgk/gsgg/index_{}.htm",
        "container": ["div.view", "div.content", "#zoom", "div.article"],
        "cutoff": "2023-08-10",
    },
    "xiangyang": {
        "name": "襄阳市生态环境局-通知公告",
        "list": "http://sthjj.xiangyang.gov.cn/zxzx/tzgg/",
        "link_pat": r'href="([^"]*(?:t20\d{6}_\d+\.shtml))"[^>]*>([^<]{4,80})</a>',
        "page_url": "http://sthjj.xiangyang.gov.cn/zxzx/tzgg/index_{}.shtml",
        "container": [".TRS_UEDITOR", "div.zoom", "div.content", "div.view"],
        "cutoff": "2023-08-10",
    },
}

def parse_pages_arg():
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            return int(a.split("=")[1])
        if a.isdigit():
            return int(a)
    return None

def get_site():
    for a in sys.argv[1:]:
        if a.startswith("--site="):
            return a.split("=")[1]
    return None

def clean_title(title):
    import html as html_mod
    title = html_mod.unescape(title or "")
    title = re.sub(r"&middot;|&nbsp;", "", title)
    title = re.sub(r"[\u200b\ufeff]", "", title)
    title = re.sub(r"^[•·]\s*", "", title)
    return title.strip()

def fetch(url, session):
    try:
        r = session.get(url, timeout=20)
        r.encoding = "utf-8"
        return r.text if r.status_code == 200 else ""
    except Exception:
        return ""

def parse_list(html, cfg, list_url):
    """解析列表页返回 [(title, url, date)]"""
    items = []
    for m in re.finditer(cfg["link_pat"], html):
        href, title = m.group(1), m.group(2).strip()
        if not title or len(title) < 4:
            continue
        # 过滤导航
        if any(x in title for x in ["首页", "邮箱", "搜索", "无障碍", "地图"]):
            continue
        url = urljoin(list_url, href)
        # 日期从 URL 提取 tYYYYMMDD
        dm = re.search(r't(\d{8})_', href)
        date = ""
        if dm:
            d = dm.group(1)
            date = f"{d[:4]}-{d[4:6]}-{d[6:8]}"
        items.append({"title": clean_title(title), "url": url, "date": date})
    return items

def extract_detail(html, page_url, cfg):
    """提取标题/日期/正文(保留 p/table HTML)/附件"""
    soup = BeautifulSoup(html, "html.parser")
    title = ""
    h1 = soup.find("h1")
    if h1:
        title = clean_title(h1.get_text(strip=True))
    if not title:
        m = re.search(r'<meta[^>]*name=["\']ArticleTitle["\'][^>]*content=["\']([^"\']*)["\']', html, re.I)
        if m:
            title = clean_title(m.group(1))
    if not title:
        t = soup.find("title")
        if t:
            title = clean_title(t.get_text(strip=True).split("_")[0])

    date = ""
    m = re.search(r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\'](\d{4}-\d{2}-\d{2})', html, re.I)
    if m:
        date = m.group(1)
    if not date:
        for kw in ["发布时间", "发布日期", "成文日期"]:
            m = re.search(r'%s[：:]\s*(\d{4}-\d{2}-\d{2})' % kw, html)
            if m:
                date = m.group(1)
                break

    body_el = None
    for sel in cfg["container"]:
        body_el = soup.select_one(sel)
        if body_el and body_el.get_text(strip=True):
            break
    if body_el is None:
        return title, date, "", []

    attachments = []
    for a in body_el.find_all("a", href=True):
        href = a["href"].strip()
        if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|wps)$', href.lower()) or "download" in href.lower():
            full = urljoin(page_url, href)
            name = a.get_text(strip=True) or href.split("/")[-1]
            attachments.append({"name": name, "url": full})

    parts = []
    for el in body_el.find_all(["p", "table", "img"]):
        if el.name == "p":
            if el.find_parent("table") or el.find("table") or el.find("p"):
                continue
            for a in el.find_all("a", href=True):
                href = a["href"].strip()
                if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|wps)$', href.lower()) or "download" in href.lower():
                    a["href"] = urljoin(page_url, href)
            txt = el.get_text(" ", strip=True)
            txt = re.sub(r"\s+", " ", txt)
            if txt:
                parts.append(f"<p>{txt}</p>")
        elif el.name == "table":
            parts.append(str(el))
        elif el.name == "img":
            src = el.get("src", "")
            if src:
                full = urljoin(page_url, src)
                alt = el.get("alt", "")
                parts.append(f'<p><img src="{full}" alt="{alt}"></p>')

    for att in attachments:
        parts.append(f'<p><a href="{att["url"]}">{att["name"]}</a></p>')

    content = "\n".join(parts)
    # 提取文本过短(<50)且容器无表格时, fallback 容器整体文本
    extracted_text = re.sub(r"<[^>]+>", "", content)
    extracted_text = re.sub(r"\s+", "", extracted_text)
    if (not content or len(extracted_text) < 50) and not body_el.find("table"):
        txt = body_el.get_text(" ", strip=True)
        if txt:
            content = f"<p>{txt}</p>"

    return title, date, content, attachments

def main():
    site_key = get_site()
    if site_key not in SITES:
        print("用法: python3 crawl_trs_static.py --site=putian|xiangyang [--pages=N]")
        return
    cfg = SITES[site_key]
    max_pages = parse_pages_arg()
    if max_pages is None:
        max_pages = 30

    print(f"[AutoPg] max_pages={max_pages} (SITE={cfg['name']})")
    session = requests.Session()
    session.headers.update(HEADERS)

    all_items = []
    seen = set()
    for pg in range(1, max_pages + 1):
        if pg == 1:
            url = cfg["list"]
        else:
            url = cfg["page_url"].format(pg)
        html = fetch(url, session)
        if not html or len(html) < 200:
            print(f"  Page {pg}: 空/失败, 停止")
            break
        items = parse_list(html, cfg, cfg["list"])
        new_items = [x for x in items if x["url"] not in seen]
        for x in new_items:
            seen.add(x["url"])
        all_items.extend(new_items)
        print(f"  Page {pg}: {len(items)}条(新{len(new_items)}) 累计{len(all_items)}")
        if new_items:
            dates = [x["date"] for x in new_items if x["date"]]
            if dates and min(dates) < cfg["cutoff"]:
                print(f"  已达 CUTOFF {cfg['cutoff']}")
                break
        if not new_items:
            break
        time.sleep(0.5)

    print(f"\n共 {len(all_items)} 条, 抓详情...")
    results = []
    for i, item in enumerate(all_items):
        html = fetch(item["url"], session)
        if not html:
            results.append({"site_name": cfg["name"], "title": item["title"], "pub_date": item["date"],
                            "content": "", "source_url": item["url"], "url": item["url"],
                            "category": "公示公告", "attachments": ""})
            continue
        title, date, content, attachments = extract_detail(html, item["url"], cfg)
        if not title:
            title = item["title"]
        if not date:
            date = item["date"]
        results.append({"site_name": cfg["name"], "title": title, "pub_date": date, "content": content,
                        "source_url": item["url"], "url": item["url"], "category": "公示公告",
                        "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else ""})
        if (i + 1) % 20 == 0:
            print(f"  [{i+1}/{len(all_items)}] {title[:40]}")
        time.sleep(0.3)

    push_to_searchdb(results, batch_label=cfg["name"])
    dates = [r["pub_date"] for r in results if r["pub_date"]]
    print(f"\n站点: {cfg['name']} 采集: {len(results)} 日期: {min(dates) if dates else '-'} ~ {max(dates) if dates else '-'}")

if __name__ == "__main__":
    main()