#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
烟台市人民政府 - 行政许可公示 爬虫 (col43294)
CMS: JCMS (jpaas-publish-server)
List: API /api-gateway/jpaas-publish-server/front/page/build/unit
       paramJson={"pageNo":N,"pageSize":15}
Detail: /col/col43294/art/YYYY/art_XXX.html

2026-08-10 修复: 正文改为保留完整 HTML (表格/链接/图片原样保留),
不再转 markdown (| --- |), 不再重复提取 (table 只处理一次)。
附件链接完整保留 (document/download API)。
"""
import requests, re, json, sqlite3, time, sys, urllib.parse
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = "/root/search.db"
SITE_NAME = "烟台市生态环境局-行政许可公示"
CATEGORY = "行政许可公示"
GROUP = "山东"
BASE_URL = "https://www.yantai.gov.cn"
LIST_API = "https://www.yantai.gov.cn/api-gateway/jpaas-publish-server/front/page/build/unit"
MAX_PAGES = 5
PER_PAGE = 15

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Accept-Language": "zh-CN,zh;q=0.9",
}

SKIP_TEXTS = [
    "【打印本页】", "【关闭窗口】", "打印本页", "关闭窗口",
    "转载分享：", "浏览量：", "相关附件：", "相关稿件：",
    "扫一扫在手机打开",
]

API_PARAMS = {
    "webId": "1",
    "pageId": "43294",
    "parseType": "bulidstatic",
    "pageType": "column",
    "tagId": "当前栏目列表",
    "tplSetId": "uo2n7cvvGR8iPKdzxdBuu",
}


def _abs(href, detail_url):
    """相对/协议相对 href 转绝对"""
    if not href:
        return href
    href = href.strip()
    if href.startswith(("http://", "https://")):
        return href
    if href.startswith("//"):
        return "https:" + href
    if href.startswith(("#", "javascript:", "mailto:")):
        return href
    return urljoin(detail_url, href)


def clean_html(raw, detail_url):
    """清洗正文 HTML: 去 script/style, href/src 转绝对, 去多余空行"""
    raw = re.sub(r"<script[\s\S]*?</script>", "", raw)
    raw = re.sub(r"<style[\s\S]*?</style>", "", raw)
    raw = re.sub(r'<a\s+href="([^"]*)"', lambda m: '<a href="%s"' % _abs(m.group(1), detail_url), raw)
    raw = re.sub(r"<a\s+href='([^']*)'", lambda m: "<a href='%s'" % _abs(m.group(1), detail_url), raw)
    raw = re.sub(r'<img[^>]*src="([^"]*)"', lambda m: m.group(0).replace(m.group(1), _abs(m.group(1), detail_url)), raw)
    raw = re.sub(r'\sstyle="[^"]*"', "", raw)
    raw = re.sub(r"\n{3,}", "\n\n", raw)
    return raw.strip()


def extract_content(content_div, detail_url):
    """提取正文: 保留 HTML 表格/链接/图片, 不转 markdown, 不重复"""
    if content_div is None:
        return ""
    # 深拷贝避免修改源 soup 影响附件提取
    import copy
    div = copy.copy(content_div)

    # 移除尾部无关区块
    for bad in div.find_all(attrs={"class": re.compile(r"(scan|share|qrcode|print|footer|toolbar)", re.I)}):
        bad.decompose()
    for bad in div.find_all(attrs={"id": re.compile(r"(scan|share|qrcode|print|footer|toolbar)", re.I)}):
        bad.decompose()
    # 移除常见尾部按钮/分享
    for node in div.find_all(string=re.compile(r"打印本页|关闭窗口|分享到|扫码在手机")):
        p = node.find_parent()
        if p:
            p.decompose()

    html_str = str(div)
    html_str = clean_html(html_str, detail_url)

    # 去掉容器自身的 class 外壳标签（保留内部）
    html_str = re.sub(r"^<div[^>]*>", "", html_str)
    html_str = re.sub(r"</div>\s*$", "", html_str)

    # 过滤 SKIP_TEXTS 段
    for sk in SKIP_TEXTS:
        html_str = re.sub(r"<[^>]*>[^<]*" + re.escape(sk) + r"[^<]*</[^>]*>", "", html_str)

    return html_str


def extract_attachments(soup):
    atts = []
    seen = set()
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if "document/download" in href and "fileUrl=" in href:
            if not href.startswith("http"):
                if href.startswith("/"):
                    href = BASE_URL + href
                else:
                    href = BASE_URL + "/" + href.lstrip("/")
            name = a.get_text(strip=True) or href.split("=")[-1][:30]
            if href not in seen:
                seen.add(href)
                atts.append({"name": name, "url": href})
        elif re.search(r"\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$", href, re.I):
            if not href.startswith("http"):
                if href.startswith("/"):
                    href = BASE_URL + href
                else:
                    href = BASE_URL + "/" + href.lstrip("/")
            name = a.get_text(strip=True) or href.split("/")[-1].split("?")[0]
            if href not in seen:
                seen.add(href)
                atts.append({"name": name, "url": href})
    return atts


def fetch_detail(url, list_title=""):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20)
        r.encoding = "utf-8"
        html = r.text
    except Exception as e:
        return None, None, None, []

    soup = BeautifulSoup(html, "html.parser")

    # Title
    title = ""
    mt = soup.find("meta", attrs={"name": "ArticleTitle"})
    if mt and mt.get("content"):
        title = mt["content"].strip()
    if not title:
        h1 = soup.find("h1")
        if h1:
            title = h1.get_text(strip=True)
    if not title:
        title = list_title

    # Date
    date_str = ""
    md = soup.find("meta", attrs={"name": "PubDate"})
    if md and md.get("content"):
        date_str = md["content"].strip()[:10]
    if not date_str:
        mt2 = soup.find("meta", attrs={"name": "MakeTime"})
        if mt2 and mt2.get("content"):
            date_str = mt2["content"].strip()[:10]

    # Content: div.text
    content_div = soup.find("div", class_="text")
    if not content_div:
        content_div = soup.find("div", class_="content")
    if not content_div:
        content_div = soup.find("div", class_="main")

    attachments = extract_attachments(soup)

    if content_div:
        content = extract_content(content_div, url)
    else:
        content = ""

    if len(content.strip()) < 20:
        # 正文过短: 用附件/链接兜底
        content = f'<p><a href="{url}">{title}</a></p>'
        if attachments:
            for a in attachments:
                content += f'<p><a href="{a["url"]}">{a["name"]}</a></p>'

    return title, date_str, content, attachments


def fetch_list(session, page):
    params = dict(API_PARAMS)
    params["paramJson"] = json.dumps({"pageNo": page, "pageSize": PER_PAGE}, ensure_ascii=False)
    qs = urllib.parse.urlencode(params)
    url = LIST_API + "?" + qs
    try:
        r = session.get(url, headers=HEADERS, timeout=30)
        data = r.json()
    except Exception as e:
        print(f"  [ERR] list page {page}: {e}", flush=True)
        return []
    items = []
    html_text = ""
    try:
        html_text = data.get("data", {}).get("html", "") or ""
    except Exception:
        pass
    if not html_text:
        return []
    soup = BeautifulSoup(html_text, "html.parser")
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if "art_" not in href and "/art/" not in href:
            continue
        title = a.get("title", "") or a.get_text(strip=True)
        if not title:
            continue
        title = re.sub(r"\s+", " ", title).strip()
        full = urljoin(BASE_URL, href) if not href.startswith("http") else href
        # 日期: 列表里找 span (日期可能被换行分割如 2026-\n 08-\n 04)
        date_str = ""
        li = a.find_parent("li")
        if li:
            sp = li.find("span")
            if sp:
                txt = re.sub(r"\s+", "", sp.get_text())
                m = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", txt)
                if m:
                    date_str = m.group(1)
        items.append({"title": title, "url": full, "date": date_str})
    return items


def main():
    max_pages = MAX_PAGES
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            max_pages = int(a.split("=")[1])
        elif a.isdigit():
            max_pages = int(a)

    print(f"[YANTAI] SITE={SITE_NAME} max_pages={max_pages}", flush=True)
    session = requests.Session()

    all_items = []
    seen = set()
    for pg in range(1, max_pages + 1):
        items = fetch_list(session, pg)
        new_count = 0
        for it in items:
            if it["url"] in seen:
                continue
            seen.add(it["url"])
            all_items.append(it)
            new_count += 1
        print(f"  Page {pg}: {len(items)}条(新{new_count}) 累计{len(all_items)}", flush=True)
        if not items:
            print(f"  [END] page {pg} empty", flush=True)
            break
        time.sleep(0.5)

    print(f"列表完成: {len(all_items)} 条, 抓详情...", flush=True)
    results = []
    for i, it in enumerate(all_items):
        title, date_str, content, attachments = fetch_detail(it["url"], it.get("title", ""))
        if not title:
            title = it["title"]
        if not date_str:
            date_str = it.get("date", "")
        summary = re.sub(r"<[^>]+>", " ", content) if content else title
        summary = re.sub(r"\s+", " ", summary).strip()[:300]
        results.append({
            "site_name": SITE_NAME,
            "source_url": it["url"],
            "url": it["url"],
            "title": title,
            "publish_date": date_str,
            "content": content,
            "summary": summary,
            "category": CATEGORY,
            "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
        })
        if (i + 1) % 10 == 0:
            print(f"  detail {i+1}/{len(all_items)}", flush=True)
        time.sleep(0.3)

    # 入库
    sys.path.insert(0, "/root/gov_crawler")
    try:
        from crawler_lib import push_to_searchdb
        print(f"  pushing {len(results)} items to searchdb...", flush=True)
        push_to_searchdb(results, batch_label=SITE_NAME)
        empty = sum(1 for r in results if not (r.get("content") or "").strip())
        print(f"  DONE. pushed={len(results)} 空正文={empty}", flush=True)
    except Exception as e:
        print(f"  [ERR] push: {e}", flush=True)


if __name__ == "__main__":
    main()