#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""莱州市人民政府 - 通知公告 (col23203) 爬虫
CMS: JPAAS (jcms, 烟台莱州)
列表: /api-gateway/jpaas-publish-server/front/page/build/unit?parseType=bulidstatic&webId=154&tplSetId=kUVXvNhbixMZJJ7baLQ9I&tagId=右侧文章列表&editType=null&pageId=23203&paramJson={"pageNo":N,"pageSize":20}
  响应 JSON data.html: <li class="bt-main-r-ul-li"> <a target="_blank" href="/col/col23203/art/2026/art_hash.html" title="完整标题">显示文本</a><span>YYYY-MM-DD</span></li>
  总数 count=4263 → 214页 × 20条
详情: /col/col23203/art/YYYY/art_hash.html
  正文 div#zoom (直接 p 子元素), 附件 <a data-link=真实路径> (href 是加密下载API)
  纯PDF型: zoom 无文本, pdfUrls JS 变量含 fileURL → 转附件 + 占位正文
"""
import sys, re, time, sqlite3, ssl, urllib.request, json, urllib.parse
from urllib.parse import urljoin
from bs4 import BeautifulSoup

SITE_NAME = "莱州市-通知公告"
SCRIPT_NAME = "crawl_laizhou_tzgg.py"
GROUP_NAME = "山东"
BASE_URL = "http://www.laizhou.gov.cn"
API_URL = BASE_URL + "/api-gateway/jpaas-publish-server/front/page/build/unit"
QUERY_BASE = {
    "parseType": "bulidstatic",
    "webId": "154",
    "tplSetId": "kUVXvNhbixMZJJ7baLQ9I",
    "pageType": "column",
    "tagId": "右侧文章列表",
    "editType": "null",
    "pageId": "23203",
}
TOTAL_PAGES = 214
DB_PATH = "/mnt/data/search.db"
OUTPUT_FILE = "/root/gov_crawler/laizhou_tzgg_output.jsonl"

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}


def http_get(url, timeout=25, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers=HEADERS)
            resp = urllib.request.urlopen(req, timeout=timeout, context=ctx)
            return resp.read().decode("utf-8", errors="replace")
        except Exception as e:
            if i == retries - 1:
                print(f"  [WARN] 请求失败 {url}: {e}", file=sys.stderr)
                return ""
            time.sleep(1.5)
    return ""


def clean_title(title):
    """strip &middot;&nbsp; 实体前缀 和省略号截断后缀"""
    title = title.replace("&middot;", "").replace("&nbsp;", "").replace("\u00b7", "")
    title = re.sub(r"^[\s\xa0·\u00b7]+", "", title)
    title = re.sub(r"\s*\.{3,}\s*$", "", title)
    return title.strip()


def fetch_list_page(page):
    """调用 JPAAS API 获取列表页 HTML"""
    paramJson = json.dumps({"pageNo": page, "pageSize": 20}, ensure_ascii=False, separators=(",", ":"))
    qd = dict(QUERY_BASE)
    qd["paramJson"] = paramJson
    url = API_URL + "?" + urllib.parse.urlencode(qd)
    html = http_get(url)
    if not html:
        return ""
    try:
        d = json.loads(html)
    except Exception:
        return ""
    if not d.get("success"):
        return ""
    return d.get("data", {}).get("html", "")


def parse_list(html):
    """从列表 HTML 提取 (url, title, date)"""
    items = []
    for m in re.finditer(r'href="([^"]*?art/[^"]*?\.html)"\s+title="([^"]*)"', html):
        href, t = m.group(1), m.group(2)
        url = urljoin(BASE_URL, href)
        title = clean_title(t.strip())
        # 日期: 紧跟 <a> 的 <span>
        after = html[m.end():m.end() + 300]
        dm = re.search(r"<span[^>]*>\s*(\d{4}-\d{2}-\d{2})\s*</span>", after)
        date = dm.group(1) if dm else ""
        if title:
            items.append((url, title, date))
    return items


def extract_detail(html, page_url):
    """提取 (title, date, content_html, attachments)"""
    soup = BeautifulSoup(html, "html.parser")

    # 标题 (meta ArticleTitle 优先)
    title = ""
    mt = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]+)"', html)
    if mt:
        title = clean_title(mt.group(1))
    if not title:
        h1 = soup.find("h1")
        if h1:
            title = clean_title(h1.get_text(strip=True))

    # 日期
    date = ""
    mp = re.search(r'<meta\s+name="PubDate"\s+content="(\d{4}-\d{2}-\d{2})', html)
    if mp:
        date = mp.group(1)
    if not date:
        dm = re.search(r"日期[:：]\s*(\d{4}-\d{2}-\d{2})", html)
        if dm:
            date = dm.group(1)

    # 正文容器
    cm = soup.find("div", id="zoom")
    if not cm:
        return title, date, "", []

    # 去掉噪声
    for s in cm.find_all("script"):
        s.decompose()
    for st in cm.find_all("style"):
        st.decompose()

    # 附件收集: 正文内 a (data-link 真实路径优先) + 正文外 a.articleattach + pdfUrls JS
    attachments = []
    seen_att = set()
    # pdfUrls JS 变量 (纯PDF型正文) 先收集（名称为真实文件名）
    pdf_m = re.findall(r"pdfUrls\.push\(\{name:\s*'([^']*)',\s*fileURL:\s*'([^']*)'\}\)", html)
    pdf_map = {}
    for fname, furl in pdf_m:
        abs_url = urljoin(page_url, furl)
        pdf_map[abs_url] = fname.strip()
    for abs_url, fname in pdf_map.items():
        if abs_url not in seen_att:
            seen_att.add(abs_url)
            attachments.append((None, fname, abs_url))
    # 正文内 <a> 附件
    for a in cm.find_all("a", href=True):
        txt = a.get_text(strip=True)
        href = a.get("data-link") or a["href"]
        if re.search(r"(?i)\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|et|ofd)(\?|$)", href) or "download" in href.lower() or "attach" in href.lower() or txt.startswith("附件"):
            abs_url = urljoin(page_url, href)
            if abs_url in seen_att:
                continue
            seen_att.add(abs_url)
            attachments.append((a, txt, abs_url))
    # 正文外 div.other > a.articleattach
    for a in soup.find_all("a", class_="articleattach", href=True):
        txt = a.get_text(strip=True) or a.get("title", "") or "附件下载"
        href = a.get("data-link") or a["href"]
        if "attach" in href.lower() or "download" in href.lower() or re.search(r"(?i)\.(pdf|doc|docx|xls|xlsx)", href):
            abs_url = urljoin(page_url, href)
            if abs_url in seen_att:
                continue
            seen_att.add(abs_url)
            attachments.append((a, txt, abs_url))

    # 替换正文内附件链接为绝对URL (文件名用 pdfUrls 真实名优先)
    for a, txt, abs_url in attachments:
        if a is None:
            continue
        real_name = pdf_map.get(abs_url, txt)
        new_a = soup.new_tag("a", href=abs_url, target="_blank")
        new_a.string = real_name if real_name else abs_url.split("/")[-1]
        if a.parent is not None:
            for sib in a.parent.find_all("img", src=re.compile(r"(?i)\.(gif|png|jpg|jpeg)")):
                if sib in list(a.parent.contents)[:list(a.parent.contents).index(a)]:
                    sib.decompose()
        a.replace_with(new_a)
    attached_hrefs = {abs_url for _, _, abs_url in attachments}

    # 正文段落提取
    parts = []
    for el in cm.find_all(recursive=False):
        if el.name == "table":
            parts.append(str(el))
        elif el.name in ("p", "div", "h1", "h2", "h3", "ul", "ol", "li"):
            txt = el.get_text("", strip=True)
            if not txt:
                continue
            inner_tables = el.find_all("table")
            el_has_attach = any(
                a for a in el.find_all("a", href=True)
                if urljoin(page_url, a["href"]) in attached_hrefs
                or a["href"] in attached_hrefs
            )
            if inner_tables or el_has_attach:
                parts.append(str(el))
                continue
            parts.append(txt)
        else:
            txt = el.get_text("", strip=True)
            if txt:
                parts.append(txt)

    # 段落去重
    seen = set()
    final_parts = []
    for p in parts:
        key = re.sub(r"\s+", "", re.sub(r"<[^>]+>", "", p))
        if key and key not in seen:
            seen.add(key)
            final_parts.append(p)

    # \n\n 分段; 未入流的附件追加
    content = "\n\n".join(final_parts)
    for a, txt, abs_url in attachments:
        if abs_url not in content:
            content += f'\n\n<p><a href="{abs_url}" target="_blank">{txt}</a></p>'

    # 纯PDF型: zoom 内无正文段落但有PDF附件 → 占位正文
    if not final_parts and attachments:
        pdf_links = [f'<p><a href="{abs_url}" target="_blank">{txt}</a></p>' for _, txt, abs_url in attachments]
        if pdf_links:
            content = "[本公告正文为PDF附件，请下载查看]\n\n" + "\n\n".join(pdf_links)

    # 清理
    content = re.sub(r"打印本页[^\n]*", "", content)
    content = re.sub(r"\n{3,}", "\n\n", content).strip()
    return title, date, content, attachments


def main():
    max_pages = 1
    args = sys.argv[1:]
    i = 0
    while i < len(args):
        a = args[i]
        if a.startswith("--pages="):
            max_pages = int(a.split("=", 1)[1])
        elif a == "--pages" and i + 1 < len(args):
            max_pages = int(args[i + 1])
            i += 1
        i += 1

    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    total_new = 0
    total_dup = 0
    total_skip = 0

    for page in range(1, max_pages + 1):
        html = fetch_list_page(page)
        if not html:
            print(f"  [WARN] 第{page}页获取失败, 跳过", file=sys.stderr)
            continue
        items = parse_list(html)
        print(f"  第{page}页: 找到 {len(items)} 条")
        for url, title, date in items:
            c.execute("SELECT id FROM gov_raw WHERE page_url=?", (url,))
            if c.fetchone():
                total_dup += 1
                continue
            dhtml = http_get(url)
            if not dhtml:
                total_skip += 1
                continue
            d_title, d_date, content, attachments = extract_detail(dhtml, url)
            if not content or len(re.sub(r"<[^>]+>", "", content).strip()) < 10:
                total_skip += 1
                continue
            if not d_title:
                d_title = title
            if not d_date:
                d_date = date
            summary = re.sub(r"<[^>]+>", "", content)[:200]
            summary = re.sub(r"\s+", " ", summary).strip()
            has_table = 1 if "<table" in content else 0
            try:
                cur = c.execute(
                    "INSERT INTO gov_raw (title, summary, content, page_url, source_url, publish_date, site_name, script_name, group_name, has_table, date_rank) VALUES (?,?,?,?,?,?,?,?,?,?,?)",
                    (d_title, summary, content, url, url, d_date, SITE_NAME, SCRIPT_NAME, GROUP_NAME, has_table, 0))
                rid = cur.lastrowid
                c.execute("INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)",
                          (rid, d_title, SITE_NAME, summary))
                conn.commit()
                total_new += 1
                print(f"    [{d_date}] {d_title[:45]}")
            except sqlite3.IntegrityError:
                total_dup += 1
            time.sleep(0.4)

    conn.close()
    print(f"\n新增: {total_new}  重复: {total_dup}  过短/404: {total_skip}  总计: {total_new+total_dup+total_skip}")


if __name__ == "__main__":
    main()