#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
河北唐山南堡经济开发区 - 计划规划
https://www.nanpu.gov.cn/news/37/
portal-saas (fastbuilder/凡科) CMS，API驱动
栏目 detailId: 1628181417801822208 (news/37 计划规划)
Usage:
  python3 crawl_nanpu.py               # 默认5页
  python3 crawl_nanpu.py --pages 10
  python3 crawl_nanpu.py --full         # 全量(18条/页, 1557条约87页)
"""
import sys, os, re, time, json, requests
from bs4 import BeautifulSoup

SEARCH_DB = os.getenv("SEARCH_DB", "/mnt/data/search.db")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
DOMAIN = "https://www.nanpu.gov.cn"
SITE_NAME = "nanpu_jhgh"
CATEGORY_ID = "1628181417801822208"
PAGE_SIZE = 18
TOTAL_ITEMS = 1557
MAX_PAGES = (TOTAL_ITEMS + PAGE_SIZE - 1) // PAGE_SIZE  # ~87
DEFAULT_PAGES = 5


# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def fetch_list(page=1):
    """通过API获取列表数据"""
    url = f"{DOMAIN}/fwebapi/cms/lowcode/60003/18505/list?cate=0"
    payload = {
        "size": PAGE_SIZE,
        "query": [{
            "esField": "DETAIL_ES.es_multi_category_6d5k7017",
            "groupEnd": "1",
            "field": "category_6d5k7017",
            "sourceType": "page",
            "dataType": "array[category]",
            "logic": "and",
            "groupBegin": "1",
            "value": CATEGORY_ID,
            "operator": "in"
        }],
        "from": (page - 1) * PAGE_SIZE,
        "sort": [],
        "_detailId": CATEGORY_ID
    }
    try:
        r = requests.post(url, json=payload, headers={
            **HEADERS,
            "Referer": f"{DOMAIN}/news/37/",
            "Accept": "application/json, text/plain, */*"
        }, timeout=15)
        data = r.json()
        if data.get("status") == "200":
            return data["data"].get("list", []), data["data"].get("page", {})
        return [], {}
    except Exception as e:
        print("  API ERROR: %s" % e, file=sys.stderr)
        return [], {}


def get_detail_content(page_url):
    """获取详情页内容：从HTML中的e_richText-11提取富文本"""
    try:
        r = requests.get(page_url, headers=HEADERS, timeout=15)
        html = r.text
    except Exception:
        return "", "", []

    soup = BeautifulSoup(html, "html.parser")

    # 标题
    title = ""
    t_el = soup.select_one("title")
    if t_el:
        t = t_el.get_text(strip=True)
        t = re.sub(r'[_\-—]\s*河北唐山南堡经济开发区管理委员会.*$', '', t)
        title = t.strip()

    # 日期
    pub_date = ""
    date_el = soup.select_one(".e_timeFormat-28.s_link")
    if date_el:
        dt = date_el.get_text(strip=True)
        m = re.search(r"(\d{4})-(\d{1,2})-(\d{1,2})", dt)
        if m:
            pub_date = "%s-%02d-%02d" % (m.group(1), int(m.group(2)), int(m.group(3)))
    else:
        # Fallback: find date in page
        dm = re.search(r"(\d{4})-(\d{1,2})-(\d{1,2})", html)
        if dm:
            pub_date = "%s-%02d-%02d" % (dm.group(1), int(dm.group(2)), int(dm.group(3)))

    # 正文内容：从 e_richText-11 提取
    content = ""
    attachments = []
    rt = soup.select_one(".e_richText-11")
    if rt:
        # Extract text content
        parts = []
        for el in rt.children:
            tag = el.name
            if tag is None:
                text = str(el).strip()
                if text:
                    parts.append("<p>%s</p>" % text)
            elif tag == "table":
                parts.append(str(el))
            elif tag in ("p", "div"):
                txt = body_text(el)
                if txt and len(txt) > 5:
                    inner = "".join(str(c) for c in el.children if c.name or str(c).strip())
                    parts.append(inner.strip())
            elif tag == "img":
                src = el.get("src", "")
                alt = el.get("alt", "")
                if src:
                    img_url = src if src.startswith("http") else DOMAIN + src
                    parts.append('<p><a href="%s">%s</a></p>' % (img_url, alt or "查看图片"))
            elif tag in ("ul", "ol"):
                for li in el.find_all("li", recursive=False):
                    txt = li.get_text(strip=True)
                    if txt:
                        parts.append("- " + txt)
            elif tag == "figure":
                img = el.find("img")
                if img:
                    src = img.get("src", "")
                    alt = img.get("alt", "")
                    if src:
                        img_url = src if src.startswith("http") else DOMAIN + src
                        parts.append('<p><a href="%s">%s</a></p>' % (img_url, alt or "查看图片"))
                tbl = el.find("table")
                if tbl:
                    parts.append(str(tbl))

        # 兜底：递归找 rt 内所有图片/表格（figure 嵌套等情况）
        if not any(p.startswith("![") for p in parts):
            for img in rt.find_all("img"):
                src = img.get("src", "")
                alt = img.get("alt", "")
                if src:
                    img_url = src if src.startswith("http") else DOMAIN + src
                    parts.append('<p><a href="%s">%s</a></p>' % (img_url, alt or "查看图片"))
        if not any(p.startswith("<table") for p in parts):
            for tbl in rt.find_all("table"):
                parts.append(str(tbl))

        content = "\n\n".join(parts)

        # Extract attachments from links
        for a in rt.find_all("a", href=True):
            href = a["href"]
            if re.search(r'\.(doc|docx|pdf|xls|xlsx|zip|rar)$', href, re.I):
                f_url = href if href.startswith("http") else DOMAIN + href
                f_title = a.get_text(strip=True) or os.path.basename(href)
                if not any(att["url"] == f_url for att in attachments):
                    attachments.append({"url": f_url, "title": f_title})

    # 清理
    content = re.sub(r'\n{3,}', '\n\n', content).strip()
    if not content or len(content) < 20:
        content = '<p><a href="%s">%s</a></p>' % (page_url, title or "详情")

    return content, pub_date, attachments


def extract_content_from_html(richtext_html):
    """从richtext HTML中提取纯文本内容"""
    if not richtext_html:
        return ""
    
    soup = BeautifulSoup(richtext_html, "html.parser")
    
    # Remove scripts and styles
    for tag in soup(["script", "style"]):
        tag.decompose()
    
    content_parts = []
    for el in soup.children:
        tag = el.name
        if tag is None:
            text = str(el).strip()
            if text:
                content_parts.append(text)
        elif tag == "table":
            content_parts.append(str(el))
        elif tag in ("p", "div"):
            if el.get_text(strip=True):
                inner = "".join(str(c) for c in el.children if c.name or str(c).strip())
                content_parts.append(inner.strip())
        elif tag == "br":
            pass
        elif tag == "img":
            src = el.get("src", "")
            alt = el.get("alt", "")
            if src:
                img_url = src if src.startswith("http") else DOMAIN + src
                content_parts.append('<p><a href="%s">%s</a></p>' % (img_url, alt or "查看图片"))
        elif tag in ("ul", "ol"):
            for li in el.find_all("li", recursive=False):
                txt = li.get_text(strip=True)
                if txt:
                    content_parts.append("- %s" % txt)
        elif tag == "a":
            href = el.get("href", "")
            txt = el.get_text(strip=True)
            if href and txt:
                f_url = href if href.startswith("http") else DOMAIN + href
                content_parts.append("[%s](%s)" % (txt, f_url))
    
    content = "\n\n".join(content_parts)
    
    # Clean up
    content = re.sub(r'\s*\n{3,}\s*', '\n\n', content)
    content = content.strip()
    
    return content


def extract_attachments(richtext_html, item_data):
    """从richtext中提取附件链接"""
    attachments = []
    if not richtext_html:
        return attachments
    
    # Extract from rich text
    soup = BeautifulSoup(richtext_html, "html.parser")
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if re.search(r'\.(doc|docx|pdf|xls|xlsx|zip|rar|txt)$', href, re.I):
            f_url = href if href.startswith("http") else DOMAIN + href
            f_title = a.get_text(strip=True) or os.path.basename(href)
            attachments.append({"url": f_url, "title": f_title})
    
    # Also check item_data for attachment field
    attachment_data = item_data.get("attachment_j6q40145", []) if item_data else []
    if isinstance(attachment_data, list):
        for att in attachment_data:
            if isinstance(att, dict):
                url = att.get("url", "") or att.get("src", "")
                if url:
                    f_url = url if url.startswith("http") else DOMAIN + url
                    f_title = att.get("title", "") or att.get("name", "") or os.path.basename(url)
                    if not any(a["url"] == f_url for a in attachments):
                        attachments.append({"url": f_url, "title": f_title})
    
    return attachments


def main():
    pages = DEFAULT_PAGES
    if "--full" in sys.argv:
        pages = MAX_PAGES
    else:
        for i, a in enumerate(sys.argv):
            if a == "--pages" and i + 1 < len(sys.argv):
                try:
                    pages = int(sys.argv[i + 1])
                except ValueError:
                    pass
                break
    pages = min(pages, MAX_PAGES)

    import sqlite3
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    c = conn.cursor()

    all_new = 0
    for pg in range(1, pages + 1):
        print("第%d页 (from=%d)" % (pg, (pg-1)*PAGE_SIZE), file=sys.stderr)
        items, page_info = fetch_list(pg)
        if not items:
            print("  第%d页 无数据" % pg, file=sys.stderr)
            if pg == 1:
                break
            continue

        print("  第%d页: %d 条 (共%s条)" % (pg, len(items), page_info.get("totalCount", "?")), file=sys.stderr)
        total_count = page_info.get("totalCount", 0)

        for it in items:
            # Get URL
            href = it.get("_href", "") or it.get("link_6W72508r", "")
            if not href:
                continue
            page_url = href if href.startswith("http") else DOMAIN + href

            # Check duplicate
            c.execute("SELECT id FROM gov_raw WHERE page_url=? AND site_name=?", (page_url, SITE_NAME))
            if c.fetchone():
                continue

            # Title from API
            title = it.get("text_2PVENH84", "").strip()
            if not title:
                continue

            # Date from API
            raw_date = it.get("prePublishTime", "")
            pub_date = raw_date[:10] if raw_date else ""

            # Get detail content
            content, detail_date, attachments = get_detail_content(page_url)
            if detail_date and not pub_date:
                pub_date = detail_date

            try:
                c.execute(
                    """INSERT INTO gov_raw (site_name, page_url, title, content, publish_date, source_url, status, attachments)
                       VALUES (?,?,?,?,?,?,?,?)""",
                    (SITE_NAME, page_url, title[:500], content, pub_date[:20],
                     page_url, "synced", json.dumps(attachments, ensure_ascii=False))
                )
                all_new += 1
                print("    + %s (%s) body:%dB attach:%d" % (title[:50], pub_date, len(content), len(attachments)), file=sys.stderr)
            except Exception as e:
                print("    ! 入库失败: %s" % e, file=sys.stderr)

            time.sleep(0.3)

        conn.commit()

    print("\n[%s] 完成! 新增 %d 条" % (SITE_NAME, all_new), file=sys.stderr)

    if all_new > 0:
        c.execute("""INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary)
                      SELECT rowid, title, site_name, substr(content,1,500) FROM gov_raw
                      WHERE site_name=? AND rowid NOT IN (SELECT rowid FROM gov_search)""", (SITE_NAME,))
        fts_added = c.execute("SELECT changes()").fetchone()[0]
        conn.commit()
        print("  gov_search 增量更新: %d 条" % fts_added, file=sys.stderr)

    conn.close()


if __name__ == "__main__":
    main()