#!/usr/bin/env python3
"""
武威市人民政府 - 各部门政策文件
URL: https://www.gswuwei.gov.cn/col/col1003/index.html?number=WWSA1533
CMS: 大汉网络 (Hanweb) + JPage AJAX分页
List API: POST /module/web/jpage/dataproxy.jsp?startrecord=N&endrecord=M&perpage=15
  Body: col=1&webid=1&path=...&columnid=1003&unitid=5827&webname=武威市人民政府
Detail: div#Zoom > p
"""
import sys, os, re, json, time, sqlite3
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

BASE = "https://www.gswuwei.gov.cn"
LIST_URL = BASE + "/col/col1003/index.html?number=WWSA1533"
SITE = "武威市人民政府-各部门政策文件"
COLUMN = "各部门政策文件"
PROVINCE = "甘肃"
PER_PAGE = 15
GROUP_SIZE = 45  # 3 pages per group in JPage

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)
session.verify = False

# JPage API params
JPAGE_PARAMS = {
    "col": 1,
    "webid": 1,
    "path": BASE + "/",
    "columnid": 1003,
    "sourceContentType": 1,
    "unitid": "5827",
    "webname": "武威市人民政府",
    "permissiontype": 0,
}
JPAGE_URL = BASE + "/module/web/jpage/dataproxy.jsp"


# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 / <br> 处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
#    公文把一行拆成 <span>提取码：</span>pwaj<span>。查阅…</span>，这些行内节点于是各自
#    成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
               'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
               'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
               'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
               'ul', 'center', 'caption'}


def body_text(el):
    """块级边界出换行、行内节点直接拼接、<br> 出换行（≈ 浏览器看到的换行结构）。"""
    if el is None:
        return ''
    import re as _re
    from bs4 import NavigableString
    out = []

    def walk(node):
        for ch in node.children:
            if isinstance(ch, NavigableString):
                out.append(str(ch))
            elif getattr(ch, 'name', None) == 'br':
                out.append('\n')
            elif getattr(ch, 'name', None) in _BLOCK_TAGS:
                out.append('\n')
                walk(ch)
                out.append('\n')
            else:
                walk(ch)
    walk(el)
    t = ''.join(out)
    t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
    t = _re.sub(r'\n{3,}', '\n\n', t)
    return t.strip()


def log(msg):
    print(msg, file=sys.stderr, flush=True)


def fetch(url, retries=3):
    for attempt in range(retries):
        try:
            r = session.get(url, timeout=30)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            log(f"  [WARN] 请求失败 (尝试 {attempt+1}/{retries}): {e}")
            time.sleep(2)
    return ""


def fetch_jpage(startrecord, endrecord):
    """调用JPage API获取列表数据"""
    url = f"{JPAGE_URL}?startrecord={startrecord}&endrecord={endrecord}&perpage={PER_PAGE}"
    headers = {"X-Requested-With": "XMLHttpRequest", "Referer": LIST_URL}
    for attempt in range(3):
        try:
            r = session.post(url, data=JPAGE_PARAMS, headers=headers, timeout=30)
            r.encoding = "utf-8"
            if r.text:
                return r.text
        except Exception as e:
            log(f"  [WARN] JPage请求失败 (尝试 {attempt+1}/3): {e}")
            time.sleep(2)
    return ""


def parse_list_xml(xml_text):
    """从JPage API返回的XML中解析条目"""
    items = []
    records = re.findall(r"<!\[CDATA\[(.*?)\]\]>", xml_text, re.DOTALL)
    for rec in records:
        href_m = re.search(r"href='([^']+)'", rec)
        title_m = re.search(r"title='([^']+)'", rec)
        date_m = re.search(r"<span[^>]*>(\d{4}[-/]\d{2}[-/]\d{2})", rec)
        if href_m and title_m:
            href = href_m.group(1)
            title = title_m.group(1).strip()
            # Strip &middot;&nbsp; prefix
            title = re.sub(r'^·\s*|^&middot;\s*|^\s*', '', title).strip()
            date_str = date_m.group(1) if date_m else ""
            items.append({
                "url": href if href.startswith("http") else urljoin(BASE, href),
                "title": title,
                "date": date_str,
            })
    return items


def fetch_detail(detail_url, list_title):
    """提取详情页正文"""
    html = fetch(detail_url)
    if not html:
        return list_title, "", list_title, []
    soup = BeautifulSoup(html, "html.parser")

    # 标题
    full_title = list_title
    meta_title = soup.find("meta", attrs={"name": "ArticleTitle"})
    if meta_title and meta_title.get("content"):
        full_title = meta_title["content"].strip()
    else:
        tit_div = soup.find("div", class_="con_tit")
        if tit_div:
            t = tit_div.get_text("", strip=True)
            if t:
                full_title = t
    full_title = re.sub(r'^·\s*', '', full_title).strip()

    # 日期 from meta pubdate (修复: 默认空而非list_title, 防标题入publish_date)
    date_str = ""
    meta_date = soup.find("meta", attrs={"name": "pubdate"})
    if meta_date and meta_date.get("content"):
        date_str = meta_date["content"][:10]
    # 兜底: 从URL提取日期 (外部转发链接无meta pubdate)
    if not date_str:
        m_u = re.search(r"t(\d{8})", detail_url)
        if m_u:
            date_str = f"{m_u.group(1)[:4]}-{m_u.group(1)[4:6]}-{m_u.group(1)[6:8]}"
        else:
            m_u2 = re.search(r"/(20\d{2})/(\d{1,2})/(\d{1,2})/", detail_url)
            if m_u2:
                date_str = f"{m_u2.group(1)}-{int(m_u2.group(2)):02d}-{int(m_u2.group(3)):02d}"

    # 正文 div#Zoom
    zoom = soup.find("div", id="Zoom")
    content = ""
    attachments = []
    if zoom:
        parts = []
        for child in zoom.find_all(["p", "div", "table"], recursive=True):
            # Skip meta and empty
            if child.name == "meta":
                continue
            text = child.get_text("", strip=True)
            if text and len(text) > 3:
                parts.append(text)
            # Check for images
            if child.name == "p":
                for img in child.find_all("img"):
                    src = img.get("src", "")
                    if src:
                        attachments.append({"name": "正文附图", "url": urljoin(BASE, src)})
        content = "\n\n".join(parts) if parts else ""
    else:
        content = body_text(zoom) if zoom else ""

    # Zoom 内 iframe PDF (内嵌附件页)
    if not content:
        iframe = zoom.find("iframe") if zoom else None
        if iframe:
            src = iframe.get("src", "")
            if src:
                if not src.startswith("http"):
                    src = urljoin(BASE, src)
                attachments.append({"name": "正文附件(PDF)", "url": src})
                content = f"[本公告正文为PDF附件，共{len(attachments)}个附件，请点击附件查看全文]"

    # 如果有图片但没有文本内容
    if not content and attachments:
        content = f"[本公告为图片格式，附件{len(attachments)}张]"
    elif attachments and not content.startswith("[本公告正文为PDF"):
        content += f"\n\n[公告附图{len(attachments)}张]"

    return full_title, content, date_str, attachments


def import_to_db(record):
    try:
        db = sqlite3.connect(DB_PATH, timeout=10)
        title = (record.get("title") or "")[:500]
        page_url = (record.get("page_url") or "")[:1000]
        content = record.get("content") or ""
        publish_date = (record.get("publish_date") or "")[:20]
        site_name = (record.get("site_name") or "unknown")[:100]
        attachments_str = json.dumps(record.get("attachments") or [], ensure_ascii=False)

        old_rowids = db.execute("SELECT rowid FROM gov_raw WHERE page_url = ?", (page_url,)).fetchall()
        for (rid,) in old_rowids:
            db.execute("DELETE FROM gov_search WHERE rowid = ?", (rid,))

        db.execute(
            "INSERT OR REPLACE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments, script_name) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?, 'crawl_wuwei.py')",
            (title, page_url, content, publish_date, site_name, page_url, attachments_str)
        )
        new_rowid = db.execute("SELECT last_insert_rowid()").fetchone()[0]
        summary = content[:500] if content else title[:500]
        db.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
                   (new_rowid, title, site_name, summary))
        db.commit()
        db.close()
        log(f"  ✅ {title[:30]}")
        return True
    except Exception as e:
        log(f"  [ERR] DB import failed: {e}")
        return False


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=5,
                        help="爬取前N页（每页15条，默认5页=75条）")
    args = parser.parse_args()

    total_needed = args.pages * PER_PAGE  # 5页 = 75条
    groups_needed = (total_needed + GROUP_SIZE - 1) // GROUP_SIZE

    log(f"📋 目标: {args.pages} 页 = {total_needed} 条, 需 {groups_needed} 组")

    all_items = []

    # Step 1: 先请求首页获取初始数据（含第一组）
    log(f"📄 获取首页初始数据...")
    html = fetch(LIST_URL)
    if html:
        xml_match = re.search(r"<datastore>(.*?)</datastore>", html, re.DOTALL)
        if xml_match:
            items = parse_list_xml(xml_match.group(0))
            log(f"  → 首页获取 {len(items)} 条")
            all_items.extend(items)

    # Step 2: 通过API获取更多组
    for g in range(1, groups_needed):
        start = g * GROUP_SIZE + 1
        end = start + GROUP_SIZE - 1
        if start > 5375:  # total records
            break
        log(f"📄 JPage组 {g+1}: records {start}-{end}")
        xml_text = fetch_jpage(start, end)
        if not xml_text:
            break
        items = parse_list_xml(xml_text)
        log(f"  → {len(items)} 条")
        all_items.extend(items)
        if len(items) < GROUP_SIZE // 2:  # 少于半组视为无更多数据
            break
        time.sleep(0.5)

    # 截取需要的数量
    all_items = all_items[:total_needed]
    log(f"\n📋 共获取 {len(all_items)} 条待爬取详情")

    count = 0
    for idx, item in enumerate(all_items, 1):
        url = item["url"]
        list_title = item["title"]
        list_date = item["date"]
        log(f"  ({idx}/{len(all_items)}) 详情: {url.split('/')[-1] or url[-30:]}")
        full_title, content, detail_date, attachments = fetch_detail(url, list_title)

        record = {
            "title": full_title,
            "page_url": url,
            "publish_date": detail_date or list_date,
            "content": content,
            "attachments": attachments,
            "site_name": SITE,
            "column": COLUMN,
            "province": PROVINCE,
        }
        ok = import_to_db(record)
        if ok:
            count += 1
        time.sleep(1)  # 防WAF

    log(f"\n✅ {SITE} 爬取完成，共入库 {count} 条")


if __name__ == "__main__":
    main()