#!/usr/bin/env python3
"""
奇台县人民政府 - 通知公告
URL: https://www.xjqt.gov.cn/p232/tzgg.html
CMS: BZRK CMS (REST API)
List: GET /content/page?channelIds=9086&page=N&size=15&orderBy=27
Detail: Content (txt) embedded in API response - no separate detail fetch needed!
"""
import sys, os, re, json, time, sqlite3, html as html_mod
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

BASE = "https://www.xjqt.gov.cn"
API_URL = BASE + "/content/page"
SITE = "奇台县人民政府-通知公告"
COLUMN = "通知公告"
PROVINCE = "新疆"
PER_PAGE = 15
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": BASE + "/p232/tzgg.html",
}
session = requests.Session()
session.headers.update(HEADERS)
session.verify = False


def log(msg):
    print(msg, file=sys.stderr, flush=True)


def fetch_api(page=1, size=15):
    """调用API获取列表数据，正文内容已包含在txt字段中"""
    params = {"channelIds": "9086", "page": page, "size": size, "orderBy": 27}
    try:
        r = session.get(API_URL, params=params, timeout=30)
        r.encoding = "utf-8"
        data = r.json()
        if data.get("code") == 200:
            return data.get("data", {})
        log(f"  [WARN] API返回码: {data.get('code')}")
        return None
    except Exception as e:
        log(f"  [ERR] API请求失败: {e}")
        return None


def extract_text_from_html(html_text):
    """从HTML正文中提取纯文本，保留段落和表格"""
    if not html_text:
        return ""
    soup = BeautifulSoup(html_text, "html.parser")

    parts = []
    # 提取段落文本
    for p in soup.find_all("p"):
        p_text = p.get_text("", strip=True)
        # 跳过纯空格/nbsp的段落
        if p_text and len(p_text.strip("\u00a0 ")) > 2:
            parts.append(p_text.strip("\u00a0 "))

    # 提取表格（保留HTML）
    for tbl in soup.find_all("table"):
        tbl_html = str(tbl)
        parts.append(f"\n[表格]\n{tbl_html}\n[/表格]")

    if parts:
        return "\n\n".join(parts)
    # 兜底：纯文本
    text = soup.get_text(" ", strip=True)
    return text


def extract_attachments(html_text, base_url=BASE):
    """从HTML正文中提取附件链接"""
    if not html_text:
        return []
    soup = BeautifulSoup(html_text, "html.parser")
    attachments = []
    for a in soup.find_all("a", href=True):
        href = a["href"]
        fname = a.get_text(strip=True)
        if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar|txt)$', href, re.I):
            full_url = urljoin(base_url, href)
            attachments.append({"name": fname or "附件", "url": full_url})
        elif re.search(r'/upload|/cms/', href, re.I) and re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', href, re.I):
            full_url = urljoin(base_url, href)
            attachments.append({"name": fname or "附件", "url": full_url})
    return attachments


def import_to_db(record):
    try:
        db = sqlite3.connect(DB_PATH, timeout=10)
        title = (record.get("title") or "")[:500]
        page_url = (record.get("page_url") or "")[:1000]
        content = record.get("content") or ""
        publish_date = (record.get("publish_date") or "")[:20]
        site_name = (record.get("site_name") or "unknown")[:100]
        attachments_str = json.dumps(record.get("attachments") or [], ensure_ascii=False)

        old = db.execute("SELECT rowid FROM gov_raw WHERE page_url = ?", (page_url,)).fetchone()
        if old:
            db.execute("DELETE FROM gov_search WHERE rowid = ?", (old[0],))
            db.execute("DELETE FROM gov_raw WHERE page_url = ?", (page_url,))

        db.execute(
            "INSERT INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?)",
            (title, page_url, content, publish_date, site_name, page_url, attachments_str)
        )
        new_rowid = db.execute("SELECT last_insert_rowid()").fetchone()[0]
        summary = content[:500] if content else title[:500]
        db.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
                   (new_rowid, title, site_name, summary))
        db.commit()
        db.close()
        log(f"  ✅ {title[:30]}")
        return True
    except Exception as e:
        log(f"  [ERR] DB import failed: {e}")
        return False


def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=5,
                        help="爬取前N页（每页15条，默认5页=75条）")
    args = parser.parse_args()

    count = 0

    for page in range(1, args.pages + 1):
        log(f"📄 列表[page={page}]")
        data = fetch_api(page=page, size=PER_PAGE)
        if not data:
            log(f"  → 无数据，结束")
            break

        items = data.get("content", [])
        if not items:
            log(f"  → 无更多数据")
            break
        log(f"  → {len(items)} 条")

        for idx, item in enumerate(items, 1):
            title = item.get("title", "").strip()
            url = item.get("urlWhole", "") or item.get("url", "")
            release_time = (item.get("releaseTime") or "")[:10]
            txt_html = item.get("txt") or ""

            log(f"  ({idx}/{len(items)}) {title[:40]}")

            # 从txt HTML中提取正文
            content = extract_text_from_html(txt_html)

            # 提取附件
            attachments = extract_attachments(txt_html)

            # 检查附件中是否有图片
            if not content and txt_html:
                soup = BeautifulSoup(txt_html, "html.parser")
                imgs = soup.find_all("img")
                if imgs:
                    content = f"[本公告为图片格式，共{len(imgs)}张附图]"
                    for img in imgs:
                        src = img.get("src", "")
                        if src:
                            full_src = urljoin(BASE, src)
                            attachments.append({"name": img.get("alt", "正文附图"), "url": full_src})

            if not content:
                content = f"[无正文内容] 原文链接：{url}"

            record = {
                "title": title,
                "page_url": url,
                "publish_date": release_time,
                "content": content,
                "attachments": attachments,
                "site_name": SITE,
                "column": COLUMN,
                "province": PROVINCE,
            }
            ok = import_to_db(record)
            if ok:
                count += 1
            time.sleep(0.3)

    log(f"\n✅ {SITE} 爬取完成，共入库 {count} 条")


if __name__ == "__main__":
    main()