#!/usr/bin/env python3
"""
乌兰煤炭集团 - 公示公告
URL: https://www.wlmtjt.com/cms/column/index/id/14.html
CMS: 自定义CMS
List: div.div_p > p > span(日期) + a(标题)
Detail: div.div_nry > h2(标题) + h5#txtContent(正文图片)
"""
import sys, os, re, json, time, sqlite3, html
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

BASE = "https://www.wlmtjt.com"
LIST_URL = BASE + "/cms/column/index/id/14.html"
SITE = "乌兰煤炭集团"
COLUMN = "公示公告"
PROVINCE = "内蒙古"
TOTAL_PAGES = 5

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)
session.verify = False

def log(msg):
    print(msg, file=sys.stderr, flush=True)

def fetch(url, retries=3):
    for attempt in range(retries):
        try:
            r = session.get(url, timeout=30)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            log(f"  [WARN] 请求失败 (尝试 {attempt+1}/{retries}): {e}")
            time.sleep(2)
    return ""

def parse_list_html(html):
    """从列表页解析标题、日期、URL"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    div_p = soup.find("div", class_="div_p")
    if not div_p:
        return items
    for p in div_p.find_all("p"):
        span = p.find("span")
        a = p.find("a", href=re.compile(r"/cms/document/detail/id/"))
        if a and span:
            title = a.get_text(strip=True)
            date_str = span.get_text(strip=True)
            href = urljoin(BASE, a["href"])
            if title and date_str:
                items.append({"url": href, "title": title, "date": date_str[:10]})
    return items

def fetch_detail(detail_url, list_title):
    """提取详情页正文"""
    html = fetch(detail_url)
    if not html:
        return "", "", list_title, []
    soup = BeautifulSoup(html, "html.parser")
    nry = soup.find("div", class_="div_nry")
    if not nry:
        return "", "", list_title, []

    # 标题：h2
    h2 = nry.find("h2")
    full_title = h2.get_text(strip=True) if h2 else list_title

    # 日期：h3中的相对时间转实际日期
    date_str = list_title  # placeholder, use from list page
    h3 = nry.find("h3")
    if h3:
        m = re.search(r"发布时间：(\d+)\s*天前", h3.get_text())
        if m:
            days_ago = int(m.group(1))
            from datetime import datetime, timedelta
            date_str = (datetime.now() - timedelta(days=days_ago)).strftime("%Y-%m-%d")

    # 正文：h5#txtContent
    # 按直接子元素（<p>等）分别取文本再 \\n\\n 连接，保留段落结构
    h5 = nry.find("h5", id="txtContent")
    if h5:
        parts = []
        for child in h5.children:
            if child.name and child.name == 'p':
                text = child.get_text("", strip=True)
                if text:
                    parts.append(text)
            elif child.name:
                text = child.get_text("", strip=True)
                if text:
                    parts.append(text)
        content = "\n\n".join(parts) if parts else ""
    else:
        content = nry.get_text("", strip=True)
    # 去掉h3头部（发布时间/字号行）
    content = re.sub(r'发布时间.*?(?:字号|大中小).*?[。]', '', content, flags=re.DOTALL)
    content = content.strip()

    # 如果有图片，记录为附件
    attachments = []
    for img in nry.find_all("img"):
        src = img.get("src", "")
        if src:
            attachments.append({"name": "公告附图", "url": urljoin(BASE, src)})

    if not content and attachments:
        content = f"[本公告为图片格式，附件{len(attachments)}张]"
    elif attachments:
        content += f"\n\n[公告附图{len(attachments)}张]"

    return full_title, content, date_str, attachments

def import_to_db(record):
    try:
        db = sqlite3.connect(DB_PATH, timeout=10)
        title = (record.get("title") or "")[:500]
        page_url = (record.get("page_url") or "")[:1000]
        content = record.get("content") or ""
        publish_date = (record.get("publish_date") or "")[:20]
        site_name = (record.get("site_name") or "unknown")[:100]
        attachments_str = json.dumps(record.get("attachments") or [], ensure_ascii=False)

        old_rowids = db.execute("SELECT rowid FROM gov_raw WHERE page_url = ?", (page_url,)).fetchall()
        for (rid,) in old_rowids:
            db.execute("DELETE FROM gov_search WHERE rowid = ?", (rid,))

        db.execute(
            "INSERT OR REPLACE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments, script_name) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?, 'crawl_wlmt.py')",
            (title, page_url, content, publish_date, site_name, page_url, attachments_str)
        )
        new_rowid = db.execute("SELECT last_insert_rowid()").fetchone()[0]
        summary = content[:500] if content else title[:500]
        # 2026-09-22: 先提交 gov_raw —— 库上触发器已维护 FTS，下面这条手动写入会因
        #   rowid 重复而 IntegrityError；不先 commit 会把 gov_raw 那条一并回滚（静默丢数据）
        db.commit()
        db.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)",
                   (new_rowid, title, site_name, summary))
        db.commit()
        db.close()
        log(f"  ✅ {title[:30]}")
        return True
    except Exception as e:
        log(f"  [ERR] DB import failed: {e}")
        return False

def main():
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--pages", type=int, default=TOTAL_PAGES)
    args = parser.parse_args()

    total_pages = min(args.pages, TOTAL_PAGES)
    count = 0

    for page in range(1, total_pages + 1):
        page_url = LIST_URL if page == 1 else f"{LIST_URL}?page={page}"
        log(f"📄 列表[{page}]: {page_url}")
        html = fetch(page_url)
        if not html:
            break
        items = parse_list_html(html)
        if not items:
            log(f"  → 无更多数据")
            break
        log(f"  → {len(items)} 条")

        for idx, item in enumerate(items, 1):
            url = item["url"]
            list_title = item["title"]
            log(f"  ({idx}/{len(items)}) 详情: {url.split('/')[-1]}")
            full_title, content, list_date, attachments = fetch_detail(url, list_title)
            item["title"] = full_title or list_title
            item["content"] = content
            item["date"] = list_date or item["date"]

            record = {
                "title": item["title"],
                "page_url": url,
                "publish_date": item["date"],
                "content": item["content"],
                "attachments": attachments,
                "site_name": SITE,
                "column": COLUMN,
                "province": PROVINCE,
            }
            ok = import_to_db(record)
            if ok:
                count += 1
            time.sleep(0.5)

    log(f"\n✅ {SITE}-{COLUMN} 爬取完成，共入库 {count} 条")

if __name__ == "__main__":
    main()