#!/usr/bin/env python3
"""
孟州市 (mengzhou.gov.cn) - 公示公告
https://www.mengzhou.gov.cn/xwdt/gsgg/
分页: index.html (p1), index_2.html (p2), ... index_{N}.html
详情: /YYYY/MM-DD/NUMBER.html
"""
import sys, os, re, time
sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import fetch_page, clean_html

SITE_NAME = "孟州市公示公告"
BASE = "https://www.mengzhou.gov.cn"
LIST_PREFIX = "https://www.mengzhou.gov.cn/xwdt/gsgg"
MAX_PAGES = 5  # 1408条, 24条/页
THREE_YEARS_AGO = "2023-06-01"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

import sqlite3
import os

def insert_to_db(items, label="mz"):
    if not items:
        print("  ⏭ 无数据")
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, category, tags)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (
                item.get("site_name","")[:200],
                item.get("url",""),
                item.get("url",""),
                (item.get("title") or "")[:500],
                (item.get("pub_date") or "")[:10],
                (item.get("summary") or "")[:500],
                item.get("content",""),
                "active", "", (item.get("tags") or "")[:100],
            ))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    db.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (SITE_NAME,))
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")

def get_list_url(page):
    if page == 1:
        return f"{LIST_PREFIX}/index.html"
    return f"{LIST_PREFIX}/index_{page}.html"

def parse_list(html):
    """提取列表页的 (title, url, date_str)"""
    items = []
    # <a class="colRtitle" href="https://www.mengzhou.gov.cn/2026/..."><em>·</em> 标题<span>[2026-06-01]</span></a>
    pattern = re.compile(
        r'<a[^>]*class="colRtitle"[^>]*href="(https://www\.mengzhou\.gov\.cn/\d{4}/\d{2}-\d{2}/\d+\.html)"[^>]*>'
        r'<em>·</em>\s*(.+?)<span>\[(\d{4}-\d{2}-\d{2})\]</span>',
        re.DOTALL
    )
    for m in pattern.finditer(html):
        href = m.group(1)
        title = m.group(2).strip()
        date_str = m.group(3)
        if title:
            items.append((title, href, date_str))
    return items

def _extract_div(html, tag, class_name):
    """提取指定class的元素内HTML，正确处理嵌套标签"""
    pattern = rf'<{tag}\s+class="{class_name}"[^>]*>'
    m = re.search(pattern, html)
    if not m:
        return None
    start = m.end()
    depth = 1
    i = start
    while i < len(html) and depth > 0:
        next_open = re.search(rf'<{tag}(\s|>)', html[i:])
        next_close = re.search(rf'</{tag}>', html[i:])
        if not next_close:
            break
        if next_open and next_open.start() < next_close.start():
            depth += 1
            i += next_open.end()
        else:
            depth -= 1
            i += next_close.end()
    return html[start:i-6]

def fetch_detail(url):
    """获取详情页的标题、正文、日期"""
    html = fetch_page(url, encoding='utf-8')
    if not html:
        return None, None, None
    # 标题
    title = ""
    m = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
    if m:
        title = m.group(1).strip()
        title = re.sub(r'\s*[-–—]\s*孟州市人民政府\s*$', '', title)
    # 日期
    pub_date = ""
    m = re.search(r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\'](\d{4}-\d{2}-\d{2})', html, re.I)
    if m:
        pub_date = m.group(1)
    # 正文 — div深度计数解决嵌套问题
    content = ""
    raw = _extract_div(html, 'div', 'cms-article-box')
    if raw:
        content = raw.strip()
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'\s*style="[^"]*"', '', content)
        # 只保留article-detail正文部分，去掉标题/信息栏/分享等
        detail_m = re.search(r'<div class="article-detail">(.*?)</div>', content, re.DOTALL)
        if detail_m:
            content = detail_m.group(1)
        content = clean_html(content)
    return title or None, content or None, pub_date or None

def main():
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")
    # 1. 爬列表页
    all_list = []
    for page in range(1, MAX_PAGES + 1):
        url = get_list_url(page)
        print(f"\n📄 第 {page} 页 ({url})...", end=" ", flush=True)
        html = fetch_page(url, encoding='utf-8')
        if not html:
            print("❌ 请求失败")
            break
        items = parse_list(html)
        if not items:
            print("0 条")
            break
        print(f"✅ {len(items)} 条")
        all_list.extend(items)
        if len(items) < 24:
            print("  (末页)")
            break
    
    print(f"\n📊 列表总计: {len(all_list)} 条")
    if not all_list:
        print("⚠️ 无数据")
        return
    
    # 2. 爬详情
    all_items, seen = [], set()
    for i, (title, url, date_str) in enumerate(all_list, 1):
        if url in seen:
            continue
        seen.add(url)
        if date_str and date_str < THREE_YEARS_AGO:
            continue
        
        print(f"  [{i}/{len(all_list)}] {title[:50]}...", end=" ", flush=True)
        dt, content, pub_date = fetch_detail(url)
        if dt:
            title = dt
        if pub_date:
            date_str = pub_date
        
        summary = re.sub(r'<[^>]+>', ' ', content or '').strip()[:300]
        summary = re.sub(r'\s+', ' ', summary)
        
        all_items.append({
            "site_name": SITE_NAME, "title": title, "url": url,
            "content": content or "", "pub_date": date_str or "",
            "summary": summary, "tags": SITE_NAME,
        })
        print(f"✅ {date_str}")
        time.sleep(0.3)
    
    # 3. 入库
    if all_items:
        insert_to_db(all_items, "mengzhou")
    print(f"\n✅ 完成! 共 {len(all_items)} 条")

if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")