#!/usr/bin/env python3
"""
宁城县 (ningchengxian.gov.cn) - 通知公告
http://www.ningchengxian.gov.cn/zwgk/zwdt/tzgg/
分页: index.html (p1), index_1.html (p2), ... index_{N-1}.html
详情: ./202606/t20260604_2774927.html
"""
import sys, os, re, time
sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import fetch_page, clean_html

SITE_NAME = "宁城县通知公告"
LIST_BASE = "http://www.ningchengxian.gov.cn/zwgk/zwdt/tzgg"
MAX_PAGES = 5  # 从JS看出共34页
THREE_YEARS_AGO = "2023-06-01"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")

import sqlite3
import os

def insert_to_db(items, label="nx"):
    if not items:
        print("  ⏭ 无数据")
        return
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            db.execute("""INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, category, tags)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (
                item.get("site_name","")[:200],
                item.get("url",""),
                item.get("url",""),
                (item.get("title") or "")[:500],
                (item.get("pub_date") or "")[:10],
                (item.get("summary") or "")[:500],
                item.get("content",""),
                "active", "", (item.get("tags") or "")[:100],
            ))
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except:
            skip += 1
    db.commit()
    db.execute("INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (SITE_NAME,))
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}, FTS已同步")

def get_list_url(page):
    if page == 1:
        return f"{LIST_BASE}/index.html"
    return f"{LIST_BASE}/index_{page-1}.html"

def parse_list(html):
    """提取列表页的 (title, url, date_str)"""
    items = []
    # <li><a href="./202606/...html"><span>2026-06-04</span>标题</a></li>
    pattern = re.compile(
        r'<a[^>]*href="(\./\d{6}/t[^"]+\.html)"[^>]*>\s*'
        r'<span>(\d{4}-\d{2}-\d{2})</span>\s*'
        r'([\s\S]+?)</a>',
        re.DOTALL
    )
    for m in pattern.finditer(html):
        href = LIST_BASE + "/" + m.group(1).lstrip("./")
        date_str = m.group(2)
        title = re.sub(r'<[^>]+>', '', m.group(3)).strip()
        if title:
            items.append((title, href, date_str))
    return items

def fetch_detail(url):
    """获取详情页的标题、正文、日期"""
    html = fetch_page(url, encoding='utf-8')
    if not html:
        return None, None, None
    # 标题
    title = ""
    m = re.search(r'<h3[^>]*>(.*?)</h3>', html, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    # 日期
    pub_date = ""
    m = re.search(r'<meta[^>]*name=["\']PubDate["\'][^>]*content=["\'](\d{4}-\d{2}-\d{2})', html, re.I)
    if m:
        pub_date = m.group(1)
    # 正文
    content = ""
    m = re.search(r'<div\s+class="content"\s+id="fsz"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL|re.I)
        # Remove trs_editor_view wrapper
        content = re.sub(r'<div\s+class="trs_editor_view[^"]*"[^>]*>', '', content)
        content = re.sub(r'\s*style="[^"]*"', '', content)
        content = clean_html(content)
    return title or None, content or None, pub_date or None

def main():
    print(f"\n{'='*50}\n🏠 {SITE_NAME}\n{'='*50}")
    # 1. 爬列表页
    all_list = []
    for page in range(1, MAX_PAGES + 1):
        url = get_list_url(page)
        print(f"\n📄 第 {page} 页 ({url})...", end=" ", flush=True)
        html = fetch_page(url, encoding='utf-8')
        if not html:
            print("❌ 请求失败")
            break
        items = parse_list(html)
        if not items:
            print("0 条")
            break
        print(f"✅ {len(items)} 条")
        all_list.extend(items)
        # Check if last page based on items count
        if len(items) < 15:
            print("  (末页)")
            break
    
    print(f"\n📊 列表总计: {len(all_list)} 条")
    if not all_list:
        print("⚠️ 无数据")
        return
    
    # 2. 爬详情
    all_items, seen = [], set()
    for i, (title, url, date_str) in enumerate(all_list, 1):
        if url in seen:
            continue
        seen.add(url)
        if date_str and date_str < THREE_YEARS_AGO:
            continue
        
        print(f"  [{i}/{len(all_list)}] {title[:50]}...", end=" ", flush=True)
        dt, content, pub_date = fetch_detail(url)
        if dt:
            title = dt
        if pub_date:
            date_str = pub_date
        
        summary = re.sub(r'<[^>]+>', ' ', content or '').strip()[:300]
        summary = re.sub(r'\s+', ' ', summary)
        
        all_items.append({
            "site_name": SITE_NAME, "title": title, "url": url,
            "content": content or "", "pub_date": date_str or "",
            "summary": summary, "tags": SITE_NAME,
        })
        print(f"✅ {date_str}")
        time.sleep(0.3)
    
    # 3. 入库
    if all_items:
        insert_to_db(all_items, "ningchengxian")
    print(f"\n✅ 完成! 共 {len(all_items)} 条")

if __name__ == "__main__":
    t0 = time.time()
    main()
    print(f"⏱ 耗时: {time.time()-t0:.1f}s")