#!/usr/bin/env python3
"""
二连浩特市人民政府 - 通知公告
https://www.elht.gov.cn/ywdt/tzgg/
ZCMS 架构 - 静态分页 index_N.shtml (pages 2-10), zcms/ui/catalog/25/pc/index_N.shtml (11+)
"""
import sys, os, re, time, json, sqlite3
from datetime import datetime, timezone, timedelta
import requests, urllib3
import os
urllib3.disable_warnings()

SITE_NAME = "二连浩特通知公告"
BASE_URL = "https://www.elht.gov.cn"
LIST_BASE = "https://www.elht.gov.cn/ywdt/tzgg"
ZCMS_URL_T = "https://www.elht.gov.cn/zcms/ui/catalog/25/pc/index_{n}.shtml"
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
MAX_PAGES = 5  # 用户策略: 最多5页
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/125.0.0.0"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"\n❌ 请求失败 {url}: {e}", flush=True)
        return None

def parse_list(html):
    """抽取列表页中的文章链接和日期"""
    items = []
    # <li><a href="/c/2026-06-12/115124.shtml" title="...">title</a><span>2026-06-12</span></li>
    for m in re.finditer(r'<li>\s*<a\s+href="(/c/[^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>\s*<span>(\d{4}-\d{2}-\d{2})</span>', html):
        href = m.group(1)
        title = m.group(2).strip() or m.group(3).strip()
        date = m.group(4)
        items.append((title, BASE_URL + href, date))
    return items

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return "", "", ""
    title = ""
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]+)"', html)
    if m:
        title = m.group(1)
    if not title:
        m = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.S)
        if m:
            title = re.sub(r'<[^>]+>', '', m.group(1)).strip()
    pub_date = ""
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})', html)
    if m:
        pub_date = m.group(1)
    if not pub_date:
        m = re.search(r'(\d{4}-\d{2}-\d{2})\s*\d{2}:\d{2}', html)
        if m:
            pub_date = m.group(1)
    content = ""
    m = re.search(r'<div\s+class="cont"\s+id="Zoom">(.*?)</div>\s*<br>\s*<div\s+id="fujian"', html, re.S)
    if m:
        content = m.group(1).strip()
    if not content or len(content) < 50:
        # Fallback: look for cont div
        m = re.search(r'<div\s+class="cont"\s+id="Zoom">(.*?)</div>\s*</div>', html, re.S)
        if m:
            content = m.group(1).strip()
    if content:
        content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.S|re.I)
        content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.S|re.I)
    return title, pub_date, content

def insert_to_db(items):
    if not items:
        return
    db = sqlite3.connect(SEARCH_DB)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")
    ok, skip = 0, 0
    for item in items:
        try:
            db.execute("INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?,?,?,?,?,?,?,?,?,?)", (
                item.get("site_name","")[:200], item.get("url",""),
                item.get("url",""), (item.get("title") or "")[:500],
                (item.get("pub_date") or "")[:10],
                (item.get("summary") or "")[:500],
                item.get("content",""), "active", "",
                (item.get("tags") or "")[:100],
            ))
            if db.total_changes > 0: ok += 1
            else: skip += 1
        except: skip += 1
    db.commit()
    site_name = items[0].get("site_name", "")
    db.execute("INSERT INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?", (site_name,))
    db.commit()
    db.close()
    print(f"\n💾 入库: 新增{ok}, 跳过{skip}")

def main(incremental=False):
    t0 = time.time()
    print(f"\n{SITE_NAME}\n{'='*40}")
    
    page_limit = 1 if incremental else MAX_PAGES
    all_items = []
    
    for page in range(1, page_limit + 1):
        if page == 1:
            url = f"{LIST_BASE}/index.shtml"
        elif page <= 10:
            url = f"{LIST_BASE}/index_{page}.shtml"
        else:
            url = ZCMS_URL_T.format(n=page)
        
        print(f"\n📄 第{page}页: {url}")
        html = fetch(url)
        if not html:
            print(f"  ❌ 获取失败")
            continue
        
        page_items = parse_list(html)
        print(f"  📊 {len(page_items)} 条")
        
        # 过滤日期
        for title, href, date in page_items:
            if date and date < CUTOFF:
                continue
            all_items.append((title, href, date))
        
        if not page_items:
            print("  ⏭ 空页, 停止翻页")
            break
        
        time.sleep(0.5)
    
    print(f"\n📊 列表总计: {len(all_items)} 条 (已过滤超3年)")
    
    seen = set()
    results = []
    filtered = 0
    for i, (title, url, list_date) in enumerate(all_items):
        if url in seen:
            continue
        seen.add(url)
        print(f"  [{len(results)+1}] {title[:50]}...", end=" ", flush=True)
        dt, date, content = fetch_detail(url)
        if not content or len(content) < 50:
            print("⏭ 无正文")
            continue
        final_title = dt or title
        final_date = date or list_date
        if final_date and final_date < CUTOFF:
            filtered += 1
            print("⏭ 超3年")
            continue
        summary = re.sub(r'<[^>]+>', '', content)[:200].strip()
        results.append({"site_name": SITE_NAME, "title": final_title,
            "url": url, "content": content, "summary": summary,
            "pub_date": final_date, "tags": SITE_NAME})
        print(f"✅ ({len(content)}字)")
        time.sleep(0.3)
    
    if filtered:
        print(f"\n⏭ 跳过超3年(详情): {filtered} 条")
    
    if results:
        insert_to_db(results)
    print(f"\n✅ 完成! 共 {len(results)} 条\n⏱ {time.time()-t0:.1f}s")

if __name__ == "__main__":
    incremental = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    main(incremental=incremental)