#!/usr/bin/env python3
"""
大同经开区-公示公告 (kfq.dt.gov.cn)
列表: /dtjjjskfqz/gsgg/listN.shtml — 静态HTML <li data-*>
详情: /dtjjjskfqz/gsgg/YYYYMM/{hash}.shtml
正文: div.TRS_Editor / ucapcontent
标题: <meta name="ArticleTitle">
日期: <meta name="PubDate">
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAMES = {
    "gsgg": "大同经开区-公示公告",
    "qtgs": "大同经开区-其他公告",
}
COLUMN_PATHS = {
    "gsgg": "gsgg/listN",
    "qtgs": "qtgs/list",
}
BASE_URL = "https://kfq.dt.gov.cn"
THREE_YEARS_AGO = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def parse_list(html):
    """从静态HTML提取列表: data-title, data-url, data-time"""
    items = []
    pattern = r'<li[^>]*data-title="([^"]*)"[^>]*data-url="([^"]*)"[^>]*data-time="([^"]*)"[^>]*>|<li[^>]*data-time="([^"]*)"[^>]*data-url="([^"]*)"[^>]*data-title="([^"]*)"[^>]*>'
    for m in re.finditer(pattern, html):
        if m.group(1) and m.group(2) and m.group(3):
            # 格式1: data-title data-url data-time (gsgg)
            title = m.group(1).strip()
            href = m.group(2)
            date_str = m.group(3)
        elif m.group(4) and m.group(5) and m.group(6):
            # 格式2: data-time data-url data-title (qtgs)
            title = m.group(6).strip()
            href = m.group(5)
            date_str = m.group(4)
        else:
            continue
        if not title:
            continue
        if href.startswith('../../'):
            href = href.replace('../../', '/')
        if not href.startswith('http'):
            href = BASE_URL + href
        items.append((title, href, date_str))
    return items

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None

    result = {}

    # 标题
    m = re.search(r'<meta[^>]*name="ArticleTitle"[^>]*content="([^"]+)"', html)
    if m:
        result["title"] = m.group(1).strip()
    if not result.get("title"):
        m = re.search(r'<h1[^>]*>(.*?)</h1>', html, re.DOTALL)
        if m:
            result["title"] = re.sub(r"<[^>]+>", "", m.group(1)).strip()

    # 日期
    m = re.search(r'<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        result["publish_date"] = m.group(1)

    # 正文: 优先 TRS_Editor
    content = None
    m = re.search(r"<div[^>]*class='[^']*TRS_Editor[^']*'[^>]*>(.*?)</div>\s*</div>", html, re.DOTALL)
    if m and len(m.group(1)) > 50:
        content = m.group(1).strip()

    # 备用: ucapcontent (嵌套在TRS_Editor内)
    if not content:
        m = re.search(r'<ucapcontent>(.*?)</ucapcontent>', html, re.DOTALL)
        if m:
            content = m.group(1).strip()

    # 备用: detailCont2
    if not content:
        m = re.search(r'id="detailCont2"[^>]*>(.*?)</div>\s*<div class="xl_con4', html, re.DOTALL)
        if m:
            content = m.group(1).strip()

    if content:
        content = re.sub(r"<script[^>]*>.*?</script>", "", content, flags=re.DOTALL | re.I)
        content = re.sub(r"<style[^>]*>.*?</style>", "", content, flags=re.DOTALL | re.I)
        result["content"] = content

    return result.get("title"), result.get("content"), result.get("publish_date")


def main(incremental=False, limit=None, column="gsgg"):
    site_name = SITE_NAMES.get(column, SITE_NAMES["gsgg"])
    list_path = COLUMN_PATHS.get(column, COLUMN_PATHS["gsgg"])
    list_url = f"{BASE_URL}/dtjjjskfqz/{list_path}.shtml"
    print(f"\n{'='*50}\n🏠 {site_name}\n{'='*50}")

    print("📄 获取列表页...", end=" ", flush=True)
    html = fetch(list_url)
    if not html:
        print("❌ 列表页失败")
        return

    items = parse_list(html)
    print(f"✅ {len(items)} 条")

    # 过滤3年内
    filtered = [(t, u, d) for t, u, d in items if d >= THREE_YEARS_AGO]
    print(f"📅 近3年: {len(filtered)} 条")

    if incremental:
        filtered = filtered[:20]
        print(f"🔄 增量模式: 取前 {len(filtered)} 条")

    if limit:
        filtered = filtered[:limit]
        print(f"🧪 测试模式: 限 {len(filtered)} 条")

    all_items, seen = [], set()
    for i, (title, url, list_date) in enumerate(filtered):
        if url in seen:
            continue
        seen.add(url)

        print(f"  [{i+1}/{len(filtered)}] {title[:50]}...", end=" ", flush=True)
        dt_title, content, pub_date = fetch_detail(url)
        final_title = dt_title or title
        final_date = pub_date or list_date

        summary = re.sub(r"<[^>]+>", " ", content or "").strip()[:300]
        summary = re.sub(r"\s+", " ", summary)

        all_items.append({
            "site_name": site_name,
            "title": final_title,
            "url": url,
            "content": content or "",
            "pub_date": final_date,
            "summary": summary,
            "tags": site_name,
        })
        print("✅")
        time.sleep(0.3)

    if all_items:
        push_to_searchdb(all_items, "dtjjjskfqz")
    else:
        print("  ⏭ 无数据，跳过推送")

    print(f"\n✅ 完成! 共 {len(all_items)} 条")


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--incremental", action="store_true")
    parser.add_argument("--limit", type=int)
    args = parser.parse_args()

    t0 = time.time()
    incremental = args.incremental or (len(sys.argv) > 1 and sys.argv[1] == '1')

    for col in SITE_NAMES:
        main(incremental=incremental, limit=args.limit, column=col)
    print(f"⏱ 总耗时: {time.time()-t0:.1f}s")