#!/usr/bin/env python3
import os
"""
crawl_sxly.py — 临猗县人民政府·公示公告
======================================
静态HTML分页 index_N.shtml（20页，10条/页）

列表：<li><span class="fr">[MM-DD]</span><a href="/doc/YYYY/MM/DD/XXX.shtml" target="_blank">Title</a>
详情：<h1 class="green">标题</h1> | 发布日期：YYYY-MM-DD | <div id="Zoom">正文
来源：<span id="branchstr">

用法:
    python3 crawl_sxly.py             # 全量（20页+近3年）
    python3 crawl_sxly.py 1           # 增量（只爬首页）
"""

import re, sys, os, time, requests

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "临猗县公示公告"
BASE_URL = "https://www.sxly.gov.cn"
LIST_URL = f"{BASE_URL}/zwzx/gggs/index.shtml"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}


def fetch_html(url):
    resp = requests.get(url, headers=HEADERS, timeout=30)
    resp.encoding = "utf-8"
    return resp.text


def get_list_items(url, known_year=None):
    """提取列表页条目"""
    try:
        html = fetch_html(url)
    except Exception as e:
        print(f"  ❌ 请求失败: {e}")
        return []

    items = []
    # <li><span class="fr">[MM-DD]</span><a href="/doc/YYYY/MM/DD/XXX.shtml" target="_blank">TITLE</a></li>
    pattern = re.compile(
        r'<li><span\s+class="fr">\[([^\]]+)\]</span><a\s+href="(/doc/(\d{4})/\d{2}/\d{2}/\d+\.shtml)"[^>]*target="_blank"[^>]*>(.*?)</a></li>',
        re.DOTALL,
    )
    for m in pattern.finditer(html):
        date_md = m.group(1).strip()
        link = m.group(2).strip()
        year = m.group(3)
        title = re.sub(r"<[^>]+>", "", m.group(4)).strip()
        if not link.startswith("http"):
            link = f"{BASE_URL}{link}"
        pub_date = f"{year}-{date_md}"
        items.append({"title": title, "url": link, "pub_date": pub_date})

    return items


def get_detail(item):
    """获取详情"""
    try:
        html = fetch_html(item["url"])
    except Exception as e:
        print(f"    ❌ 详情页请求失败: {e}")
        item["content"] = ""
        item["source"] = SITE_NAME
        return item

    # 正文 — <div id="Zoom">
    m = re.search(r'<div\s+id="Zoom"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
    content = m.group(1).strip() if m else ""

    # 标题 — <h1 class="green">
    m = re.search(r'<h1[^>]*class="green"[^>]*>(.*?)</h1>', html, re.DOTALL)
    if m:
        detail_title = re.sub(r"<[^>]+>", "", m.group(1)).strip()
        if detail_title:
            item["title"] = detail_title

    # 来源 — <span id="branchstr">
    m = re.search(r'<span\s+id="branchstr">([^<]+)</span>', html)
    source = m.group(1).strip().split(",")[0].strip() if m else SITE_NAME
    source = source or SITE_NAME

    # 日期 — <p class="explain">发布日期：YYYY-MM-DD
    m = re.search(r"发布日期：(\d{4}-\d{1,2}-\d{1,2})", html)
    if m:
        item["pub_date"] = m.group(1)

    item["content"] = content
    item["source"] = SITE_NAME  # 统一site_name
    return item


def main():
    incremental = len(sys.argv) > 1 and sys.argv[1] in ("1", "--incremental")
    mode = "增量" if incremental else "全量"
    print(f"🔍 {SITE_NAME} [{mode}]")

    t0 = time.time()
    all_items = []
    max_pages = 1 if incremental else 20

    for page in range(1, max_pages + 1):
        url = LIST_URL if page == 1 else f"{LIST_URL.rstrip('.shtml')}_{page}.shtml"
        items = get_list_items(url)
        if not items:
            if page > 1:
                print(f"  📄 第{page}页: 无数据，结束")
                break
            print(f"  📄 第1页: 无数据")
            break
        # 近3年过滤
        filtered = [it for it in items if not it["pub_date"] or it["pub_date"] >= "2023-06"]
        print(f"  📄 第{page}页: {len(items)}条 (近3年{len(filtered)}条)")
        all_items.extend(filtered)

    if not all_items:
        print("  ⏭ 无数据")
        return

    # 爬详情
    print(f"\n  爬取详情页 ({len(all_items)}条)...")
    for i, item in enumerate(all_items):
        get_detail(item)
        if (i + 1) % 10 == 0:
            print(f"    ... {i+1}/{len(all_items)}", end="\r")

    elapsed = time.time() - t0
    print(f"\n📊 共获取 {len(all_items)} 条，耗时 {elapsed:.0f}s")

    # 入库
    import sqlite3
    db = sqlite3.connect(SEARCH_DB, timeout=60)
    db.execute("PRAGMA journal_mode=WAL")
    db.execute("PRAGMA synchronous=NORMAL")

    ok, skip = 0, 0
    for it in all_items:
        try:
            db.execute(
                "INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
                (
                    SITE_NAME, it["url"], it["url"],
                    (it["title"] or "")[:500], it.get("pub_date", ""),
                    (it["title"] or "")[:500], it.get("content", ""),
                    "active", "公示公告", "",
                ),
            )
            if db.total_changes > 0:
                ok += 1
            else:
                skip += 1
        except Exception as e:
            skip += 1

    db.commit()
    db.execute(
        "INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) "
        "SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r "
        "WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?",
        (SITE_NAME,),
    )
    db.commit()
    db.close()
    print(f"  💾 入库: 新增{ok}, 跳过{skip}")


if __name__ == "__main__":
    main()