#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
寻甸县-公示公告 独立爬虫 (gsgg 栏目)
站点: www.kmxd.gov.cn
栏目: /zfxxgkml/fdzdgknr/gsgg/ (公示公告)
列表: index.shtml / index_N.shtml (50条/页, 10页共500条)
详情: /c/YYYY-MM-DD/ID.shtml, 正文 div.content>div.activity, <p style=...> 规范化
"""
import re, os, sys, time
import requests, warnings
warnings.filterwarnings('ignore')
from datetime import datetime, timedelta

_MAX_PG = None
for _a in sys.argv[1:]:
    if _a.startswith("--pages="):
        try:
            _MAX_PG = int(_a.split("=", 1)[1])
        except Exception:
            pass
    elif _a.isdigit():
        _MAX_PG = int(_a)
if _MAX_PG is not None:
    print(f'[AutoPg] max_pages={_MAX_PG}')

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "寻甸县-公示公告"
BASE_URL = "http://www.kmxd.gov.cn"
LIST_PATH = "/zfxxgkml/fdzdgknr/gsgg"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36"}


def fetch(url):
    for retry in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            if r.status_code == 200:
                r.encoding = "utf-8"
                return r.text
            print(f"  [WARN] {url} HTTP {r.status_code}, retry {retry+1}")
        except Exception as e:
            print(f"  [WARN] {url} err: {e}, retry {retry+1}")
        time.sleep(1)
    return None


def parse_list(html):
    """列表: <a href="/c/YYYY-MM-DD/ID.shtml">标题</a> + 日期"""
    items = []
    # 用 <div class="data-table-item"> 块解析
    for block in re.findall(r'<div class="data-table-item[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.S):
        m = re.search(r'href="(/c/\d{4}-\d{2}-\d{2}/\d+\.shtml)"[^>]*>\s*([^<]+?)\s*<', block)
        dm = re.search(r'(\d{4}-\d{2}-\d{2})', block)
        if m:
            title = m.group(2).strip()
            date = dm.group(1) if dm else ""
            items.append({
                "url": BASE_URL + m.group(1),
                "title": title,
                "date": date,
            })
    # 兜底: 直接正则
    if not items:
        for m in re.finditer(r'<a href="(/c/(\d{4}-\d{2}-\d{2})/(\d+)\.shtml)"\s+target=[\'"]_blank[\'"]>\s*([^<]+?)\s*<', html):
            items.append({
                "url": BASE_URL + m.group(1),
                "title": m.group(4).strip(),
                "date": m.group(2),
            })
    return items


def extract_detail(html, fallback_title):
    """详情: div.content -> div.activity 内 <p> 段落 (栈匹配 div 平衡)"""
    title = fallback_title
    m = re.search(r'<div class="content">\s*<h1>([^<]+)</h1>', html)
    if m:
        title = m.group(1).strip()

    body = ""
    start = html.find('<div class="activity">')
    if start >= 0:
        # 栈匹配 activity div
        i = start + len('<div class="activity">')
        depth = 1
        while i < len(html) and depth > 0:
            nxt_open = html.find('<div', i)
            nxt_close = html.find('</div>', i)
            if nxt_close == -1:
                break
            if nxt_open != -1 and nxt_open < nxt_close:
                depth += 1
                i = nxt_open + 4
            else:
                depth -= 1
                i = nxt_close + 6
        body = html[start:i]
    else:
        # 兜底: content div 内所有 p
        m = re.search(r'<div class="content">(.*?)</div>\s*(?:<!--|$)', html, re.S)
        if m:
            body = m.group(1)
            ai = body.find('<div class="activity">')
            if ai >= 0:
                body = body[ai:]

    # 规范化 <p style=...> -> <p>
    body = re.sub(r'<p[^>]*>', '<p>', body)
    # 附件: 保留原 HTML 内嵌 <a href>
    body = body.strip()
    return title, body


def run(max_pages=200):
    print(f"\n{'='*50}")
    print(f"🚀 {SITE_NAME}")
    print(f"{'='*50}")

    total_pages = min(max_pages, _MAX_PG or max_pages)
    all_items = []
    for pg in range(1, total_pages + 1):
        url = f"{BASE_URL}{LIST_PATH}/index.shtml" if pg == 1 else f"{BASE_URL}{LIST_PATH}/index_{pg}.shtml"
        html = fetch(url)
        if not html:
            print(f"  第{pg}页: 抓取失败 -> 结束")
            break
        items = parse_list(html)
        if not items:
            print(f"  第{pg}页: 无条目 -> 结束")
            break
        dates = [it["date"] for it in items if it["date"]]
        print(f"  第{pg}页: {len(items)} 条 ({min(dates) if dates else '?'} ~ {max(dates) if dates else '?'})")
        all_items.extend(items)
        if dates and max(dates) < CUTOFF:
            print(f"  该页已全部早于截断日{CUTOFF}，停止翻页")
            break
        time.sleep(0.2)

    print(f"\n📊 列表共 {len(all_items)} 条")
    kept = [it for it in all_items if not it["date"] or it["date"] >= CUTOFF]
    print(f"  窗口内(>= {CUTOFF}): {len(kept)} 条")

    # 去重 (URL)
    seen = set()
    uniq = []
    for it in kept:
        if it["url"] not in seen:
            seen.add(it["url"])
            uniq.append(it)
    if len(uniq) != len(kept):
        print(f"  ⚠ 去重: {len(kept)} -> {len(uniq)}")

    # 抓详情
    out = []
    new = skip = err = 0
    for i, it in enumerate(uniq, 1):
        detail_html = fetch(it["url"])
        if not detail_html:
            err += 1
            continue
        title, body = extract_detail(detail_html, it["title"])
        text_len = len(re.sub(r'<[^>]+>', '', body).strip()) if body else 0
        has_img = '<img' in (body or '')
        if not body or (text_len < 5 and not has_img):
            print(f"  ⚠ 空正文: {title[:50]}")
            err += 1
            continue
        out.append({
            "site_name": SITE_NAME,
            "title": title,
            "pub_date": it["date"],
            "content": body,
            "source_url": it["url"],
            "url": it["url"],
        })
        if i % 20 == 0:
            print(f"  ...{i}/{len(uniq)} (文本{text_len}字)")
        time.sleep(0.15)

    print(f"\n📦 待入库: {len(out)}, 失败: {err}")
    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
    from crawler_lib import push_to_searchdb
    push_to_searchdb(out, batch_label=SITE_NAME)


if __name__ == "__main__":
    run(max_pages=200)