#!/usr/bin/env python3
"""
crawl_yiyang.py — 益阳市人民政府·公示公告 爬虫
============================================
列表：https://www.yiyang.gov.cn/yiyang/2/3/4/default.htm
      分页 default_2.htm ... default_9.htm（约 120 条）
编码：GB2312
详情：<h3>标题 | 发布时间：YYYY-MM-DD | <div class="xl_text"> 正文

用法:
    python3 crawl_yiyang.py             # 全量爬
    python3 crawl_yiyang.py --limit=5   # 测试只跑5条
"""

import re, sys, os, time
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import (
    fetch_page, parse_date, clean_html, THREE_YEARS_AGO,
    extract_date_from_html, extract_content_by_selector, extract_content_by_regex,
    save_jsonl, push_to_searchdb
)

# ═══════════════════════════════════════════
#  配置区
# ═══════════════════════════════════════════

SITE_NAME = "益阳市人民政府"
BASE_URL  = "https://www.yiyang.gov.cn"
LIST_URL  = "https://www.yiyang.gov.cn/yiyang/2/3/4/default.htm"

ENCODING = "gb2312"

# 列表页：<a href="content_XXXX.html">标题</a>
LIST_LINK_RE = re.compile(r'<a\s+href=[\'"](content_\d+\.html)[\'"][^>]*>([^<]+)</a>', re.I)

# 详情页标题：<h3>...</h3>
TITLE_RE = re.compile(r'<h3[^>]*>(.*?)</h3>', re.DOTALL)

# 详情页日期
DATE_RE_PAT = re.compile(r'发布时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})')

# 详情页正文选择器
DETAIL_SELECTOR = "div.xl_text"


def _fetch(url):
    """带 GB2312 编码的请求"""
    return fetch_page(url, encoding=ENCODING)


def _abs_urls(html, base_url):
    """将 HTML 中的相对路径（src/href）转为绝对 URL"""
    def _abs(m):
        attr = m.group(1)
        val = m.group(2)
        if val.startswith(('http://', 'https://', '//', 'data:', 'javascript:', '#', 'tel:', 'mailto:')):
            return m.group(0)
        abs_val = urljoin(base_url, val)
        return f'{attr}="{abs_val}"'
    html = re.sub(r'(src|href)="([^"]+)"', _abs, html, flags=re.I)
    return html


# ═══════════════════════════════════════════
#  列表页提取
# ═══════════════════════════════════════════

def get_list_items_from_page(html, page_url):
    """从列表页提取 (title, url) 列表"""
    items = []
    for m in LIST_LINK_RE.finditer(html):
        href = m.group(1).strip()
        title = m.group(2).strip()
        if not title or not href:
            continue
        full_url = urljoin(page_url, href)
        items.append((title, full_url))
    return items


def get_all_list_items():
    """遍历所有分页，返回完整列表"""
    all_items = []

    # 第1页
    html = _fetch(LIST_URL)
    if not html:
        print("  ❌ 首页请求失败")
        return all_items

    page_items = get_list_items_from_page(html, LIST_URL)
    print(f"  📃 第1页: {len(page_items)} 条")
    all_items.extend(page_items)

    # 第2页起
    for page_num in range(2, 50):  # 安全上限
        page_url = f"https://www.yiyang.gov.cn/yiyang/2/3/4/default_{page_num}.htm"
        html = _fetch(page_url)
        if not html:
            break
        items = get_list_items_from_page(html, page_url)
        if not items:
            break
        print(f"  📃 第{page_num}页: {len(items)} 条")
        all_items.extend(items)

    return all_items


# ═══════════════════════════════════════════
#  详情页提取
# ═══════════════════════════════════════════

def get_detail(url):
    """访问详情页，返回 {title, content, pub_date}"""
    html = _fetch(url)
    if not html:
        return {"title": "", "content": "", "pub_date": ""}

    # 1. 标题
    title = ""
    m = TITLE_RE.search(html)
    if m:
        title = m.group(1).strip()

    # 2. 日期
    pub_date = ""
    m = DATE_RE_PAT.search(html)
    if m:
        pub_date = parse_date(m.group(1))

    # 3. 正文
    content = extract_content_by_selector(html, DETAIL_SELECTOR)
    if content:
        content = clean_html(content)
        content = _abs_urls(content, url)

    return {"title": title, "content": content, "pub_date": pub_date}


# ═══════════════════════════════════════════
#  主流程
# ═══════════════════════════════════════════

def crawl(max_items=None):
    """全量爬取"""
    print(f"  📡 爬取列表页...")
    all_list_items = get_all_list_items()
    print(f"  📊 列表共 {len(all_list_items)} 条")

    # 去重
    seen_urls = set()
    unique = []
    for title, url in all_list_items:
        if url not in seen_urls:
            seen_urls.add(url)
            unique.append((title, url))

    print(f"  🎯 去重后: {len(unique)} 条")

    if max_items:
        unique = unique[:max_items]

    # 爬详情
    collected = []
    ok, fail = 0, 0
    for i, (title, url) in enumerate(unique, 1):
        detail = get_detail(url)
        final_title = detail["title"] or title

        if not detail["content"] or len(detail["content"]) < 20:
            print(f"    ⚠️ [{i}/{len(unique)}] {final_title[:30]} — 无正文")
            fail += 1
            continue

        entry = {
            "site_name": SITE_NAME,
            "title": final_title,
            "url": url,
            "content": detail["content"],
            "pub_date": detail["pub_date"],
            "tags": SITE_NAME,
        }
        collected.append(entry)
        save_jsonl(entry)
        ok += 1

        if i % 5 == 0 or i == len(unique):
            print(f"  [{i}/{len(unique)}] ✅{ok} ❌{fail}")

    # 推送到服务器 search.db
    if collected:
        push_to_searchdb(collected, batch_label="yiyang")
    print(f"  ✅ 完成: 共{ok}条有效记录")
    return ok, fail


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description=f"爬虫: {SITE_NAME}")
    parser.add_argument("--limit", type=int, help="测试：只跑 N 条")
    args = parser.parse_args()

    print(f"\n📡 [{SITE_NAME}] 开始爬取")
    t0 = time.time()
    crawl(max_items=args.limit)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s\n")