#!/usr/bin/env python3
"""
crawl_xiangyin.py — 湘阴县人民政府·环境影响评价 爬虫
================================================
列表：<ul class="xxdk-news-list pd30"> → <li><a href="content_xxx.html" title="标题">标题</a><span>日期</span>
分页：index.htm → index_{n}.htm（0索引，最多5页）
详情：<div class="content-wrapper">
编码：GBK

用法:
    python3 crawl_xiangyin.py             # 全量爬（最多5页，近3年，含"项目"）
    python3 crawl_xiangyin.py --limit=5   # 测试只跑5条
    python3 crawl_xiangyin.py --stats     # 看统计
"""

import re, sys, os, time
from urllib.parse import urljoin

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import (
    fetch_page, parse_date, clean_html, THREE_YEARS_AGO,
    extract_date_from_html, extract_content_by_selector, extract_content_by_regex,
    save_jsonl, push_to_searchdb
)

# ═══════════════════════════════════════════
#  配置区
# ═══════════════════════════════════════════

SITE_NAME = "湘阴县人民政府"
BASE_URL  = "https://www.xiangyin.gov.cn"
LIST_BASE = "https://www.xiangyin.gov.cn/31951/61032/61111/"

# 列表页：index.htm, index_1.htm, index_2.htm ...
LIST_URLS = [urljoin(LIST_BASE, "index.htm")]
for n in range(1, 5):  # 最多5页（0索引）
    LIST_URLS.append(urljoin(LIST_BASE, f"index_{n}.htm"))

# 列表项正则
LIST_PATTERN = r"<a href='([^']+)' target='_blank' title='([^']+)'>"

# 详情页正文 CSS 选择器
DETAIL_SELECTOR = ".content-wrapper"

# 日期正则
DATE_PATTERN = re.compile(r'<span>\s*(\d{4}-\d{2}-\d{2})\s*</span>')

# 详情页编码为 GBK，requests 的自动检测可能不对
DETAIL_ENCODING = 'gbk'


def _abs_urls(html, base_url):
    """将 HTML 中的相对路径（src/href）转为绝对 URL"""
    def _abs(m):
        attr = m.group(1)
        val = m.group(2)
        if val.startswith(('http://', 'https://', '//', 'data:', 'javascript:', '#', 'tel:', 'mailto:')):
            return m.group(0)
        abs_val = urljoin(base_url, val)
        return f'{attr}="{abs_val}"'
    html = re.sub(r'(src|href)="([^"]+)"', _abs, html, flags=re.I)
    return html


def _decode_gbk(html_bytes):
    """尝试解码 GBK/UTF-8"""
    for enc in ['gbk', 'gb2312', 'utf-8']:
        try:
            return html_bytes.decode(enc)
        except:
            continue
    return html_bytes.decode('utf-8', errors='replace')


# ═══════════════════════════════════════════
#  列表页提取
# ═══════════════════════════════════════════

def get_list_items(html):
    """从列表页 HTML 提取条目"""
    items = []
    for m in re.finditer(LIST_PATTERN, html):
        href = m.group(1).strip()
        title = m.group(2).strip()
        if not title or not href:
            continue
        full_url = urljoin(LIST_BASE, href)
        items.append({"title": title, "url": full_url})
    # 匹配日期
    dates = DATE_PATTERN.findall(html)
    for i, item in enumerate(items):
        if i < len(dates):
            item["pub_date"] = dates[i]
        else:
            item["pub_date"] = ""
    return items


def get_page_items(url):
    """获取单页的列表条目"""
    import requests
    try:
        r = requests.get(url, headers={
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
        }, timeout=20)
        if r.status_code != 200:
            return []
        html = _decode_gbk(r.content)
        return get_list_items(html)
    except:
        return []


# ═══════════════════════════════════════════
#  详情页提取
# ═══════════════════════════════════════════

def get_detail(url):
    """访问详情页，返回正文 HTML 和发布时间"""
    import requests
    try:
        r = requests.get(url, headers={
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
        }, timeout=20)
        if r.status_code != 200:
            return {"content": "", "pub_date": ""}
        html = r.content.decode(DETAIL_ENCODING, errors='replace')
    except:
        return {"content": "", "pub_date": ""}

    content = extract_content_by_selector(html, DETAIL_SELECTOR)
    if not content or len(content) < 50:
        m = re.search(r'<div class="content-wrapper">(.*?)</div>\s*</div>', html, re.DOTALL)
        if m:
            content = clean_html(m.group(1))

    if content:
        content = _abs_urls(content, url)

    pub_date = extract_date_from_html(html)

    return {"content": content, "pub_date": pub_date}


# ═══════════════════════════════════════════
#  主流程
# ═══════════════════════════════════════════

def crawl(max_items=None):
    """全量爬取（最多5页，近3年，含"项目"）"""
    print(f"  📡 抓取列表页...")

    all_items = []
    for page_url in LIST_URLS:
        items = get_page_items(page_url)
        print(f"    {page_url.split('/')[-1]}: {len(items)} 条")
        all_items.extend(items)
        if len(items) == 0:
            break

    # 去重
    seen = set()
    unique = []
    for it in all_items:
        if it["url"] not in seen:
            seen.add(it["url"])
            unique.append(it)

    print(f"  去重后: {len(unique)} 条")

    # 过滤：标题含"项目" + 近3年
    filtered = []
    for it in unique:
        if "项目" not in it["title"]:
            continue
        if it.get("pub_date") and it["pub_date"] < THREE_YEARS_AGO:
            continue
        filtered.append(it)

    print(f"  过滤后: {len(filtered)} 条（含\"项目\"+近3年）")

    if max_items:
        filtered = filtered[:max_items]

    # 爬正文
    collected = []
    ok, fail = 0, 0
    for i, item in enumerate(filtered, 1):
        detail = get_detail(item["url"])
        entry = {
            "site_name": SITE_NAME,
            "title": item["title"],
            "url": item["url"],
            "content": detail["content"],
            "pub_date": detail["pub_date"] or item.get("pub_date", ""),
            "tags": SITE_NAME,
        }
        collected.append(entry)
        save_jsonl(entry)
        ok += 1
        if i % 5 == 0 or i == len(filtered):
            print(f"  [{i}/{len(filtered)}] ✅{ok} ❌{fail}")

    # 推送到服务器 search.db
    if collected:
        push_to_searchdb(collected, batch_label=SITE_NAME.replace(" ", "_"))
    print(f"  ✅ 完成: 共{ok}条")
    return ok, fail


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description=f"爬虫: {SITE_NAME}")
    parser.add_argument("--limit", type=int, help="测试：只跑 N 条")
    parser.add_argument("--stats", action="store_true", help="查看 eia.db 统计")
    args = parser.parse_args()

    if args.stats:
        print("📊 统计数据: 请登录服务器查看 search.db")
        sys.exit(0)

    print(f"\n📡 [{SITE_NAME}] 开始爬取 (最多5页, 含\"项目\"+近3年)")
    t0 = time.time()
    crawl(max_items=args.limit)
    print(f"⏱ 耗时: {time.time()-t0:.1f}s\n")