#!/usr/bin/env python3
"""怀化新闻网 - 公告通知 爬虫
M2O CMS，静态HTML列表+详情页
列表 /folder77/ ~ /folder77/index_30.html，30条/页
正文从详情页 <article> 提取
"""

import os
import sys
import re
import requests
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "0745news.cn-公告通知"
BASE_URL = "https://www.0745news.cn"
FOLDER = "/folder3287/folder2068/folder2/folder77"
TOTAL_PAGES = 30

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}

CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
print(f"[info] 日期过滤: >= {CUTOFF_DATE}")


def fetch_list_page(page):
    """获取列表页HTML"""
    if page == 1:
        url = f"{BASE_URL}{FOLDER}/"
    else:
        url = f"{BASE_URL}{FOLDER}/index_{page}.html"
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"[error] 第{page}页请求失败: {e}")
        return ""


def parse_list_items(html):
    """解析列表页中的条目"""
    items = []
    # 匹配: <a href="URL" class="fl" title="..."><div class="title_time"><p>title</p>...</a><span class="fr">DATE</span>
    pattern = r'<a href="([^"]+)"[^>]*title="[^"]*"[^>]*>\s*<div class="title_time">\s*<p class="">([^<]*)</p>'
    dates_pattern = r'<span class="fr">([0-9]{4}-[0-9]{2}-[0-9]{2})</span>'

    hrefs = re.findall(pattern, html)
    dates = re.findall(dates_pattern, html)

    for (url, title), date in zip(hrefs, dates):
        if not url.startswith("http"):
            url = BASE_URL + url
        items.append({
            "title": title.strip(),
            "date": date.strip(),
            "url": url,
        })
    return items


def fetch_detail(html):
    """从详情页HTML提取 <article> 内正文"""
    m = re.search(r'<article>([\s\S]*?)</article>', html)
    if m:
        return m.group(1).strip()
    # 备用: 取 content div 内容
    m = re.search(r'<div class="content">([\s\S]*?)</div>\s*<div class="copyright_notice">', html)
    if m:
        content_html = m.group(1)
        # 去掉标题和info区，只保留正文
        content_html = re.sub(r'<h2 class="title".*?</h2>', '', content_html)
        content_html = re.sub(r'<div class="clearfix info">.*?</div>', '', content_html)
        return content_html.strip()
    return ""


def fetch_detail_page(url):
    """获取详情页HTML"""
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        r.encoding = "utf-8"
        return r.text
    except Exception as e:
        print(f"  [warn] 详情页请求失败: {url[-60:]}, {e}")
        return ""


def main():
    import sqlite3

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    c.execute("PRAGMA journal_mode=WAL")

    new_count = 0
    skip_count = 0
    old_skip = 0
    err_count = 0

    for page in range(1, TOTAL_PAGES + 1):
        html = fetch_list_page(page)
        if not html:
            continue

        items = parse_list_items(html)
        if not items:
            print(f"[warn] 第{page}页未解析到条目")
            continue

        page_new = 0
        page_skip = 0
        for item in items:
            date = item["date"]
            if date < CUTOFF_DATE:
                old_skip += 1
                continue

            title = item["title"]
            url = item["url"]

            # 去重
            c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=?", (url,))
            if c.fetchone()[0] > 0:
                page_skip += 1
                continue

            # 取详情页正文
            detail_html = fetch_detail_page(url)
            if not detail_html:
                err_count += 1
                continue

            content = fetch_detail(detail_html)
            if not content:
                content = f"[无正文] {title}"

            try:
                c.execute("""
                    INSERT OR IGNORE INTO gov_raw (page_url, title, content, site_name, publish_date, summary, category)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                """, (url, title, content, SITE_NAME, date, title, "公告通知"))
                if c.rowcount > 0:
                    page_new += 1
            except Exception as e:
                print(f"  [error] 入库失败: {e}")
                err_count += 1

        conn.commit()
        new_count += page_new
        skip_count += page_skip
        print(f"[page {page}/{TOTAL_PAGES}] +{page_new} new, {page_skip} dup, cumulative: {new_count} new")

    conn.close()
    print(f"\n[完成] 怀化新闻网: 新增 {new_count} 条, 重复跳过 {skip_count}, 超3年跳过 {old_skip}, 错误 {err_count}")


if __name__ == "__main__":
    main()