#!/usr/bin/env python3
"""
crawl_yueyang_tzgg.py - 岳阳市人民政府 公告公示 爬虫
站点: yueyang.gov.cn/gggs/szbm/
编码: GB2312
列表: ul.news-list > li > a + span (日期), li.line 跳过
分页: default_0.htm ~ default_4.htm (5页, 15条/页)
详情: div.content#zoom, 标题 h1.content-title
附件: <a href="...pdf|doc|docx|xls|xlsx">text</a>

用法:
    python3 crawl_yueyang_tzgg.py              # 增量爬取 (7天内)
    python3 crawl_yueyang_tzgg.py --full       # 全量爬取 (最多5页)
    python3 crawl_yueyang_tzgg.py --help       # 帮助
"""

import re
import sys
import time
import json
import os
import sqlite3
from datetime import datetime, timedelta

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
import urllib.parse

# ─── 配置 ────────────────────────────────────────────────────────────────

BASE_URL = "https://www.yueyang.gov.cn"
CHANNEL_URL = f"{BASE_URL}/gggs/szbm"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "岳阳市-公告公示"
GROUP = "湖南"
INCREMENTAL_DAYS = 7
MAX_PAGES = 5

HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/125.0.0.0 Safari/537.36"
    ),
}

# ─── 工具函数 ────────────────────────────────────────────────────────────

def log(msg):
    print(f"[{SITE_NAME}] {msg}", flush=True)


def fetch(url, retries=3, delay=1):
    """抓取页面，返回 decoded text"""
    for attempt in range(1, retries + 1):
        try:
            resp = requests.get(url, headers=HEADERS, timeout=30)
            resp.encoding = "gb2312"  # 站点编码 GB2312
            if resp.status_code == 200:
                return resp.text
            elif resp.status_code == 404:
                return None  # 404 直接跳过
            else:
                log(f"HTTP {resp.status_code} for {url}")
        except requests.RequestException as e:
            log(f"请求失败 (尝试 {attempt}/{retries}): {e}")
            if attempt < retries:
                time.sleep(delay)
    return None


def make_abs_url(href):
    """将相对URL转为绝对URL"""
    if not href:
        return ""
    if href.startswith("http://") or href.startswith("https://"):
        return href
    return urljoin(CHANNEL_URL + "/", href)


def date_in_range(date_str, days=INCREMENTAL_DAYS):
    """检查日期是否在指定天数内"""
    if not date_str:
        return True  # 无日期默认保留
    try:
        dt = datetime.strptime(date_str.strip(), "%Y-%m-%d")
        return dt >= datetime.now() - timedelta(days=days)
    except ValueError:
        return True


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def extract_attachments(soup, base_url):
    """从详情页提取附件列表，返回 JSON 字符串"""
    attachments = []
    ext_pattern = re.compile(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', re.I)
    for a_tag in soup.find_all("a", href=True):
        href = a_tag["href"].strip()
        if ext_pattern.search(href):
            name = a_tag.get_text(strip=True) or os.path.basename(href)
            full_url = make_abs_url(href)
            attachments.append({"name": name, "url": full_url})
    return json.dumps(attachments, ensure_ascii=False) if attachments else ""


def extract_content_html(soup):
    """提取详情页正文 (div.content#zoom)，保留 HTML 结构，
    同时将表格转为 markdown 格式"""
    content_div = soup.find("div", class_="content", id="zoom")
    if not content_div:
        content_div = soup.find("div", id="zoom")
    if not content_div:
        # 备选方案: 查找常见正文容器
        for selector in ["div.TRS_Editor", "div.Custom_UnionStyle", "div.content"]:
            content_div = soup.select_one(selector)
            if content_div:
                break
    if not content_div:
        return ""

    # 将 <table> 转为 markdown 表格
    for table in content_div.find_all("table"):
        md_table = html_table_to_html(table)
        # 用 markdown 代码块替换原始 table
        table.replace_with(BeautifulSoup(f"\n{md_table}\n", "html.parser"))

    # 返回可读的 HTML 文本 (保留换行、段落结构)
    # 用 <br> 和 <p> 生成换行，方便阅读
    return str(content_div)


# ─── 列表页解析 ──────────────────────────────────────────────────────────

def parse_list_page(html):
    """
    解析列表页，返回 [(title, url, date_str), ...]
    结构: ul.news-list > li > a + span (日期), li.line 跳过
    """
    soup = BeautifulSoup(html, "html.parser")
    items = []

    ul = soup.find("ul", class_="news-list")
    if not ul:
        # 尝试直接查找 li
        ul = soup

    for li in ul.find_all("li", recursive=False):
        # 跳过分隔线 (li.line)
        if li.get("class") and "line" in (li.get("class") or []):
            continue
        # 也检查 class 属性为字符串的情况
        li_class = li.get("class", [])
        if isinstance(li_class, str):
            li_class = li_class.split()
        if "line" in li_class:
            continue

        a_tag = li.find("a")
        if not a_tag:
            continue

        href = a_tag.get("href", "").strip()
        # 优先取 title 属性
        title = (a_tag.get("title") or a_tag.get_text(strip=True) or "").strip()
        if not title or not href:
            continue

        full_url = make_abs_url(href)

        # 日期从 span 取
        span = li.find("span")
        date_str = ""
        if span:
            date_str = span.get_text(strip=True)
            # 统一为 YYYY-MM-DD 格式
            date_match = re.search(r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})', date_str)
            if date_match:
                raw = date_match.group(1)
                date_str = raw.replace("/", "-")

        # 清理标题中的空白
        title = re.sub(r'\s+', ' ', title).strip()

        items.append((title, full_url, date_str))

    return items


# ─── 详情页解析 ──────────────────────────────────────────────────────────

def parse_detail_page(html, list_title, list_date):
    """
    解析详情页，返回 (title, date, content, attachments_json)
    """
    if not html:
        return list_title, list_date, "", ""

    soup = BeautifulSoup(html, "html.parser")

    # 标题: h1.content-title
    title = list_title
    h1 = soup.find("h1", class_="content-title")
    if not h1:
        h1 = soup.find("h1")
    if h1:
        t = h1.get_text(strip=True)
        if t:
            title = re.sub(r'\s+', ' ', t).strip()

    # 日期: 从正文或页面提取
    date_str = list_date
    # 尝试从页面提取日期 (多种常见格式)
    date_patterns = [
        # meta 标签
        (r'<meta[^>]*name=["\']?[Pp]ub[Đđ]ate["\']?[^>]*content=["\']?(\d{4}-\d{2}-\d{2})', None),
        (r'<meta[^>]*name=["\']?dcterms.issued["\']?[^>]*content=["\']?(\d{4}-\d{2}-\d{2})', None),
        # 页面文本中的日期
        (r'发布日期[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', None),
        (r'发布时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', None),
        (r'日期[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', None),
        (r'时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', None),
        (r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})\s*\d{1,2}[:：]\d{1,2}', None),
    ]
    for pattern, _ in date_patterns:
        m = re.search(pattern, html)
        if m:
            date_str = m.group(1).replace("/", "-")
            break

    # 正文
    content = extract_content_html(soup)
    if not content:
        content = ""

    # 附件
    attachments = extract_attachments(soup, BASE_URL)

    return title, date_str, content, attachments


# ─── 数据库操作 ──────────────────────────────────────────────────────────

def get_db_connection():
    """获取数据库连接"""
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    return conn


def save_to_db(conn, record):
    """
    保存一条记录到 gov_raw 表
    record = {
        "page_url", "title", "publish_date", "site_name",
        "group_name", "summary", "content", "attachments", "source_url"
    }
    """
    try:
        conn.execute(
            """INSERT OR IGNORE INTO gov_raw
               (page_url, title, publish_date, site_name, group_name, summary, content, attachments, source_url)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (
                record["page_url"],
                record["title"],
                record.get("publish_date", ""),
                record["site_name"],
                record.get("group_name", ""),
                record.get("summary", ""),
                record.get("content", ""),
                record.get("attachments", ""),
                record.get("source_url", ""),
            ),
        )
        return conn.execute("SELECT changes()").fetchone()[0] > 0
    except sqlite3.IntegrityError:
        return False
    except Exception as e:
        log(f"DB 错误: {e}")
        return False


# ─── 主流程 ──────────────────────────────────────────────────────────────

def main():
    # 解析参数
    full_mode = "--full" in sys.argv
    if "--help" in sys.argv or "-h" in sys.argv:
        print(__doc__)
        return

    log(f"启动爬取 (mode: {'全量' if full_mode else '增量'})")
    if not full_mode:
        log(f"增量窗口: {INCREMENTAL_DAYS} 天")

    all_items = []

    # ── 1. 遍历列表页 ──
    total_pages = MAX_PAGES if full_mode else 1  # 增量只扫第1页
    for page in range(total_pages):
        if page == 0:
            page_url = f"{CHANNEL_URL}/default.htm"
        else:
            page_url = f"{CHANNEL_URL}/default_{page}.htm"
        log(f"抓取列表页 {page + 1}/{total_pages}: {page_url}")

        html = fetch(page_url)
        if not html:
            log(f"列表页 {page_url} 获取失败，跳过")
            continue

        items = parse_list_page(html)
        log(f"  解析到 {len(items)} 条记录")

        if not full_mode and items:
            # 增量模式: 只保留 INCREMENTAL_DAYS 天内的
            filtered = [it for it in items if date_in_range(it[2], INCREMENTAL_DAYS)]
            log(f"  增量过滤后: {len(filtered)} 条 (7天内)")
            all_items.extend(filtered)
            # 如果当天页仍有超过7天的文章，停止继续翻页
            has_old = any(not date_in_range(it[2], INCREMENTAL_DAYS) for it in items)
            if has_old:
                log(f"  已触及7天前的数据，停止翻页")
                break
        else:
            all_items.extend(items)

        time.sleep(0.5)

    log(f"共获取 {len(all_items)} 条待处理")

    if not all_items:
        log("无新数据，退出")
        return

    # ── 2. 连接数据库 ──
    conn = get_db_connection()
    c = conn.cursor()

    # ── 3. 逐条抓取详情并入库 ──
    inserted = 0
    skipped = 0
    errors = 0

    for idx, (title, page_url, date_str) in enumerate(all_items):
        # 检查是否已存在
        c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,))
        if c.fetchone():
            skipped += 1
            continue

        # 抓取详情
        detail_html = fetch(page_url)
        if not detail_html:
            log(f"  [跳过] 详情页无法获取: {title[:40]}")
            skipped += 1
            continue

        # 解析详情
        detail_title, detail_date, content, attachments = parse_detail_page(
            detail_html, title, date_str
        )
        if not content:
            log(f"  [跳过] 正文为空: {detail_title[:40]}")
            skipped += 1
            continue

        # 构建摘要
        # 去掉 HTML 标签取纯文本前200字
        text_only = BeautifulSoup(content, "html.parser").get_text(strip=True)
        summary = text_only[:200].replace("\n", " ").strip()

        record = {
            "page_url": page_url,
            "title": detail_title,
            "publish_date": detail_date,
            "site_name": SITE_NAME,
            "group_name": GROUP,
            "summary": summary,
            "content": content,
            "attachments": attachments,
            "source_url": CHANNEL_URL,
        }

        if save_to_db(conn, record):
            inserted += 1
            if inserted % 5 == 0:
                conn.commit()
                log(f"  进度: {inserted}/{len(all_items)} 已入库")
        else:
            errors += 1

        time.sleep(0.3)

    conn.commit()
    conn.close()

    # ── 4. 统计 ──
    log(f"\n{'=' * 45}")
    log(f"爬取完成: {SITE_NAME}")
    log(f"  总计: {len(all_items)} 条")
    log(f"  新增: {inserted} 条")
    log(f"  跳过: {skipped} 条 (已存在/无内容)")
    log(f"  错误: {errors} 条")
    log(f"{'=' * 45}")


if __name__ == "__main__":
    main()