#!/usr/bin/env python3
"""
贵州贵安新区管理委员会 - 项目环评
https://www.gaxq.gov.cn/zwgk/xxgkml/zdlyxxgk/hjbh/xmhp/index.html
TRS CMS, createPageHTML分页
"""
import requests
import re
import sys
import os
import json
from datetime import datetime, timedelta
from urllib.parse import urljoin

BASE_URL = "https://www.gaxq.gov.cn/zwgk/xxgkml/zdlyxxgk/hjbh/xmhp"
LIST_URL = f"{BASE_URL}/index.html"
SITE_NAME = "gaxq_xmhp"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}

session = requests.Session()
session.headers.update(HEADERS)

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")


def get_total_pages(html):
    """解析 createPageHTML(total_pages, curr, prefix, ext, total_records)"""
    m = re.search(r"createPageHTML\((\d+)", html)
    if m:
        return int(m.group(1))
    return 1


def parse_list_page(html):
    """解析列表页：<tr class="c"> 中的链接和日期"""
    items = []
    rows = re.findall(
        r'<tr class="c">\s*<td[^>]*>\d+</td>\s*<td class="tits">\s*<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>\s*</td>\s*<td[^>]*>([^<]+)</td>',
        html, re.DOTALL
    )
    for url, title, date_str in rows:
        title = re.sub(r"\s+", " ", title).strip()
        date_str = date_str.strip()
        if not url.startswith("http"):
            url = urljoin(LIST_URL, url)
        items.append({"url": url, "title": title, "date": date_str})
    return items


def parse_detail(html, url):
    """解析详情页"""
    result = {"content": "", "pub_date": "", "attachments": []}

    # 标题
    m = re.search(r"<h1[^>]*>(.*?)</h1>", html, re.DOTALL)
    if m:
        result["title"] = re.sub(r"<[^>]+>", "", m.group(1)).strip()

    # 日期
    m = re.search(r"pubdata\s*=\s*'([^']+)'", html)
    if m:
        result["pub_date"] = m.group(1).strip()

    # 正文
    m = re.search(r'<font id="Zoom">(.*?)</font>', html, re.DOTALL)
    if m:
        zoom = m.group(1)
        # 提取附件链接
        for a in re.finditer(r'<a[^>]*href="([^"]+)"[^>]*>([^<]+)</a>', zoom):
            href, text = a.group(1), a.group(2)
            if href.endswith((".pdf", ".doc", ".docx", ".zip", ".rar", ".xls", ".xlsx")):
                full_url = urljoin(url, href)
                result["attachments"].append({"url": full_url, "title": text.strip()})
        result["content"] = zoom
    else:
        result["content"] = ""

    return result


def insert_item(item, detail, conn):
    """插入到数据库"""
    c = conn.cursor()

    source_url = item["url"]
    title = detail.get("title", item["title"])
    pub_date = detail.get("pub_date", item["date"])
    content = detail["content"]
    attachments = detail["attachments"]

    attach_json = json.dumps(attachments, ensure_ascii=False) if attachments else ""

    # 构造纯文本预览
    text_content = re.sub(r"<[^>]+>", "", content).strip() if content else ""

    # 正文验证
    if not content or len(text_content) < 50:
        print(f"  ⚠ 正文过短: {title[:30]}... ({len(text_content)} chars)")

    try:
        c.execute("""
            INSERT OR IGNORE INTO gov_raw 
            (title, page_url, source_url, content, publish_date, site_name)
            VALUES (?, ?, ?, ?, ?, ?)
        """, (
            title,
            source_url,
            source_url,
            content,
            pub_date,
            SITE_NAME
        ))
        affected = c.rowcount
        conn.commit()
        if affected > 0:
            print(f"  ✓ {title[:30]}...")
        return affected
    except Exception as e:
        print(f"  ✗ 插入失败: {e}")
        return 0


def crawl(days_back=365):
    """全量/增量爬取"""
    import sqlite3

    now = datetime.now()
    cutoff = now - timedelta(days=days_back)

    resp = session.get(LIST_URL, timeout=30)
    resp.encoding = "utf-8"
    total_pages = get_total_pages(resp.text)

    print(f"共 {total_pages} 页")

    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    total = 0
    skip_count = 0

    for page in range(total_pages):
        if page == 0:
            url = LIST_URL
        else:
            url = f"{BASE_URL}/index_{page}.html"

        print(f"\n--- 第 {page+1}/{total_pages} 页 ---")
        try:
            if page == 0:
                # 第一页已请求过，直接用
                page_html = resp.text
            else:
                resp = session.get(url, timeout=30)
                resp.encoding = "utf-8"
                page_html = resp.text
        except Exception as e:
            print(f"  ✗ 请求失败: {e}")
            continue

        items = parse_list_page(page_html)
        print(f"  发现 {len(items)} 条")

        for item in items:
            # 日期过滤
            try:
                item_date = datetime.strptime(item["date"], "%Y-%m-%d")
                if item_date < cutoff and days_back < 3650:
                    skip_count += 1
                    continue
            except ValueError:
                pass

            # 获取详情
            try:
                resp2 = session.get(item["url"], timeout=30)
                resp2.encoding = "utf-8"
            except Exception as e:
                print(f"  ✗ 详情页请求失败 {item['url'][:60]}: {e}")
                continue

            detail = parse_detail(resp2.text, item["url"])
            affected = insert_item(item, detail, conn)
            if affected > 0:
                total += 1

    conn.close()
    print(f"\n=== 完成: 新增 {total} 条, 跳过 {skip_count} 条 ===")
    return total


if __name__ == "__main__":
    days = int(sys.argv[1]) if len(sys.argv) > 1 else 365
    crawl(days_back=days)