#!/usr/bin/env python3
"""
习水县人民政府 - 生态环境
https://www.xsx.gov.cn/zwgk/zdlyxx/sthj/
TRS CMS, createPageHTML分页
"""

import requests
import re
import sys
import os
from datetime import datetime, timedelta
from urllib.parse import urljoin

BASE_URL = "https://www.xsx.gov.cn/zwgk/zdlyxx/sthj"
LIST_URL = f"{BASE_URL}/index.html"
SITE_NAME = "xsx_sthj"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}

session = requests.Session()
session.headers.update(HEADERS)


def get_total_pages(html):
    m = re.search(r"createPageHTML\((\d+)", html)
    return int(m.group(1)) if m else 1


def parse_list_page(html):
    items = []
    # <li> with <a> and <span>date</span>
    rows = re.findall(
        r'<li>\s*<a[^>]*title="([^"]*)"[^>]*href="([^"]+)"[^>]*>(.*?)</a>\s*<span>([^<]+)</span>',
        html, re.DOTALL
    )
    for title_attr, url, link_text, date_str in rows:
        title = title_attr or re.sub(r"\s+", " ", link_text).strip()
        date_str = date_str.strip()
        if not url.startswith("http"):
            url = urljoin(LIST_URL, url)
        items.append({"url": url, "title": title, "date": date_str})
    return items


def parse_detail(html, url):
    result = {"content": "", "pub_date": ""}

    # 标题
    m = re.search(r"<title>(.*?)</title>", html, re.DOTALL)
    if m:
        result["title"] = m.group(1).strip()

    # 日期
    m = re.search(r"pubdata\s*=\s*'([^']+)'", html)
    if m:
        result["pub_date"] = m.group(1).strip()[:10]  # YYYY-MM-DD

    # 正文
    m = re.search(r'<font id="Zoom">(.*?)</font>', html, re.DOTALL)
    if m:
        result["content"] = m.group(1)
    else:
        result["content"] = ""

    return result


def insert_item(item, detail, conn):
    import sqlite3
    c = conn.cursor()

    source_url = item["url"]
    title = detail.get("title", item["title"])
    pub_date = detail.get("pub_date", item["date"])
    content = detail.get("content", "")

    # 验证正文
    text_content = re.sub(r"<[^>]+>", "", content).strip() if content else ""
    if not content or len(text_content) < 50:
        print(f"  ⚠ 正文过短: {title[:30]}... ({len(text_content)} chars)")

    try:
        c.execute("""
            INSERT OR IGNORE INTO gov_raw 
            (title, page_url, source_url, content, publish_date, site_name)
            VALUES (?, ?, ?, ?, ?, ?)
        """, (
            title,
            source_url,
            source_url,
            content,
            pub_date,
            SITE_NAME
        ))
        affected = c.rowcount
        conn.commit()
        if affected > 0:
            print(f"  ✓ {title[:30]}...")
        return affected
    except Exception as e:
        print(f"  ✗ 插入失败: {e}")
        return 0


def crawl(days_back=365):
    import sqlite3

    now = datetime.now()
    cutoff = now - timedelta(days=days_back)

    resp = session.get(LIST_URL, timeout=30)
    resp.encoding = "utf-8"
    total_pages = get_total_pages(resp.text)

    print(f"共 {total_pages} 页")

    conn = sqlite3.connect(os.getenv("SEARCH_DB", "/root/search.db"), timeout=60)
    total = 0
    skip_count = 0

    for page in range(total_pages):
        if page == 0:
            url = LIST_URL
        else:
            url = f"{BASE_URL}/index_{page}.html"

        print(f"\n--- 第 {page+1}/{total_pages} 页 ---")
        try:
            resp = session.get(url, timeout=30)
            resp.encoding = "utf-8"
        except Exception as e:
            print(f"  ✗ 请求失败: {e}")
            continue

        items = parse_list_page(resp.text)
        print(f"  发现 {len(items)} 条")

        for item in items:
            # 日期过滤
            try:
                item_date = datetime.strptime(item["date"][:10], "%Y-%m-%d")
                if item_date < cutoff and days_back < 3650:
                    skip_count += 1
                    continue
            except ValueError:
                pass

            # 详情页
            try:
                resp2 = session.get(item["url"], timeout=30)
                resp2.encoding = "utf-8"
            except Exception as e:
                print(f"  ✗ 详情失败 {item['url'][:60]}: {e}")
                continue

            detail = parse_detail(resp2.text, item["url"])
            affected = insert_item(item, detail, conn)
            if affected > 0:
                total += 1

    conn.close()
    print(f"\n=== 完成: 新增 {total} 条, 跳过 {skip_count} 条 ===")
    return total


if __name__ == "__main__":
    days = int(sys.argv[1]) if len(sys.argv) > 1 else 365
    crawl(days_back=days)