#!/usr/bin/env python3
"""常德经济技术开发区 - 通知公告 爬虫
PowerCMS，静态HTML列表+详情页
列表 /zhdt/tzgg ~ /zhdt/tzgg_60，20条/页，共1185条
正文从 <div class="article_txt"> 提取
/etc/hosts: 223.111.128.46 cdjkq.changde.gov.cn (CloudWAF)
注意: CloudWAF限制每秒1次请求，所有请求间隔至少1.5秒
"""

import os
import sys
import re
import time
import requests
from datetime import datetime, timedelta

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "常德经开区-通知公告"
BASE_URL = "https://cdjkq.changde.gov.cn"
TOTAL_PAGES = 5
PAGE_SIZE = 20  # ~20 items per page, ~100 total
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}

CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
print(f"[info] 日期过滤: >= {CUTOFF_DATE}")


REQUEST_INTERVAL = 1.5  # CloudWAF限制每秒1次请求
_last_req_time = 0


def fetch(url):
    global _last_req_time
    # 请求间隔控制
    elapsed = time.time() - _last_req_time
    if elapsed < REQUEST_INTERVAL:
        time.sleep(REQUEST_INTERVAL - elapsed)
    try:
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = 'utf-8'
        _last_req_time = time.time()
        # 检查是否被WAF拦截
        if "疑似正在对网站进行CC攻击" in r.text:
            print(f"[warn] WAF拦截! 等待60秒后重试...")
            time.sleep(60)
            r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
            r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"[error] 请求失败: {url[-60:]}, {e}")
        return ""


def parse_list_items(html):
    """从列表页解析条目: <li> → date + url + title"""
    items = []
    # <li class="first"> or <li>
    #   <span class="date">2026-06-22</span>
    #   <a href="/zhdt/tzgg/content_78207706" target="_blank" title="TITLE">TITLE</a>
    # </li>
    pattern = r'<li[^>]*>\s*<span class="date">([^<]+)</span>\s*<a href="([^"]+)"[^>]*title="([^"]*)"'
    for m in re.finditer(pattern, html):
        date = m.group(1).strip()
        url = m.group(2).strip()
        title = m.group(3).strip()
        if not url.startswith("http"):
            url = BASE_URL + url
        items.append({"title": title, "date": date, "url": url})
    return items


def fetch_detail(html):
    """从详情页提取 <div class="article_txt"> 内正文"""
    # Try to find the article content area
    m = re.search(r'<div class="article_txt"[^>]*>([\s\S]*?)</div>\s*</div>\s*<!--', html)
    if not m:
        m = re.search(r'<div class="article_txt"[^>]*>([\s\S]*?)</div>', html)
    if m:
        content = m.group(1).strip()
        # Remove scripts and styles
        content = re.sub(r'<script[^>]*>[\s\S]*?</script>', '', content)
        content = re.sub(r'<style[^>]*>[\s\S]*?</style>', '', content)
        return content
    return ""


def main():
    import sqlite3

    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    c.execute("PRAGMA journal_mode=WAL")

    # Ensure table exists
    c.execute("""CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        page_url TEXT UNIQUE,
        title TEXT,
        content TEXT,
        site_name TEXT,
        publish_date TEXT,
        summary TEXT,
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
        category TEXT
    )""")
    conn.commit()

    new_count = 0
    skip_count = 0
    old_skip = 0
    err_count = 0

    for page in range(1, TOTAL_PAGES + 1):
        if page == 1:
            url = f"{BASE_URL}/zhdt/tzgg"
        else:
            url = f"{BASE_URL}/zhdt/tzgg_{page}"

        html = fetch(url)
        if not html:
            err_count += 1
            continue

        items = parse_list_items(html)
        if not items:
            print(f"[warn] 第{page}页未解析到条目")
            continue

        page_new = 0
        page_skip = 0
        for item in items:
            date = item["date"]
            if date < CUTOFF_DATE:
                old_skip += 1
                continue

            title = item["title"]
            url = item["url"]

            # 去重
            c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=?", (url,))
            if c.fetchone()[0] > 0:
                page_skip += 1
                continue

            # 详情页
            detail_html = fetch(url)
            if not detail_html:
                err_count += 1
                continue

            content = fetch_detail(detail_html)
            if not content:
                content = f"[无正文] {title}"

            try:
                c.execute("""
                    INSERT OR IGNORE INTO gov_raw (page_url, title, content, site_name, publish_date, summary, category)
                    VALUES (?, ?, ?, ?, ?, ?, ?)
                """, (url, title, content, SITE_NAME, date, title, "通知公告"))
                if c.rowcount > 0:
                    page_new += 1
            except Exception as e:
                print(f"  [error] 入库失败: {e}")
                err_count += 1

        conn.commit()
        new_count += page_new
        skip_count += page_skip
        print(f"[page {page}/{TOTAL_PAGES}] +{page_new} new, {page_skip} dup, total: {new_count}")

    conn.close()
    print(f"\n[完成] {SITE_NAME}: 新增 {new_count} 条, 重复跳过 {skip_count}, 超3年跳过 {old_skip}, 错误 {err_count}")


if __name__ == "__main__":
    main()