#!/usr/bin/env python3
"""爬取 hfsikang.com - 环评公示栏目"""
import requests
from bs4 import BeautifulSoup
import sqlite3
import os
import re
import sys
from datetime import datetime

BASE_URL = "http://www.hfsikang.com"
LIST_URL = "http://www.hfsikang.com/category/eia-publicity"
SITE_NAME = "hfsikang.com-环评公示"
CUTOFF_DATE = "2023-06-16"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Referer": "http://www.hfsikang.com",
}

def parse_publish_date(text):
    """解析发布时间 2026/06/10 → 2026-06-10"""
    text = text.strip().replace("发布时间：", "").replace("发布时间:", "").strip()
    try:
        return datetime.strptime(text, "%Y/%m/%d").strftime("%Y-%m-%d")
    except:
        try:
            return datetime.strptime(text, "%Y-%m-%d").strftime("%Y-%m-%d")
        except:
            return text

def get_page_count():
    """获取总页数"""
    resp = requests.get(LIST_URL, headers=HEADERS, timeout=30)
    resp.encoding = "utf-8"
    soup = BeautifulSoup(resp.text, "html.parser")
    pager = soup.select_one("div.pager")
    if not pager:
        return 1
    links = pager.find_all("a")
    last_page = 1
    for a in links:
        txt = a.get_text(strip=True)
        if txt.isdigit():
            last_page = max(last_page, int(txt))
    return last_page

def get_list_page(page):
    """获取某一页的标题/链接/时间"""
    if page == 1:
        url = LIST_URL
    else:
        url = f"{LIST_URL}/page/{page}"
    resp = requests.get(url, headers=HEADERS, timeout=30)
    resp.encoding = "utf-8"
    soup = BeautifulSoup(resp.text, "html.parser")
    items = []
    for li in soup.select("ul.default > li"):
        title_tag = li.select_one("h2 a.titels")
        if not title_tag:
            continue
        title = title_tag.get_text(strip=True)
        href = title_tag.get("href", "")
        if href and not href.startswith("http"):
            href = BASE_URL + href
        
        time_em = li.select_one("p.infot em")
        pub_date = ""
        if time_em:
            pub_date = parse_publish_date(time_em.get_text(strip=True))
        
        items.append({"title": title, "url": href, "publish_date": pub_date, "page": page})
    return items

def get_detail(url):
    """获取详情页的正文HTML"""
    resp = requests.get(url, headers=HEADERS, timeout=30)
    resp.encoding = "utf-8"
    soup = BeautifulSoup(resp.text, "html.parser")
    enter = soup.select_one("div.enter.suojin")
    if enter:
        # 移除无关元素
        for tag in enter.select("div.bqc, div.bdsharebuttonbox, div.next_post, script, style"):
            tag.decompose()
        return str(enter)
    return ""

def crawl():
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    
    # 创建表（如果不存在）
    c.execute("""
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT,
            page_url TEXT UNIQUE,
            content TEXT,
            publish_date TEXT,
            site_name TEXT DEFAULT '',
            crawl_time TEXT DEFAULT (datetime('now', '+8 hours')),
            similar TEXT
        )
    """)
    conn.commit()
    
    total_pages = get_page_count()
    print(f"总页数: {total_pages}")
    
    total_added = 0
    total_skipped = 0
    cutoff_reached = False
    
    for page in range(1, total_pages + 1):
        if cutoff_reached:
            break
        print(f"  第{page}页...")
        items = get_list_page(page)
        for item in items:
            pub = item["publish_date"]
            if pub and pub < CUTOFF_DATE:
                cutoff_reached = True
                print(f"    截止日期 {CUTOFF_DATE}，跳过后续")
                break
            
            # 先检查是否已存在
            c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (item["url"],))
            if c.fetchone():
                total_skipped += 1
                continue
            
            content = get_detail(item["url"])
            if not content:
                print(f"    警告: 空正文跳过 {item['title'][:30]}")
                total_skipped += 1
                continue
            
            c.execute(
                "INSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, site_name) VALUES (?, ?, ?, ?, ?)",
                (item["title"], item["url"], content, pub, SITE_NAME)
            )
            if c.rowcount > 0:
                total_added += 1
            conn.commit()
        
        # 如果本页已经遇到截止日期，不再翻下一页
        if cutoff_reached:
            break
    
    conn.close()
    print(f"完成: 新增 {total_added} 条, 跳过 {total_skipped} 条")

if __name__ == "__main__":
    crawl()