#!/usr/bin/env python3
import os
"""淮南煤化工产业园-通知公告爬虫 ahccci.huainan.gov.cn"""
import re, sys, time, json, urllib.request, urllib.error, sqlite3, ssl
from datetime import datetime, timedelta
from bs4 import BeautifulSoup

BASE_URL = "https://ahccci.huainan.gov.cn"
LIST_API = "https://ahccci.huainan.gov.cn/content/column/12494462"
SITE_NAME = "淮南煤化工产业园-通知公告"
CUTOFF_DATE = (datetime.now() - timedelta(days=365*3)).strftime("%Y-%m-%d")
MAX_PAGES = 10
DELAY = 1.0
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}

def fetch(url, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers=HEADERS)
            with urllib.request.urlopen(req, context=ctx, timeout=20) as r:
                return r.read().decode("utf-8", errors="replace")
        except Exception as e:
            if i < retries - 1:
                time.sleep(DELAY * (i+1))
                continue
            return None

def parse_list(html):
    """Extract (url, title, date) from list HTML"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    ul = soup.find("ul", class_=re.compile(r"doc_list"))
    if not ul:
        return items
    for li in ul.find_all("li"):
        a = li.find("a")
        date_span = li.find("span", class_="date")
        if a and a.get("href") and date_span:
            url = a["href"].strip()
            if not url.startswith("http"):
                url = BASE_URL + url
            title = a.get("title", a.get_text(strip=True))
            date = date_span.get_text(strip=True)
            if re.match(r"\d{4}-\d{2}-\d{2}", date):
                items.append((url, title, date))
    return items

def parse_detail(html, url):
    """Extract content from detail page"""
    soup = BeautifulSoup(html, "html.parser")
    
    title_el = soup.select_one("h1.wztit")
    title = title_el.get_text(strip=True) if title_el else ""
    
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    date = meta_date["content"][:10] if meta_date and meta_date.get("content") else ""
    
    meta_source = soup.find("meta", attrs={"name": "ContentSource"})
    source = meta_source["content"] if meta_source and meta_source.get("content") else ""
    
    content_div = soup.select_one("#zoom.wzcon, div.wzcon")
    content = ""
    if content_div:
        # Remove unnecessary elements
        for el in content_div.find_all(["script", "style"]):
            el.decompose()
        content = str(content_div)
    
    return title, date, source, content

def save_to_db(items):
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    inserted = 0
    for url, title, date, source, content in items:
        try:
            summary = f"来源：{source}" if source else ""
            c.execute("""
                INSERT OR IGNORE INTO gov_raw 
                (page_url, title, publish_date, site_name, source_url, summary, content)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (url, title, date, SITE_NAME, url, summary, content))
            if c.rowcount > 0:
                inserted += 1
            else:
                c.execute("UPDATE gov_raw SET content=? WHERE page_url=? AND (content IS NULL OR content='')",
                         (content, url))
                if c.rowcount > 0:
                    inserted += 1
        except Exception as e:
            print(f"  [ERROR] {url[:60]} - {e}", flush=True)
    conn.commit()
    conn.close()
    return inserted

def main():
    print(f"[hn] 淮南煤化工产业园-通知公告 - 截止: {CUTOFF_DATE}", flush=True)
    
    all_items = []
    for page in range(MAX_PAGES):
        if page == 0:
            url = f"{BASE_URL}/xwzx/tzgg/index.html"
        else:
            url = f"{LIST_API}?pageIndex={page}"
        
        print(f"[hn] 列表 {page+1}: {url}", flush=True)
        html = fetch(url)
        if not html:
            print(f"  [SKIP] 获取失败", flush=True)
            continue
        
        items = parse_list(html)
        if not items:
            print(f"  [END] 空列表", flush=True)
            break
        
        dates = [d for _,_,d in items]
        oldest = min(dates)
        newest = max(dates)
        print(f"  → {len(items)}条, {oldest} ~ {newest}", flush=True)
        
        for url, title, date in items:
            if date >= CUTOFF_DATE:
                all_items.append((url, title, date))
        
        if oldest < CUTOFF_DATE:
            print(f"  [END] {oldest} < 截止", flush=True)
            break
        time.sleep(DELAY)
    
    print(f"\n[hn] 共收集 {len(all_items)} 条", flush=True)
    
    detailed = []
    for i, (url, title, date) in enumerate(all_items, 1):
        print(f"[hn] 详情 {i}/{len(all_items)}: {title[:30]}...", flush=True)
        html = fetch(url)
        if not html:
            detailed.append((url, title, date, "", ""))
            continue
        dt, dd, ds, dc = parse_detail(html, url)
        ft = dt or title
        fd = dd or date
        cl = len(dc) if dc else 0
        print(f"  → 正文 {cl}字", flush=True)
        detailed.append((url, ft, fd, ds, dc))
        time.sleep(DELAY)
    
    inserted = save_to_db(detailed)
    print(f"\n[hn] ✅ 入库 {inserted} 条", flush=True)
    
    conn = sqlite3.connect(DB_PATH)
    try:
        conn.execute("INSERT INTO gov_search(gov_search, rowid, title, site_name, summary) VALUES('rebuild', 0, '', '', '')")
    except:
        pass
    conn.commit()
    conn.close()
    print(f"[hn] ✅ 完成", flush=True)

if __name__ == "__main__":
    main()