#!/usr/bin/env python3
"""
黄冈市生态环境局 - 环保公示 (hbj.hg.gov.cn)
列表: /zwgk/public/column/6636213?type=4&catId=7026845&action=list
详情: /zwgk/public/6636213/{contentId}.html
CMS: 政府信息公开平台
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta
import requests, urllib3
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

SITE_NAME = "黄冈市生态环境局"
BASE_URL = "https://hbj.hg.gov.cn"
LIST_URL = "https://hbj.hg.gov.cn/zwgk/public/column/6636213?type=4&catId=7026845&action=list"
MAX_PAGES = 5
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=20, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except:
        return None

def parse_list(html):
    """<a href="http://hbj.hg.gov.cn/zwgk/public/...">title</a><span>date</span>"""
    items = []
    for m in re.finditer(r'<a[^>]*href="((?:https?:)?//hbj\.hg\.gov\.cn[^"]*/public/[^"]+)"[^>]*>(.*?)</a>\s*<span[^>]*>(\d{4}-\d{2}-\d{2})', html, re.DOTALL):
        href = m.group(1)
        title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        date = m.group(3).strip()
        if title and href:
            items.append((title, href, date))
    return items

def get_total_pages(html):
    pages = re.findall(r'page_(\d+)', html)
    if pages: return max(int(p) for p in pages)
    return 1

def fetch_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None
    title = ""
    m = re.search(r'<title>([^<]+)', html)
    if m: title = re.sub(r'[-_—].*', '', m.group(1)).strip()
    content = ""
    # 黄冈正文在 div.wzcon (j-fontContent)
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, 'html.parser')
    wz_div = soup.find('div', class_='wzcon') or soup.find('div', class_='ls-article-info')
    if wz_div:
        raw = str(wz_div)
        raw = re.sub(r'<script[^>]*>.*?</script>', '', raw, flags=re.DOTALL|re.I)
        raw = re.sub(r'<style[^>]*>.*?</style>', '', raw, flags=re.DOTALL|re.I)
        raw = re.sub(r' style="[^"]*"', '', raw)
        raw = re.sub(r'<div[^>]*class="clear"[^>]*>.*?</div>', '', raw, flags=re.DOTALL)
        content = raw.strip()
    if not content:
        m = re.search(r'(<p[^>]*class="MsoNormal"[^>]*>.*?</p>)', html, re.DOTALL)
        if m: content = m.group(1)
    date = ""
    m = re.search(r'PubDate["\'][^>]*content=["\']([^"\']+)', html, re.I)
    if m: date = m.group(1)[:10]
    return title, content, date

def run(args=None):
    incremental = False
    if args and '1' in args:
        incremental = True
    print(f"爬取: {SITE_NAME}")
    max_pages = 1 if incremental else MAX_PAGES
    results = []
    
    for page in range(1, max_pages + 1):
        url = LIST_URL + f"&page={page}" if page > 1 else LIST_URL
        html = fetch(url)
        if not html:
            continue
        items = parse_list(html)
        if not items:
            if page > 1: break
            continue
        print(f"  第{page}页: {len(items)} 条")
        
        for i, (title, url2, date) in enumerate(items):
            print(f"  [{i+1}/{len(items)}] {title[:40]}...", end=" ", flush=True)
            dt, content, d2 = fetch_detail(url2)
            if not content or len(content) < 30:
                print("内容过短")
                continue
            results.append({
                "site_name": SITE_NAME, "title": dt or title,
                "url": url2, "content": content,
                "pub_date": d2 or date,
                "category": "环评公示", "tags": "黄冈",
            })
            print(f"✅ ({len(content)}字)")
            time.sleep(0.3)
        
        if incremental:
            break
        time.sleep(0.5)
    
    if results:
        print(f"\n  入库 {len(results)} 条")
        push_to_searchdb(results, "hg_hjbh")
    print("完成")

if __name__ == "__main__":
    run(sys.argv[1] if len(sys.argv)>1 else None)