#!/usr/bin/env python3
"""
环保公示网 (www.huanbao58.com) — 全部栏目
=========================================
企业自行发布的环保公示聚合平台。
列表: /?page=N (20条/页)
详情: detail.php?itemid=N (内容在 table.detailtb)
"""
import sys, os, re, urllib.parse, warnings
warnings.filterwarnings("ignore")

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

import requests
from bs4 import BeautifulSoup

SITE_NAME = "环保公示网-全部"
BASE_URL = "https://www.huanbao58.com"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}


def fetch(url, retries=3):
    for attempt in range(retries):
        try:
            r = requests.get(url, headers=HEADERS, timeout=30)
            r.encoding = 'utf-8'
            return r.text
        except Exception as e:
            if attempt < retries - 1:
                import time
                time.sleep(2)
            else:
                print(f"  [ERROR] {url[:80]}: {e}", file=sys.stderr)
                return None


def fetch_list_page(page_no):
    """Fetch a page of the '全部' list"""
    if page_no == 1:
        url = BASE_URL + "/"
    else:
        url = BASE_URL + f"/?page={page_no}"
    text = fetch(url)
    if not text:
        return []
    soup = BeautifulSoup(text, 'html.parser')
    items = []
    table = soup.find('table')
    if not table:
        return items
    for tr in table.find_all('tr'):
        a = tr.find('a')
        if not a:
            continue
        href = a.get('href', '')
        if 'detail.php?itemid=' not in href:
            continue
        itemid = re.search(r'itemid=(\d+)', href)
        if not itemid:
            continue
        # Title: from the link text (skip icon text)
        title = a.get_text(strip=True)
        # Date from last td
        tds = tr.find_all('td')
        date = tds[-1].get_text(strip=True) if len(tds) > 1 else ""
        detail_url = urllib.parse.urljoin(BASE_URL, href)
        items.append({"title": title, "url": detail_url, "date": date, "itemid": itemid.group(1)})
    return items


def parse_detail(detail_url):
    """Extract content from detail page"""
    text = fetch(detail_url)
    if not text:
        return None, None, None, None, []
    soup = BeautifulSoup(text, 'html.parser')
    # Title from <title>
    title_tag = soup.find('title')
    title = title_tag.get_text(strip=True) if title_tag else ""
    title = re.sub(r'-环保公示网$', '', title).strip()
    # Content from table.detailtb
    table = soup.find('table', class_='detailtb')
    if not table:
        return title, None, None, None, []
    pub_date = ""
    attachments = []
    cate_type = ""
    field_rows = []  # collect all key-value rows
    
    for tr in table.find_all('tr'):
        th = tr.find('th')
        td = tr.find('td')
        if not th or not td:
            continue
        key = th.get_text(strip=True)
        
        if key == '类型':
            cate_type = td.get_text(strip=True)
            field_rows.append(f"**{key}**: {cate_type}")
        elif key == '标题':
            txt = td.get_text(strip=True)
            field_rows.append(f"**{key}**: {txt}")
            if txt and not title:
                title = txt
        elif key == '公示日期':
            pub_date = td.get_text(strip=True)
            field_rows.append(f"**{key}**: {pub_date}")
        elif key == '详细介绍':
            # Content in <pre> or plain text
            pre = td.find('pre')
            if pre:
                txt = pre.get_text(strip=True)
            else:
                txt = td.get_text(strip=True)
            if txt:
                field_rows.append(f"**{key}**:\n\n{txt}")
        elif key == '附件列表':
            attach_links = []
            for a in td.find_all('a'):
                href = a.get('href', '')
                fname = a.get_text(strip=True) or ""
                if '/download.php' in href:
                    full_url = urllib.parse.urljoin(BASE_URL, href)
                    attachments.append({"name": fname, "url": full_url})
                    if fname:
                        attach_links.append(f"- [{fname}]({full_url})")
                    else:
                        attach_links.append(f"- [下载]({full_url})")
            if attach_links:
                field_rows.append(f"**附件列表**:\n" + "\n".join(attach_links))
    
    # Combine all: field rows as the main body
    body = "\n\n".join(field_rows) if field_rows else None
    return title, pub_date, cate_type, body, attachments


def crawl(pages=5):
    all_items = []
    # 站点仅首页服务端渲染列表（20条），翻页通过Vue前端加载，最多取首页
    for page_no in range(1, 2):  # only page 1 is server-rendered
        items = fetch_list_page(page_no)
        if not items:
            break
        all_items.extend(items)
        print(f"Page {page_no}: {len(items)} items", flush=True)
    print(f"\nTotal list items: {len(all_items)}", flush=True)
    results = []
    empty_body = 0
    with_attach = 0
    for i, item in enumerate(all_items):
        title = item["title"]
        detail_url = item["url"]
        list_date = item["date"]
        parsed_title, pub_date, cate_type, body, attachments = parse_detail(detail_url)
        final_title = parsed_title or title
        final_date = pub_date or list_date
        attach_str = ";".join([a["name"] for a in attachments]) if attachments else ""
        if attachments:
            with_attach += 1
        if not body:
            empty_body += 1
        results.append({
            "site_name": SITE_NAME,
            "title": final_title,
            "url": detail_url,
            "source_url": detail_url,
            "pub_date": final_date,
            "content": body or "",
            "summary": (re.sub(r'\s+', ' ', body or "")[:300]) if body else "",
            "attachments": attach_str,
            "category": cate_type or "",
            "tags": f"huanbao58,{cate_type}" if cate_type else "huanbao58",
        })
        if (i + 1) % 10 == 0:
            print(f"  Parsed: {i+1}/{len(all_items)}", flush=True)
    print(f"Parsed: {len(results)} items ({empty_body} empty body, {with_attach} with attachments)", flush=True)
    push_to_searchdb(results, "huanbao58_all")


if __name__ == "__main__":
    pages = int(sys.argv[sys.argv.index("--pages") + 1]) if "--pages" in sys.argv else 5
    crawl(pages=pages)