#!/usr/bin/env python3
"""crawl_huineng_anoce.py — 汇能控股集团 公示公告 (huinenggroup.com)"""
import re, sys, time, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "http://www.huinenggroup.com"
SITE_NAME = "汇能控股集团有限公司-公示公告"
LIST_PATH = "/web/html/anoce"
PER_PAGE = 10

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
}
session = requests.Session()
session.headers.update(HEADERS)


def fetch_list_page(page_num):
    if page_num == 1:
        url = f"{BASE_URL}{LIST_PATH}/"
    else:
        url = f"{BASE_URL}{LIST_PATH}/list{page_num}.html"
    try:
        resp = session.get(url, timeout=30)
        resp.encoding = "utf-8"
        return resp.text
    except Exception as e:
        print(f"  \u274c \u5217\u8868\u9875{page_num}\u8bf7\u6c42\u5931\u8d25: {e}")
        return None


def parse_list(html):
    items = []
    soup = BeautifulSoup(html, "html.parser")
    newslist = soup.find("div", class_="newslist")
    if not newslist:
        return items
    for li in newslist.find_all("li"):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"]
        title_h3 = li.find("h3")
        title = title_h3.get_text(strip=True) if title_h3 else ""
        if not title:
            continue
        date_div = li.find("div", class_="date")
        pub_date = ""
        if date_div:
            d1 = date_div.find("p", class_="d1")
            d2 = date_div.find("p", class_="d2")
            if d1 and d2:
                day = d1.get_text(strip=True)
                month_year = d2.get_text(strip=True)
                pub_date = f"{month_year}-{day}"
        full_url = urljoin(f"{BASE_URL}{LIST_PATH}/", href)
        items.append({"title": title, "url": full_url, "date": pub_date})
    return items


def extract_detail(html, url):
    soup = BeautifulSoup(html, "html.parser")
    title = ""
    title_tag = soup.find("title")
    if title_tag:
        title = title_tag.get_text(strip=True)
    date = ""
    content = ""
    content_div = soup.find("div", class_="content")
    if content_div:
        parts = []
        processed = set()
        for el in content_div.find_all(["p", "table", "img", "h1", "h2", "h3", "h4"], recursive=True):
            if id(el) in processed:
                continue
            processed.add(id(el))
            if el.name == "table":
                md = table_to_markdown(el)
                if md:
                    parts.append(md)
            elif el.name == "img":
                src = el.get("src", "")
                if src:
                    alt = el.get("alt", "")
                    full_src = urljoin(url, src) if not src.startswith("http") else src
                    parts.append(f"![{alt}]({full_src})" if alt else f"![]({full_src})")
            elif el.name == "p":
                txt = el.get_text(" ", strip=True)
                if txt:
                    txt = re.sub(r"[^\S\n]+", " ", txt)
                    parts.append(txt)
            else:
                txt = el.get_text(" ", strip=True)
                if txt:
                    parts.append(txt)
        content = "\n\n".join(parts)
    attachments = []
    for a_tag in soup.find_all("a", href=True):
        href = a_tag["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", href, re.I):
            fname = a_tag.get_text(strip=True) or href.split("/")[-1]
            full_url = urljoin(url, href)
            attachments.append({"name": fname, "url": full_url})
    return title, content, date, attachments


def table_to_markdown(table, *args, **kwargs):
    """保留 HTML 表格结构（不转 md）"""
    return str(table)

def main():
    import argparse
    parser = argparse.ArgumentParser(description="汇能控股集团公示公告爬虫")
    parser.add_argument("--pages", type=int, default=5)
    args = parser.parse_args()

    sys.path.insert(0, "/root/gov_crawler")
    from crawler_lib import push_to_searchdb

    total = 0
    for page in range(1, args.pages + 1):
        print(f"[\u5217\u8868 {page}/{args.pages}]", end=" ", flush=True)
        html = fetch_list_page(page)
        if not html:
            continue
        items = parse_list(html)
        if not items:
            print("\u26a0\ufe0f \u65e0\u6570\u636e")
            break
        print(f"{len(items)} \u6761")
        for item in items:
            detail_url = item["url"]
            print(f"  \u2192 {item['title'][:40]}...", end=" ", flush=True)
            try:
                resp = session.get(detail_url, timeout=30)
                resp.encoding = "utf-8"
            except Exception as e:
                print(f"\u274c {e}")
                continue
            title, content, det_date, attachments = extract_detail(resp.text, detail_url)
            use_title = title or item["title"]
            use_date = det_date or item["date"]
            summary = content[:200].replace("\n", " ") if content else use_title
            if attachments:
                links = "\n\n**\u9644\u4ef6\uff1a**\n"
                for att in attachments:
                    links += f"- [{att['name']}]({att['url']})\n"
                content += links
            push_to_searchdb(
                items=[{
                    "title": use_title, "content": content, "summary": summary,
                    "url": detail_url, "site_name": SITE_NAME,
                    "pub_date": use_date,
                    "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
                }]
            )
            total += 1
            print(f"\u2705 {len(content)}\u5b57" if content else "\u26a0\ufe0f \u7a7a")
            time.sleep(0.3)
        time.sleep(0.5)

    print(f"\n=== \u5b8c\u6210\uff01\u5171\u5165\u5e93 {total} \u6761 ===")


if __name__ == "__main__":
    sys.exit(main())