#!/usr/bin/env python3
"""crawl_sxylny_qtxx.py — 陕西榆林能源集团 其他信息 (sxylny.com)"""
import re, sys, time, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://www.sxylny.com"
SITE_NAME = "陕西榆林能源集团有限公司-其他信息"
LIST_PATH = "/html/qtxx"

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
}
session = requests.Session()
session.headers.update(HEADERS)


def fetch_list_page(page_num):
    if page_num == 1:
        url = f"{BASE_URL}{LIST_PATH}/"
    else:
        url = f"{BASE_URL}{LIST_PATH}/qtxx_{page_num}.html"
    try:
        resp = session.get(url, timeout=30)
        resp.encoding = "utf-8"
        return resp.text
    except Exception as e:
        print(f"  \u274c \u5217\u8868\u9875{page_num}\u5931\u8d25: {e}")
        return None


def parse_list(html):
    items = []
    soup = BeautifulSoup(html, "html.parser")
    xl = soup.find("div", class_="xinwen-list")
    if not xl:
        return items
    dl = xl.find("dl")
    if not dl:
        return items
    for dd in dl.find_all("dd"):
        a = dd.find("a", href=True)
        if not a:
            continue
        href = a["href"]
        title = a.get_text(strip=True)
        if not title:
            continue
        date_div = dd.find("div", class_="xinwen-date")
        pub_date = ""
        if date_div:
            h1 = date_div.find("h1")
            h2 = date_div.find("h2")
            if h1 and h2:
                day = h1.get_text(strip=True)
                ym = h2.get_text(strip=True)
                pub_date = f"{ym}-{day}"
        full_url = href if href.startswith("http") else urljoin(BASE_URL, href)
        items.append({"title": title, "url": full_url, "date": pub_date})
    return items


def extract_detail(html, url):
    soup = BeautifulSoup(html, "html.parser")
    title = ""
    h1 = soup.find("div", class_="danpian-h1")
    if h1:
        title = h1.get_text(strip=True)
    date = ""
    h2 = soup.find("div", class_="danpian-h2")
    if h2:
        m = re.search(r"(\d{4}-\d{2}-\d{2})", h2.get_text())
        if m:
            date = m.group(1)
    content = ""
    con = soup.find("div", class_="danpian-con")
    if con:
        parts = []
        processed = set()
        for el in con.find_all(["p", "table", "img", "h1", "h2", "h3", "h4"], recursive=True):
            if id(el) in processed:
                continue
            processed.add(id(el))
            if el.name == "table":
                md = table_to_markdown(el)
                if md:
                    parts.append(md)
            elif el.name == "img":
                src = el.get("src", "")
                if src:
                    alt = el.get("alt", "")
                    full_src = src if src.startswith("http") else urljoin(url, src)
                    parts.append(f"![{alt}]({full_src})" if alt else f"![]({full_src})")
            elif el.name == "p":
                txt = el.get_text(" ", strip=True)
                if txt:
                    txt = re.sub(r"[^\S\n]+", " ", txt)
                    parts.append(txt)
            else:
                txt = el.get_text(" ", strip=True)
                if txt:
                    parts.append(txt)
        content = "\n\n".join(parts)
    attachments = []
    for a_tag in soup.find_all("a", href=True):
        href = a_tag["href"]
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", href, re.I):
            fname = a_tag.get_text(strip=True) or href.split("/")[-1]
            full_url = href if href.startswith("http") else urljoin(url, href)
            attachments.append({"name": fname, "url": full_url})
    return title, content, date, attachments


def table_to_markdown(table, *args, **kwargs):
    """保留 HTML 表格结构（不转 md）"""
    return str(table)

def main():
    import argparse
    parser = argparse.ArgumentParser(description="陕西榆林能源集团其他信息爬虫")
    parser.add_argument("--pages", type=int, default=5)
    args = parser.parse_args()

    sys.path.insert(0, "/root/gov_crawler")
    from crawler_lib import push_to_searchdb

    total = 0
    for page in range(1, args.pages + 1):
        print(f"[\u5217\u8868 {page}/{args.pages}]", end=" ", flush=True)
        html = fetch_list_page(page)
        if not html:
            continue
        items = parse_list(html)
        if not items:
            print("\u26a0\ufe0f \u65e0\u6570\u636e")
            break
        print(f"{len(items)} \u6761")
        for item in items:
            detail_url = item["url"]
            print(f"  \u2192 {item['title'][:40]}...", end=" ", flush=True)
            try:
                resp = session.get(detail_url, timeout=30)
                resp.encoding = "utf-8"
            except Exception as e:
                print(f"\u274c {e}")
                continue
            title, content, det_date, attachments = extract_detail(resp.text, detail_url)
            use_title = title or item["title"]
            use_date = det_date or item["date"]
            summary = content[:200].replace("\n", " ") if content else use_title
            if attachments:
                links = "\n\n**\u9644\u4ef6\uff1a**\n"
                for att in attachments:
                    links += f"- [{att['name']}]({att['url']})\n"
                content += links
            push_to_searchdb(
                items=[{
                    "title": use_title, "content": content, "summary": summary,
                    "url": detail_url, "site_name": SITE_NAME,
                    "pub_date": use_date,
                    "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
                }]
            )
            total += 1
            print(f"\u2705 {len(content)}\u5b57" if content else "\u26a0\ufe0f \u7a7a")
            time.sleep(0.3)
        time.sleep(0.5)

    print(f"\n=== \u5b8c\u6210\uff01\u5171\u5165\u5e93 {total} \u6761 ===")


if __name__ == "__main__":
    sys.exit(main())