#!/usr/bin/env python3
"""crawl_wengan.py — 瓮安县人民政府 通知公告 (wengan.gov.cn)"""
import re, sys, time, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://www.wengan.gov.cn"
LIST_BASE = "/xw/tzgg/"
SITE_NAME = "瓮安县人民政府"
COLUMN_NAME = "通知公告"

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)

TOTAL_PAGES = 464


def get_list_url(page):
    if page == 1:
        return f"{BASE_URL}{LIST_BASE}"
    return f"{BASE_URL}{LIST_BASE}index_{page-1}.html"


def parse_list(html):
    items = []
    soup = BeautifulSoup(html, "html.parser")
    ul = soup.find("ul", class_="NewsList")
    if not ul:
        return items
    for li in ul.find_all("li"):
        a = li.find("a", href=True)
        span = li.find("span")
        if not a:
            continue
        title = a.get("title") or a.get_text(strip=True)
        href = a["href"]
        url = urljoin(BASE_URL, href)
        date = span.get_text(strip=True) if span else ""
        if date:
            m = re.search(r"(\d{4}-\d{2}-\d{2})", date)
            date = m.group(1) if m else ""
        items.append((title, url, date))
    return items


def extract_detail(html, url):
    soup = BeautifulSoup(html, "html.parser")
    title = ""
    m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]+)"', html)
    if m:
        title = m.group(1).strip()
    if not title:
        t = soup.find("title")
        if t:
            title = t.get_text(strip=True)

    date = ""
    m = re.search(r"var\s+pubdata\s*=\s*'(\d{4}-\d{2}-\d{2})", html)
    if m:
        date = m.group(1)
    if not date:
        m = re.search(r'PubDate"\s*content="(\d{4}-\d{2}-\d{2})', html)
        if m:
            date = m.group(1)

    content_parts = []
    zoom = soup.find("font", id="Zoom")
    if zoom:
        editor = zoom.find("div", class_="trs_editor_view")
        container = editor if editor else zoom
        for child in container.children:
            if child.name == "p":
                text = child.get_text(strip=True)
                # 图片优先
                imgs = child.find_all("img")
                for img in imgs:
                    src = img.get("src", "")
                    alt = img.get("alt", "") or img.get("title", "") or ""
                    if src:
                        src = urljoin(BASE_URL, src)
                        content_parts.append(f"![{alt}]({src})")
                if text:
                    content_parts.append(text)
            elif child.name == "table":
                md = table_to_markdown(child)
                if md:
                    content_parts.append(md)
            elif child.name == "div" and "ue_table" in child.get("class", []):
                for t in child.find_all("table"):
                    md = table_to_markdown(t)
                    if md:
                        content_parts.append(md)

    content = "\n\n".join(content_parts)

    attachments = []
    zoom = soup.find("font", id="Zoom")
    if zoom:
        for a_tag in zoom.find_all("a", href=True):
            href = a_tag["href"]
            if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", href, re.I):
                fname = a_tag.get_text(strip=True) or href.split("/")[-1]
                furl = urljoin(BASE_URL, href)
                attachments.append({"name": fname, "url": furl})

    return title, content, date, attachments


def table_to_markdown(table, *args, **kwargs):
    """保留 HTML 表格结构（不转 md）"""
    return str(table)

def main():
    import argparse
    parser = argparse.ArgumentParser(description="瓮安县通知公告爬虫")
    parser.add_argument("--pages", type=int, default=10)
    args = parser.parse_args()

    sys.path.insert(0, "/root/gov_crawler")
    from crawler_lib import push_to_searchdb

    total = 0
    pages = min(args.pages, TOTAL_PAGES)
    for page in range(1, pages + 1):
        url = get_list_url(page)
        print(f"[列表 {page}/{pages}] {url}")
        try:
            resp = session.get(url, timeout=30)
            resp.encoding = "utf-8"
        except Exception as e:
            print(f"  \u274c {e}")
            continue
        items = parse_list(resp.text)
        if not items:
            print(f"  \u26a0\ufe0f 无列表项")
            break
        print(f"  \U0001f4c4 {len(items)} 条")
        for title, detail_url, list_date in items:
            print(f"    \u2192 {title[:40]}...", end=" ", flush=True)
            try:
                resp2 = session.get(detail_url, timeout=30)
                resp2.encoding = "utf-8"
            except Exception as e:
                print(f"\u274c {e}")
                continue
            det_title, content, det_date, attachments = extract_detail(resp2.text, detail_url)
            use_title = det_title or title
            use_date = det_date or list_date
            summary = content[:200].replace("\n", " ") if content else use_title
            if attachments:
                links = "\n\n**附件：**\n"
                for att in attachments:
                    links += f"- [{att['name']}]({att['url']})\n"
                content += links
            push_to_searchdb(
                items=[{
                    "title": use_title,
                    "content": content,
                    "summary": summary,
                    "url": detail_url,
                    "site_name": SITE_NAME,
                    "pub_date": use_date,
                    "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
                }]
            )
            total += 1
            status = "ok" if content else "empty"
            print(f"\u2705 {len(content)}字" if content else "\u26a0\ufe0f 空")

        time.sleep(0.5)

    print(f"\n=== 完成！共入库 {total} 条 ===")


if __name__ == "__main__":
    sys.exit(main())