#!/usr/bin/env python3
"""crawl_yangxin_hj.py — 阳信县 建设项目环境影响评价 (yangxin.gov.cn)"""
import re, sys, time, json
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "http://www.yangxin.gov.cn"
SITE_NAME = "阳信县人民政府-建设项目环境影响评价"
COLUMN_NAME = "建设项目环境影响评价"
# 环评栏目在公共监管(YX057)下，colid=271893
INFOTYPE_ID = "YX057"
COLID = 271893

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
session = requests.Session()
session.headers.update(HEADERS)


def fetch_list_page(page):
    """通过 POST 获取列表页"""
    url = "http://www.yangxin.gov.cn/module/xxgk/search.jsp"
    data = {
        "infotypeId": INFOTYPE_ID,
        "jdid": "1",
        "area": "",
        "divid": "div4",
        "vc_title": "",
        "vc_number": "",
        "currpage": str(page),
        "isAllList": "1",
        "standardXxgk": "1",
    }
    try:
        resp = session.post(url, data=data, timeout=30)
        resp.encoding = "utf-8"
        return resp.text
    except Exception as e:
        print(f"  \u274c 列表页请求失败: {e}")
        return None


def parse_list(html):
    """提取 art_271893 (建设项目环境影响评价) 的列表项"""
    items = []
    soup = BeautifulSoup(html, "html.parser")
    for li in soup.find_all("li"):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"]
        # 只取 colid=271893 的环评项目
        if f"art_{COLID}" not in href:
            continue
        title = a.get("title") or a.get_text(strip=True)
        url = urljoin(BASE_URL, href)
        # 日期
        b = li.find("b")
        date = b.get_text(strip=True) if b else ""
        items.append((title, url, date))
    return items


def extract_detail(html, url):
    soup = BeautifulSoup(html, "html.parser")
    # 标题
    title = ""
    h1 = soup.select_one("div.artcle h1")
    if h1:
        title = h1.get_text(strip=True)
    # 日期
    date = ""
    m = re.search(r"发布日期[：:](\d{4}-\d{2}-\d{2})", html)
    if m:
        date = m.group(1)
    # 正文: div.content 内的 p 和 table
    content_parts = []
    content_div = soup.find("div", class_="content")
    if content_div:
        for child in content_div.children:
            if child.name == "p":
                text = child.get_text(strip=True)
                if text:
                    content_parts.append(text)
            elif child.name == "div":
                # 可能有表格在 div 内
                for t in child.find_all("table"):
                    md = table_to_markdown(t)
                    if md:
                        content_parts.append(md)
                for p in child.find_all("p"):
                    text = p.get_text(strip=True)
                    if text:
                        content_parts.append(text)
            elif child.name == "table":
                md = table_to_markdown(child)
                if md:
                    content_parts.append(md)
    content = "\n\n".join(content_parts)

    # 附件: content 内的下载链接
    attachments = []
    if content_div:
        for a_tag in content_div.find_all("a", href=True):
            href = a_tag["href"]
            if "/module/download/downfile.jsp" in href or re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", href, re.I):
                fname = a_tag.get_text(strip=True) or href.split("/")[-1]
                furl = urljoin(BASE_URL, href)
                attachments.append({"name": fname, "url": furl})

    return title, content, date, attachments


def table_to_markdown(table, *args, **kwargs):
    """保留 HTML 表格结构（不转 md）"""
    return str(table)

def main():
    import argparse
    parser = argparse.ArgumentParser(description="阳信县建设项目环境影响评价爬虫")
    parser.add_argument("--pages", type=int, default=5)
    args = parser.parse_args()

    sys.path.insert(0, "/root/gov_crawler")
    from crawler_lib import push_to_searchdb

    total = 0
    for page in range(1, args.pages + 1):
        print(f"[列表 {page}/{args.pages}]", end=" ", flush=True)
        html = fetch_list_page(page)
        if not html:
            continue
        items = parse_list(html)
        if not items:
            print(f"\u26a0\ufe0f 无环评项")
            continue
        print(f"  {len(items)} 条环评")
        for title, detail_url, list_date in items:
            print(f"    \u2192 {title[:40]}...", end=" ", flush=True)
            try:
                resp = session.get(detail_url, timeout=30)
                resp.encoding = "utf-8"
            except Exception as e:
                print(f"\u274c {e}")
                continue
            det_title, content, det_date, attachments = extract_detail(resp.text, detail_url)
            use_title = det_title or title
            use_date = det_date or list_date
            summary = content[:200].replace("\n", " ") if content else use_title
            if attachments:
                links = "\n\n**附件：**\n"
                for att in attachments:
                    links += f"- [{att['name']}]({att['url']})\n"
                content += links
            push_to_searchdb(
                items=[{
                    "title": use_title,
                    "content": content,
                    "summary": summary,
                    "url": detail_url,
                    "site_name": SITE_NAME,
                    "pub_date": use_date,
                    "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
                }]
            )
            total += 1
            print(f"\u2705 {len(content)}字" if content else "\u26a0\ufe0f 空")
        time.sleep(0.5)
    print(f"\n=== 完成！共入库 {total} 条 ===")


if __name__ == "__main__":
    sys.exit(main())