#!/usr/bin/env python3
"""广西生态环境厅拟批准项目公示爬虫 (TRS CMS)"""
import sys, re, os, json
sys.path.insert(0, '/root')
sys.path.insert(0, '/root/gov_crawler')
from urllib.parse import urljoin
import urllib.request, ssl
from bs4 import BeautifulSoup

SITE_NAME = "广西生态环境厅拟批准项目"
BASE_URL = "http://sthjt.gxzf.gov.cn/zfxxgk/zfxxgkgl/fdzdgknr/hjjccf/xzxk/npzxm"
PAGES = 5

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE


def fetch_list_page(page_num):
    """获取列表页"""
    if page_num == 1:
        url = f"{BASE_URL}/index.shtml"
    else:
        url = f"{BASE_URL}/index_{page_num-1}.shtml"

    try:
        req = urllib.request.Request(url, headers={
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            "Accept-Language": "zh-CN,zh;q=0.9",
        })
        resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
        html = resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"  [ERROR] {e}")
        return []

    soup = BeautifulSoup(html, 'html.parser')
    items = []

    for li in soup.find_all('li'):
        a_tag = li.find('a', href=re.compile(r't\d+\.shtml'))
        if not a_tag:
            continue
        href = a_tag.get('href', '')
        title = re.sub(r'\s+', ' ', a_tag.get_text(strip=True)).strip()
        if not title or len(title) < 8:
            continue
        # Date: （2026-07-14）
        li_text = li.get_text()
        date_match = re.search(r'（\s*(\d{4}-\d{1,2}-\d{1,2})\s*）', li_text)
        date_str = date_match.group(1) if date_match else ''

        full_url = urljoin(BASE_URL + '/', href)
        items.append({
            'url': full_url,
            'title': title,
            'date': date_str,
        })
    return items


def fetch_detail(url):
    """抓取详情页"""
    try:
        req = urllib.request.Request(url, headers={
            "User-Agent": "Mozilla/5.0",
            "Accept-Language": "zh-CN,zh;q=0.9",
        })
        resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
        html = resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        return "", []

    soup = BeautifulSoup(html, 'html.parser')
    content_div = soup.find('div', class_='article-con')
    if not content_div:
        content_div = soup.find('div', class_='article')

    content = ""
    attachments = []

    if content_div:
        parts = []
        for el in content_div.find_all(['p', 'table', 'img']):
            if el.name == 'p':
                if el.find_parent('table'):
                    continue
                txt = el.get_text(' ', strip=True)
                txt = re.sub(r'\s+', ' ', txt)
                txt = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', txt)
                if txt:
                    parts.append(txt)
            elif el.name == 'table':
                rows = el.find_all('tr')
                md_rows = []
                col_count = None
                for i, tr in enumerate(rows):
                    cells = tr.find_all(['td', 'th'])
                    if col_count is None:
                        col_count = len(cells)
                    row_text = '| ' + ' | '.join(
                        re.sub(r'\s+', ' ', re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', c.get_text(' ', strip=True)))
                        for c in cells
                    ) + ' |'
                    md_rows.append(row_text)
                    if i == 0 and col_count and col_count > 1:
                        sep = '| ' + ' | '.join(['---'] * col_count) + ' |'
                        md_rows.append(sep)
                if md_rows:
                    parts.append('\n'.join(md_rows))
            elif el.name == 'img':
                src = el.get('src', '')
                alt = el.get('alt', '图片')
                if src and not src.startswith('data:'):
                    full_src = urljoin(url, src)
                    parts.append(f'![{alt}]({full_src})')

        content = '\n\n'.join(parts)

        for a_tag in content_div.find_all('a', href=True):
            href = a_tag['href']
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', href, re.I):
                full_url = urljoin(url, href)
                name = a_tag.get_text(strip=True) or os.path.basename(href)
                attachments.append({'name': name, 'url': full_url})

    if attachments:
        if content:
            content += '\n\n'
        content += '\n'.join(f'[{att["name"]}]({att["url"]})' for att in attachments)

    return content, attachments


def main():
    all_items = []
    print(f"=== {SITE_NAME} ===")

    for page in range(1, PAGES + 1):
        print(f"  [Page {page}/{PAGES}] fetching...", end=' ')
        items = fetch_list_page(page)
        print(f"{len(items)} items")
        all_items.extend(items)

    print(f"\n=== 共 {len(all_items)} 条列表项 ===")

    seen = set()
    unique = []
    for item in all_items:
        if item['url'] not in seen:
            seen.add(item['url'])
            unique.append(item)
    print(f"去重后: {len(unique)} 条")

    db_items = []
    for i, item in enumerate(unique):
        print(f"  [{i+1}/{len(unique)}] {item['title'][:40]}...")
        content, attachments = fetch_detail(item['url'])
        summary = re.sub(r'\s+', '', content)[:200] if content else ''
        att_json = json.dumps(attachments, ensure_ascii=False) if attachments else ''

        db_items.append({
            'site_name': SITE_NAME,
            'source_url': item['url'],
            'url': item['url'],
            'title': item['title'],
            'pub_date': item.get('date', ''),
            'summary': summary,
            'content': content,
            'attachments': att_json,
        })

    try:
        from crawler_lib import push_to_searchdb
        push_to_searchdb(db_items, SITE_NAME)
        print(f"\n✅ 入库完成: {len(db_items)} 条")
    except Exception as e:
        print(f"\n❌ 入库异常: {e}")
        import traceback
        traceback.print_exc()
        with open('/root/gxsthj_npzxm.json', 'w', encoding='utf-8') as f:
            json.dump(db_items, f, ensure_ascii=False, indent=2)
        print("  数据已保存到 /root/gxsthj_npzxm.json")


if __name__ == '__main__':
    main()