#!/usr/bin/env python3
"""田阳区公示公告爬虫 (TRS CMS)"""
import sys, re, os, json
sys.path.insert(0, '/root')
sys.path.insert(0, '/root/gov_crawler')
from urllib.parse import urljoin
import urllib.request, ssl
from bs4 import BeautifulSoup

SITE_NAME = "田阳区公示公告"
BASE_URL = "http://www.gxty.gov.cn/zfxxgk/fdzdgknr_1_1_1/gsgg_1/gsgg_11739"
PAGES = 5

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)
def fetch_list_page(page_num):
    """获取列表页"""
    if page_num == 1:
        url = f"{BASE_URL}/index.shtml"
    else:
        url = f"{BASE_URL}/index_{page_num-1}.shtml"

    try:
        req = urllib.request.Request(url, headers={
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            "Accept-Language": "zh-CN,zh;q=0.9",
        })
        resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
        html = resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"  [ERROR] {e}")
        return []

    soup = BeautifulSoup(html, 'html.parser')
    items = []

    # Try <ul><li> pattern
    for li in soup.find_all('li'):
        a_tag = li.find('a')
        if not a_tag:
            continue
        href = a_tag.get('href', '')
        title = a_tag.get_text(strip=True)
        if not title or len(title) < 8:
            continue
        if not href or href.startswith('javascript') or href == '#':
            continue
        # Skip sidebar/footer links - only take ./tXXXXX.shtml
        if not re.search(r'/t\d+\.shtml$', href) and not re.search(r't\d+\.shtml$', href):
            continue

        full_url = urljoin(BASE_URL + '/', href)
        items.append({
            'url': full_url,
            'title': re.sub(r'\s+', ' ', title).strip(),
        })

    return items


def parse_detail(url):
    """解析详情页"""
    try:
        req = urllib.request.Request(url, headers={
            "User-Agent": "Mozilla/5.0",
            "Accept-Language": "zh-CN,zh;q=0.9",
        })
        resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
        html = resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        return "", "", []

    soup = BeautifulSoup(html, 'html.parser')

    # 日期
    pub_date = ""
    meta_date = soup.find('meta', attrs={"name": "PubDate"})
    if meta_date:
        pub_date = meta_date.get('content', '')[:10]

    # 正文
    content_div = soup.find('div', class_='article-con')
    if not content_div:
        content_div = soup.find('div', class_='article')

    content = ""
    attachments = []

    if content_div:
        parts = []
        for el in content_div.find_all(['p', 'table', 'img']):
            if el.name == 'p':
                if el.find_parent('table'):
                    continue
                txt = el.get_text(' ', strip=True)
                txt = re.sub(r'\s+', ' ', txt)
                txt = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', txt)
                if txt:
                    parts.append(txt)
            elif el.name == 'table':
                tbl_html = html_table_to_html(el, url)
                if tbl_html:
                    parts.append(tbl_html)
            elif el.name == 'img':
                src = el.get('src', '')
                alt = el.get('alt', '图片')
                if src and not src.startswith('data:'):
                    full_src = urljoin(url, src)
                    parts.append(f'![{alt}]({full_src})')

        content = '\n\n'.join(parts)

        # 附件
        for a_tag in content_div.find_all('a', href=True):
            href = a_tag['href']
            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', href, re.I):
                full_url = urljoin(url, href)
                name = a_tag.get_text(strip=True) or os.path.basename(href)
                attachments.append({'name': name, 'url': full_url})

    # 内嵌附件
    if attachments:
        if content:
            content += '\n\n'
        content += '\n'.join(f'[{att["name"]}]({att["url"]})' for att in attachments)

    return content, pub_date, attachments


def main():
    all_items = []
    print(f"=== {SITE_NAME} ===")

    for page in range(1, PAGES + 1):
        print(f"  [Page {page}/{PAGES}] fetching...", end=' ')
        items = fetch_list_page(page)
        print(f"{len(items)} items")
        all_items.extend(items)

    print(f"\n=== 共 {len(all_items)} 条列表项 ===")

    # 去重
    seen = set()
    unique = []
    for item in all_items:
        if item['url'] not in seen:
            seen.add(item['url'])
            unique.append(item)
    print(f"去重后: {len(unique)} 条")

    # 爬详情
    db_items = []
    for i, item in enumerate(unique):
        print(f"  [{i+1}/{len(unique)}] {item['title'][:40]}...")
        content, pub_date, attachments = parse_detail(item['url'])
        item['date'] = pub_date
        summary = re.sub(r'\s+', '', content)[:200] if content else ''
        att_json = json.dumps(attachments, ensure_ascii=False) if attachments else ''

        db_items.append({
            'site_name': SITE_NAME,
            'source_url': item['url'],
            'url': item['url'],
            'title': item['title'],
            'pub_date': pub_date,
            'summary': summary,
            'content': content,
            'attachments': att_json,
        })

    # 入库
    try:
        from crawler_lib import push_to_searchdb
        push_to_searchdb(db_items, SITE_NAME)
        print(f"\n✅ 入库完成: {len(db_items)} 条")
    except Exception as e:
        print(f"\n❌ 入库异常: {e}")
        import traceback
        traceback.print_exc()
        with open('/root/gxty_tzgg.json', 'w', encoding='utf-8') as f:
            json.dump(db_items, f, ensure_ascii=False, indent=2)
        print("  数据已保存到 /root/gxty_tzgg.json")


if __name__ == '__main__':
    main()