#!/usr/bin/env python3
"""兴仁市人民政府-通知公告爬虫

TRS CMS (拓尔思)
列表: /xwzx/tzgg/index.html (327页, ~4896条, 2013~2026)
      第N页: index_{N-1}.html (createPageHTML(327,0,"index","html",4896))
详情: /xwzx/tzgg/YYYYMM/tYYYYMMDD_XXXXX.html
内容: font#Zoom > div.trs_editor_view > p
日期: meta PubDate
"""

import requests
import re
import sys
import json
import time
import os
import argparse
import html as html_lib
from bs4 import BeautifulSoup
from urllib.parse import urljoin

sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import push_to_searchdb
import sqlite3

DB_PATH = '/root/search.db'


def get_db_connection():
    try:
        conn = sqlite3.connect(DB_PATH, timeout=30)
        conn.execute("PRAGMA busy_timeout=30000")
        conn.execute("PRAGMA journal_mode=WAL")
        return conn
    except Exception as e:
        print(f'  WARN DB connect: {e}')
        return None


BASE_URL = 'https://www.gzxr.gov.cn'

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}

SITE_NAME = '兴仁市-通知公告'

sess = requests.Session()
sess.headers.update(HEADERS)


def fetch_page(url):
    resp = sess.get(url, timeout=15)
    resp.encoding = 'utf-8'
    return resp.text


def parse_list(html):
    """解析列表页"""
    soup = BeautifulSoup(html, 'html.parser')
    items = []

    ul = soup.select_one('div.right_list dl dd ul')
    if not ul:
        return items

    for li in ul.find_all('li', recursive=False):
        a = li.find('a', href=True)
        if not a:
            continue

        href = a.get('href', '')
        if 'tzgg' not in href:
            continue

        # 标题优先从 a[title] 取（完整标题），其次从文本（可能截断）
        title = a.get('title', '') or a.get_text(strip=True)
        if not title:
            continue

        detail_url = href if href.startswith('http') else urljoin(BASE_URL, href)

        # 日期
        i_tag = li.find('i')
        pub_date = i_tag.get_text(strip=True) if i_tag else ''

        items.append({
            'title': title,
            'url': detail_url,
            'pub_date': pub_date,
        })

    return items


def fetch_detail(item):
    """抓取详情页"""
    url = item['url']

    try:
        html = fetch_page(url)
    except Exception as e:
        print(f'  WARN skip: {url} - {e}')
        return item['title'], item.get('pub_date', ''), '', []

    soup = BeautifulSoup(html, 'html.parser')

    # 标题
    h2 = soup.find('h2')
    title = h2.get_text(strip=True) if h2 else item['title']

    # 日期 - meta PubDate
    meta = soup.find('meta', attrs={'name': 'PubDate'})
    pub_date = ''
    if meta and meta.get('content'):
        m = re.search(r'(\d{4}-\d{2}-\d{2})', meta['content'])
        if m:
            pub_date = m.group(1)

    if not pub_date:
        pub_date = item.get('pub_date', '')

    # 正文 — 2026-08-13 二修: 改为纯HTML输出 (段落<p>, 链接<a>, 图片<img>, 表格保留)
    # (原输出为纯文本\n\n+原始<table>混合, search_app 命中 '<table' 判定走HTML原样分支
    #  → 纯文本段落换行被浏览器折叠成一段墙, 用户可见"分段有问题")
    font_zoom = soup.find('font', id='Zoom')
    content = ''
    if font_zoom:
        parts = []
        for elem in font_zoom.find_all(['p', 'table', 'img'], recursive=True):
            if elem.name == 'p':
                if elem.find('p'):
                    continue
                if elem.find_parent('table'):  # 2026-08-13 修复: 跳过表内p, 避免与table整体双重捕获
                    continue
                frags = []
                for child in elem.children:
                    if isinstance(child, str):
                        t = child.strip()
                        if t:
                            frags.append(html_lib.escape(t))
                    elif child.name == 'br':
                        frags.append('<br/>')
                    elif child.name == 'img':
                        src = child.get('src', '')
                        alt = child.get('alt', '') or ''
                        if src:
                            frags.append('<img src="%s" alt="%s"/>' % (urljoin(url, src), html_lib.escape(alt)))
                    elif child.name == 'a':
                        a_href = child.get('href', '')
                        a_text = child.get_text(strip=True)
                        if a_href and a_text:
                            full_href = urljoin(url, a_href)
                            frags.append('<a href="%s" target="_blank">%s</a>' % (full_href, html_lib.escape(a_text)))
                    elif child.name in ('span', 'strong', 'b', 'em'):
                        frags.append(html_lib.escape(child.get_text(' ', strip=True)) + ' ')
                txt = ''.join(frags)
                txt = re.sub(r'[ \t]+', ' ', txt).strip()
                txt = re.sub(r' +<br/>', '<br/>', txt)
                if txt:
                    parts.append(txt)
            elif elem.name == 'table':
                parts.append(str(elem))
            elif elem.name == 'img':
                src = elem.get('src', '')
                alt = elem.get('alt', '') or ''
                if src:
                    parts.append('<p><img src="%s" alt="%s"/></p>' % (urljoin(url, src), html_lib.escape(alt)))
        # 2026-08-13 修复: 合并连续单字CJK段落 (Word转换封面逐字拆段: 遴/选/文/件 → 遴选文件)
        merged = []
        last_single = False
        for p in parts:
            _plain = re.sub(r'<[^>]+>', '', p)
            is_single = len(_plain) == 1 and re.match(r'[\u4e00-\u9fff]', _plain) is not None
            if is_single:
                if last_single:
                    merged[-1] += p
                else:
                    merged.append(p)
                last_single = True
            else:
                merged.append(p)
                last_single = False
        parts = merged
        # 纯文本段落包 <p> (表格/图片已自带块级标签不重复包裹)
        final = []
        for p in parts:
            if p.startswith('<table') or p.startswith('<p') or p.startswith('<img'):
                final.append(p)
            else:
                final.append('<p>%s</p>' % p)
        content = '\n\n'.join(final)

    # 附件
    attachments = []
    if font_zoom:
        for a in font_zoom.find_all('a', href=True):
            href = a.get('href', '')
            text = a.get_text(strip=True)
            if href and text and re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', href, re.I):
                if not href.startswith('http'):
                    href = urljoin(url, href)
                attachments.append({'name': text, 'url': href})
                content += '\n\n<p><a href="%s" target="_blank">%s</a></p>' % (href, html_lib.escape(text))

    return title, pub_date, content, attachments


def main():
    parser = argparse.ArgumentParser(description='兴仁市-通知公告')
    parser.add_argument('--pages', type=int, default=0, help='页数，0=全量327页')
    args = parser.parse_args()

    max_pages = args.pages if args.pages > 0 else 327
    all_items = []

    for page in range(1, max_pages + 1):
        if page == 1:
            url = f'{BASE_URL}/xwzx/tzgg/index.html'
        else:
            url = f'{BASE_URL}/xwzx/tzgg/index_{page-1}.html'

        print(f'Page {page}/{max_pages}...')
        try:
            html = fetch_page(url)
            items = parse_list(html)
            if not items:
                print(f'  empty, stop')
                break
            print(f'  found {len(items)} items')
            all_items.extend(items)
        except Exception as e:
            print(f'  WARN: {e}')
            break
        time.sleep(0.3)

    print(f'\nTotal list: {len(all_items)} items')

    # 详情
    records = []
    for i, item in enumerate(all_items):
        print(f'  [{i+1}/{len(all_items)}] {item["title"][:40]}...')
        title, pub_date, content, attachments = fetch_detail(item)

        if not title:
            title = item['title']
        title = re.sub(r'\s+', ' ', title).strip()
        summary = content[:500] if content else ''

        record = {
            'url': item['url'],
            'source_url': item['url'],
            'title': title,
            'content': content,
            'summary': summary,
            'pub_date': pub_date,
            'site_name': SITE_NAME,
            'attachments': json.dumps(attachments, ensure_ascii=False) if attachments else '',
        }
        records.append(record)
        time.sleep(0.2)

    # 入库
    print(f'\nInserting {len(records)} records...')
    push_to_searchdb(records, SITE_NAME)

    # FTS验证
    conn = get_db_connection()
    if conn:
        try:
            cursor = conn.execute(
                "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)
            )
            raw_count = cursor.fetchone()[0]
            cursor = conn.execute(
                "SELECT COUNT(*) FROM gov_search WHERE site_name=?", (SITE_NAME,)
            )
            fts_count = cursor.fetchone()[0]
            print(f'DB: {raw_count} rows (FTS: {fts_count} rows)')
        except Exception as e:
            print(f'  WARN DB query: {e}')
        finally:
            conn.close()

    print('DONE')


if __name__ == '__main__':
    main()