#!/usr/bin/env python3
"""河北生态信息网-公示公告爬虫

Laravel CMS
列表: /article/list?id=3&page=N (~41页, ~652条, 2022~2026)
详情: /article/detail?id=N
内容: div.public_content > p
日期: 更新时间：YYYY-MM-DD
"""

import requests
import re
import sys
import json
import time
import os
import argparse
from bs4 import BeautifulSoup
from urllib.parse import urljoin

sys.path.insert(0, '/root/gov_crawler')
from crawler_lib import push_to_searchdb
import sqlite3

DB_PATH = '/root/search.db'


def get_db_connection():
    try:
        conn = sqlite3.connect(DB_PATH, timeout=30)
        conn.execute("PRAGMA busy_timeout=30000")
        conn.execute("PRAGMA journal_mode=WAL")
        return conn
    except Exception as e:
        print(f'  WARN DB connect: {e}')
        return None


BASE_URL = 'https://www.hbstxxw.com'
LIST_URL = '/article/list?id=3'

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}

SITE_NAME = '河北生态信息网-公示公告'

sess = requests.Session()
sess.headers.update(HEADERS)


def fetch_page(url):
    resp = sess.get(url, timeout=15)
    resp.encoding = 'utf-8'
    return resp.text


def parse_list(html):
    """解析列表页"""
    soup = BeautifulSoup(html, 'html.parser')
    items = []

    # 所有 li 中的链接
    for a in soup.select('ul.search_list li a[href*="/article/detail?id="]'):
        href = a.get('href', '')
        if not href:
            continue

        # 标题在 <span> 内
        span = a.find('span')
        title = span.get_text(strip=True) if span else a.get_text(strip=True)
        if not title:
            continue

        detail_url = href if href.startswith('http') else urljoin(BASE_URL, href)

        # 日期: li > i.date
        li = a.find_parent('li')
        date_str = ''
        if li:
            i_tag = li.find('i', class_='date')
            if i_tag:
                date_str = i_tag.get_text(strip=True)

        items.append({
            'title': title,
            'url': detail_url,
            'pub_date': date_str,
        })

    return items


def fetch_detail(item):
    """抓取详情页"""
    url = item['url']

    try:
        html = fetch_page(url)
    except Exception as e:
        print(f'  WARN skip: {url} - {e}')
        return item['title'], item.get('pub_date', ''), '', []

    soup = BeautifulSoup(html, 'html.parser')

    # 标题
    h1 = soup.find('h1')
    title = h1.get_text(strip=True) if h1 else item['title']

    # 日期
    pub_date = item.get('pub_date', '')
    subtitle = soup.find('div', class_='public_Subtitle')
    if subtitle:
        span = subtitle.find('span')
        if span:
            txt = span.get_text(strip=True)
            m = re.search(r'(\d{4}-\d{2}-\d{2})', txt)
            if m:
                pub_date = m.group(1)

    # 正文
    content_div = soup.find('div', class_='public_content')
    content = ''
    attachments = []

    if content_div:
        parts = []
        for elem in content_div.find_all(['p', 'table', 'img'], recursive=True):
            if elem.name == 'p':
                if elem.find('p'):
                    continue
                txt = ''
                for child in elem.children:
                    if isinstance(child, str):
                        t = child.strip()
                        if t:
                            txt += t + ' '
                    elif child.name == 'br':
                        txt += '\n'
                    elif child.name == 'img':
                        src = child.get('src', '')
                        alt = child.get('alt', '')
                        if src:
                            full_src = urljoin(url, src)
                            txt += f'![{alt}]({full_src}) '
                    elif child.name == 'a':
                        a_href = child.get('href', '')
                        a_text = child.get_text(strip=True)
                        if a_href and a_text:
                            full_href = urljoin(url, a_href)
                            txt += f'[{a_text}]({full_href}) '
                            if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', a_href, re.I):
                                attachments.append({'name': a_text, 'url': full_href})
                    elif child.name in ('span', 'strong', 'b', 'em'):
                        txt += child.get_text(' ', strip=True) + ' '
                txt = re.sub(r'[ \t]+', ' ', txt).strip()
                if txt:
                    parts.append(txt)
            elif elem.name == 'table':
                parts.append(str(elem))
            elif elem.name == 'img':
                src = elem.get('src', '')
                alt = elem.get('alt', '')
                if src:
                    full_src = urljoin(url, src)
                    parts.append(f'![{alt}]({full_src})')
        content = '\n\n'.join(parts)

    return title, pub_date, content, attachments


def get_last_page_num(html):
    """从分页获取总页数"""
    soup = BeautifulSoup(html, 'html.parser')
    page_links = soup.select('ul.page li.pageItem a[href]')
    max_page = 0
    for a in page_links:
        href = a.get('href', '')
        m = re.search(r'page=(\d+)', href)
        if m:
            p = int(m.group(1))
            if p > max_page:
                max_page = p
    return max_page


def main():
    parser = argparse.ArgumentParser(description='河北生态信息网-公示公告')
    parser.add_argument('--pages', type=int, default=0, help='页数，0=自动检测全量')
    args = parser.parse_args()

    # 第一页 -> 获取总页数
    print('Fetching page 1...')
    html = fetch_page(BASE_URL + LIST_URL)
    items = parse_list(html)
    print(f'  Found {len(items)} items')

    max_pages = args.pages
    if max_pages == 0:
        max_pages = get_last_page_num(html)
        print(f'  Auto detected {max_pages} pages')

    for page in range(2, max_pages + 1):
        url = f'{BASE_URL}{LIST_URL}&page={page}'
        print(f'Page {page}...')
        try:
            html = fetch_page(url)
            page_items = parse_list(html)
            if not page_items:
                print(f'  empty, stop')
                break
            print(f'  found {len(page_items)} items')
            items.extend(page_items)
        except Exception as e:
            print(f'  WARN: {e}')
            break
        time.sleep(0.3)

    print(f'\nTotal list: {len(items)} items')

    # 详情
    records = []
    for i, item in enumerate(items):
        print(f'  [{i+1}/{len(items)}] {item["title"][:40]}...')
        title, pub_date, content, attachments = fetch_detail(item)

        if not title:
            title = item['title']
        title = re.sub(r'\s+', ' ', title).strip()

        summary = content[:500] if content else ''

        record = {
            'url': item['url'],
            'source_url': item['url'],
            'title': title,
            'content': content,
            'summary': summary,
            'pub_date': pub_date,
            'site_name': SITE_NAME,
            'attachments': json.dumps(attachments, ensure_ascii=False) if attachments else '',
        }
        records.append(record)
        time.sleep(0.2)

    # 入库
    print(f'\nInserting {len(records)} records...')
    push_to_searchdb(records, SITE_NAME)

    # FTS验证
    conn = get_db_connection()
    if conn:
        try:
            cursor = conn.execute(
                "SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,)
            )
            raw_count = cursor.fetchone()[0]
            cursor = conn.execute(
                "SELECT COUNT(*) FROM gov_search WHERE site_name=?", (SITE_NAME,)
            )
            fts_count = cursor.fetchone()[0]
            print(f'DB: {raw_count} rows (FTS: {fts_count} rows)')
        except Exception as e:
            print(f'  WARN DB query: {e}')
        finally:
            conn.close()

    print('DONE')


if __name__ == '__main__':
    main()