#!/usr/bin/env python3
"""
茂名论坛-《茂名环保》专栏 (bbs.gdmm.com/forum-912)
Discuz! forum, GBK charset
"""
import sys
import os
import re
import time
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = os.environ.get('DB_PATH', '/root/search.db')
BASE_URL = 'https://bbs.gdmm.com/'
FORUM_URL = 'https://bbs.gdmm.com/forum-912-{page}.html'
SITE_NAME = '茂名论坛-《茂名环保》专栏'
GROUP_NAME = '论坛'
MAX_PAGES = 5      # 前5页

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
}

session = requests.Session()
session.headers.update(HEADERS)


def fetch_html(url, encoding='gbk'):
    """Fetch and decode HTML."""
    try:
        r = session.get(url, timeout=30)
        r.encoding = encoding
        return r.text
    except Exception as e:
        print(f'  FETCH ERROR: {e}')
        return ''


def extract_threads_from_list(html):
    """Extract (tid, title) from forum list page."""
    threads = []
    # Find normalthread_XXXXX / stickthread_XXXXX patterns
    pattern = r'(?:normalthread|stickthread)_(\d+).*?<a[^>]*href="(?:https://bbs\.gdmm\.com/)?thread-\d+-\d+-\d+\.html"[^>]*class="s xst"[^>]*>(.*?)</a>'
    for m in re.finditer(pattern, html, re.DOTALL):
        tid = m.group(1)
        title_html = m.group(2)
        title = re.sub(r'<[^>]+>', '', title_html).strip()
        if title:
            threads.append((tid, title))
    return threads


def extract_detail(html, url):
    """Extract subject, content, date, attachments from thread detail page."""
    result = {
        'subject': '',
        'content': '',
        'date': '',
        'author': '',
        'attachments': [],
    }
    
    # Subject from <span id="thread_subject">
    m = re.search(r'<span[^>]*id="thread_subject"[^>]*>(.*?)</span>', html)
    if m:
        subject = re.sub(r'<[^>]+>', '', m.group(1)).strip()
        result['subject'] = subject
    
    # Post content from <td id="postmessage_XXXXX">
    m = re.search(r'<td[^>]*id="postmessage_(\d+)"[^>]*>(.*?)</td>', html, re.DOTALL)
    if m:
        post_html = m.group(2)
        soup = BeautifulSoup(post_html, 'html.parser')
        
        content_parts = []
        
        # Process child elements
        for child in soup.children:
            if child.name == 'div':
                div_text = child.get_text(separator=' ', strip=True)
                if div_text:
                    content_parts.append(div_text)
            elif child.name == 'table':
                # Extract table as text
                rows = child.find_all('tr')
                table_lines = []
                for row in rows:
                    cells = row.find_all(['td', 'th'])
                    cell_texts = [c.get_text(strip=True) for c in cells]
                    table_lines.append(' | '.join(cell_texts))
                if table_lines:
                    content_parts.append('\n'.join(table_lines))
            elif child.name == 'br':
                if content_parts and not content_parts[-1].endswith('\n'):
                    content_parts[-1] += '\n'
            elif isinstance(child, str):
                text = child.strip()
                if text:
                    content_parts.append(text)
        
        full_text = '\n\n'.join(content_parts)
        full_text = re.sub(r'\n{3,}', '\n\n', full_text)
        result['content'] = full_text.strip()
        
        # Fallback: plain text
        if not result['content']:
            full_text = soup.get_text(separator=' ', strip=True)
            result['content'] = full_text
    
    # Author
    m = re.search(r'<a[^>]*class="xw1"[^>]*>(.*?)</a>', html)
    if m:
        result['author'] = m.group(1).strip()
    
    # Date - from 发表于 YYYY-M-D HH:MM
    m = re.search(r'发表于\s+(\d{4}[-/]\d{1,2}[-/]\d{1,2}\s+\d{1,2}:\d{2})', html)
    if m:
        date_str = m.group(1).replace('/', '-')
        parts = date_str.replace('-', ' ').replace(':', ' ').split()
        if len(parts) >= 4:
            y, mo, d, h, mi = parts[0], parts[1].zfill(2), parts[2].zfill(2), parts[3].zfill(2), parts[4].zfill(2)
            result['date'] = f'{y}-{mo}-{d} {h}:{mi}:00'
        else:
            result['date'] = date_str
    
    # Attachments - forum.php?mod=attachment links (relative or absolute)
    attch_links = re.findall(r'<a[^>]*href="([^"]*forum\.php\?mod=attachment[^"]*)"[^>]*>(.*?)</a>', html, re.DOTALL)
    for href, text in attch_links:
        clean_text = re.sub(r'<[^>]+>', '', text).strip()
        full_url = urljoin(BASE_URL, href)
        result['attachments'].append(f'[{clean_text}]({full_url})')
    
    # attachment.php links
    attch_links2 = re.findall(r'<a[^>]*href="(attachment\.php\?aid=[^"]*)"[^>]*>(.*?)</a>', html, re.DOTALL)
    for href, text in attch_links2:
        clean_text = re.sub(r'<[^>]+>', '', text).strip()
        full_url = urljoin(BASE_URL, href)
        result['attachments'].append(f'[{clean_text}]({full_url})')
    
    # baidu pan links in content
    pan_links = re.findall(r'(https?://pan\.baidu\.com[^\s<]*)', html)
    for link in pan_links[:5]:
        if link not in str(result['attachments']):
            result['attachments'].append(f'[百度网盘]({link})')
    
    return result


def save_to_db(items):
    """Save items to originals.db."""
    import sqlite3
    
    if not items:
        print('No items to save.')
        return
    
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute('PRAGMA busy_timeout=30000')
    cur = conn.cursor()
    
    cur.execute('''
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT,
            content TEXT,
            summary TEXT,
            site_name TEXT,
            group_name TEXT,
            page_url TEXT,
            publish_date TEXT,
            attachments TEXT,
            source_file TEXT,
            crawled_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    ''')
    
    new_count = 0
    for item in items:
        cur.execute('SELECT id FROM gov_raw WHERE page_url = ?', (item['page_url'],))
        if cur.fetchone():
            continue
        
        summary = item['content'][:200] if item['content'] else item['subject']
        
        cur.execute('''
            INSERT INTO gov_raw (title, content, summary, site_name, group_name, page_url, publish_date, attachments)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        ''', (
            item['subject'],
            item['content'],
            summary,
            item.get('site_name', SITE_NAME),
            item.get('group_name', GROUP_NAME),
            item['page_url'],
            item.get('date', ''),
            '\n'.join(item.get('attachments', [])) if item.get('attachments') else '',
        ))
        new_count += 1
    
    conn.commit()
    conn.close()
    print(f'Inserted {new_count} new records.')


def sync_fts():
    """Sync FTS indexes."""
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute('PRAGMA busy_timeout=30000')
    cur = conn.cursor()
    
    cur.execute('''
        INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary)
        SELECT r.id, r.title, r.site_name, r.summary
        FROM gov_raw r
        LEFT JOIN gov_search f ON f.rowid = r.id
        WHERE f.rowid IS NULL
    ''')
    search_count = cur.rowcount
    print(f'Synced {search_count} to gov_search.')
    

    
    conn.commit()
    conn.close()


def main():
    import argparse
    parser = argparse.ArgumentParser(description='Crawl 茂名论坛-茂名环保')
    parser.add_argument('--max-pages', type=int, default=MAX_PAGES, help='Max pages to crawl')
    parser.add_argument('--test', action='store_true', help='Test mode: 1 page only')
    parser.add_argument('--max-items', type=int, default=0, help='Max items to crawl (0=all)')
    parser.add_argument('--skip-db', action='store_true', help='Skip DB save (debug)')
    args = parser.parse_args()
    
    max_pages = 1 if args.test else args.max_pages
    max_items = args.max_items
    
    all_items = []
    total_fetched = 0
    
    for page in range(1, max_pages + 1):
        print(f'\n=== Page {page} ===')
        url = FORUM_URL.format(page=page)
        html = fetch_html(url)
        if not html:
            print(f'  Failed to fetch page {page}')
            continue
        
        threads = extract_threads_from_list(html)
        print(f'  Found {len(threads)} threads on page {page}')
        
        if not threads:
            print('  No threads found, skipping')
            continue
        
        for tid, list_title in threads:
            if max_items > 0 and total_fetched >= max_items:
                break
            
            detail_url = f'https://bbs.gdmm.com/thread-{tid}-1-1.html'
            print(f'  [{total_fetched+1}] TID={tid}: {list_title[:60]}')
            
            detail_html = fetch_html(detail_url)
            if not detail_html:
                print('    FAILED to fetch detail')
                continue
            
            detail = extract_detail(detail_html, detail_url)
            
            subject = detail['subject'] or list_title
            
            # PDF empty content fallback
            content = detail['content']
            if len(content.strip()) < 20:
                attachments_text = '\n'.join(detail.get('attachments', [])) if detail.get('attachments') else ''
                if attachments_text:
                    content = f'<p><a href="{detail_url}">{subject}</a></p>\n\n附件：\n{attachments_text}'
                else:
                    content = f'<p><a href="{detail_url}">{subject}</a></p>'
            
            item = {
                'subject': subject,
                'content': content,
                'page_url': detail_url,
                'date': detail.get('date', ''),
                'site_name': SITE_NAME,
                'group_name': GROUP_NAME,
                'attachments': detail.get('attachments', []),
            }
            all_items.append(item)
            total_fetched += 1
            time.sleep(0.5)
        
        if max_items > 0 and total_fetched >= max_items:
            break
    
    print(f'\n=== Total: {len(all_items)} items ===')
    
    if all_items and not args.skip_db:
        save_to_db(all_items)
        sync_fts()
    
    print('Done.')


if __name__ == '__main__':
    main()