#!/usr/bin/env python3
"""
茂名高新技术产业开发区 - 环境保护信息公开 爬虫
http://www.mmht.gov.cn/zdlyxxgkzl/hjbhxxgk/index.html
"""
import requests
from bs4 import BeautifulSoup
import re, os, sys, time
from urllib.parse import urljoin

BASE_URL = "http://www.mmht.gov.cn"
LIST_BASE = "http://www.mmht.gov.cn/zdlyxxgkzl/hjbhxxgk/index"
SITE_NAME = "茂名高新区环保信息"
MAX_PAGES = 5
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url, encoding=None):
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        if encoding:
            r.encoding = encoding
        else:
            # Auto-detect encoding
            import chardet
            r.encoding = chardet.detect(r.content)['encoding'] or 'utf-8'
        return r.text
    except Exception as e:
        print(f"  [ERROR] fetch failed: {url} - {e}")
        return None

def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    import urllib.parse
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)


def extract_list_items(html):
    """Extract article URLs from list page."""
    soup = BeautifulSoup(html, 'html.parser')
    items = []
    for a in soup.find_all('a', href=True):
        href = a['href']
        if '/content/post_' in href and href.endswith('.html'):
            full_url = urljoin(BASE_URL, href)
            list_title = a.get_text(strip=True)
            if list_title:
                items.append((full_url, list_title))
    return items

def extract_detail(html, page_url):
    """Extract title, content, attachments from detail page."""
    soup = BeautifulSoup(html, 'html.parser')
    
    # 1. Title from <title>
    title_tag = soup.find('title')
    title = title_tag.get_text(strip=True) if title_tag else ""
    # Clean suffix: remove " - 茂名高新技术产业开发区网站"
    title = re.sub(r'\s*[-–—]\s*茂名高新技术产业开发区网站\s*$', '', title)
    
    # 2. Date from desc div
    date = ""
    desc = soup.find('div', class_='desc')
    if desc:
        date_match = re.search(r'(\d{4})[-年](\d{1,2})[-月](\d{1,2})', desc.get_text())
        if date_match:
            date = f"{date_match.group(1)}-{date_match.group(2).zfill(2)}-{date_match.group(3).zfill(2)}"
    
    # 3. Content + Attachments
    content_parts = []
    attachments = []
    
    # The content is inside article-content > div.content
    art = soup.find('div', class_='article-content')
    if art:
        content_div = art.find('div', class_='content')
        if not content_div:
            content_div = art
        
        # Find attachments
        for a in content_div.find_all('a', href=True):
            href = a['href']
            if any(href.endswith(ext) for ext in ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.rar', '.zip']):
                full_url = urljoin(page_url, href)
                attach_name = a.get_text(strip=True) or href.split('/')[-1]
                attachments.append({'name': attach_name, 'url': full_url})
        # Extract paragraphs, tables, images
        # Title may also appear as first <p> in zwcon - skip exact matches only
        for el in content_div.find_all(['p', 'table', 'img']):
            if el.name == 'p':
                # Skip <p> inside <table> — handled by table rendering below
                if el.find_parent('table'):
                    continue
                txt = el.get_text(strip=True)
                if txt:
                    # Only skip if <p> text exactly matches the title
                    if title and txt == title:
                        continue
                    content_parts.append(txt)
            elif el.name == 'table':
                tbl_html = html_table_to_html(el, page_url)
                if tbl_html:
                    content_parts.append(tbl_html)
            elif el.name == 'img':
                src = el.get('src', '')
                alt = el.get('alt', '')
                if src:
                    content_parts.append(f'![{alt}]({urljoin(page_url, src)})')
    
    content = '\n\n'.join(content_parts)
    return title, content, attachments, date

def main():
    items_all = []
    
    # Step 1: Collect items from pages 1-5
    # Page 1: index.html, Page N: index_N.html
    for page in range(1, MAX_PAGES + 1):
        if page == 1:
            url = f"{LIST_BASE}.html"
        else:
            url = f"{LIST_BASE}_{page}.html"
        
        print(f"Fetching list page {page}: {url}")
        html = fetch(url)
        if not html:
            print(f"  [SKIP] Page {page} failed")
            continue
        
        items = extract_list_items(html)
        print(f"  Found {len(items)} items")
        items_all.extend(items)
        time.sleep(0.3)
    
    print(f"\nTotal items to process: {len(items_all)}")
    
    # Step 2: Fetch each detail page
    results = []
    for i, (page_url, list_title) in enumerate(items_all):
        print(f"[{i+1}/{len(items_all)}] {list_title[:50]}...")
        
        html = fetch(page_url)
        if not html:
            results.append({
                'title': list_title, 'content': '', 'page_url': page_url,
                'attachments': [], 'date': '',
            })
            continue
        
        title, content, attachments, date = extract_detail(html, page_url)
        if not title:
            title = list_title
        
        results.append({
            'title': title, 'content': content, 'page_url': page_url,
            'attachments': attachments, 'date': date,
        })
        
        print(f"  -> {title[:50]} | date={date} | attach={len(attachments)} | content_len={len(content)}")
        time.sleep(0.2)
    
    # Step 3: Push to search DB
    SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
    
    try:
        import sqlite3
        conn = sqlite3.connect(SEARCH_DB, timeout=60)
        c = conn.cursor()
        
        inserted = 0
        for r in results:
            summary = r['content'][:200] if r['content'] else ''
            attachments_json = str(r['attachments']) if r['attachments'] else ''
            has_table = 1 if '<table' in r['content'] else 0
            c.execute(
                """INSERT OR REPLACE INTO gov_raw (title, content, page_url, source_url, site_name, publish_date, summary, attachments, has_table, category, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, '环评审批', 'crawl_mmht_hjbh.py')""",
                (r['title'], r['content'], r['page_url'], r['page_url'], SITE_NAME,
                 r['date'], summary, attachments_json, has_table)
            )
            inserted += 1
        
        conn.commit()
        conn.close()
        print(f"\n✅ Pushed {inserted} records to search DB")
        
    except Exception as e:
        print(f"\n[ERROR] DB push failed: {e}")
    
    print(f"\n{'='*50}")
    print(f"站点: {SITE_NAME}")
    print(f"栏目: 环境保护信息公开")
    print(f"采集页数: {MAX_PAGES}")
    print(f"采集条数: {len(results)}")
    print(f"{'='*50}")

if __name__ == '__main__':
    main()