#!/usr/bin/env python3
"""
衢州市生态环境局 - 行政许可 爬虫
使用 Playwright 加载动态页面
"""
import requests, os, sys, time, json, re, asyncio
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from playwright.async_api import async_playwright

BASE_URL = "https://www.qz.gov.cn"
COL_URL = "https://www.qz.gov.cn/col/col1229709391/index.html"
SITE_NAME = "衢州市生态环境行政许可"
MAX_PAGES = 5
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=15)
        import chardet
        r.encoding = chardet.detect(r.content)['encoding'] or 'utf-8'
        return r.text
    except Exception as e:
        print(f"  [ERROR] fetch failed: {url} - {e}")
        return None

async def get_article_list(playwright_page, page_num):
    """Get article URLs from a specific page of the column."""
    if page_num == 1:
        url = COL_URL
    else:
        url = f"{COL_URL}?pageNo={page_num}"
    
    await playwright_page.goto(url, wait_until="domcontentloaded", timeout=30000)
    await asyncio.sleep(2)
    
    articles = await playwright_page.evaluate('''
        () => {
            const links = document.querySelectorAll('a[href*="/col/col1229709391/art/"]');
            const seen = new Set();
            const results = [];
            links.forEach(a => {
                const href = a.href.split('?')[0];
                if (!seen.has(href) && a.textContent.trim().length > 5) {
                    seen.add(href);
                    results.push({
                        href: href,
                        text: a.textContent.trim(),
                    });
                }
            });
            return results;
        }
    ''')
    return articles

def extract_detail(html, page_url):
    """Extract title, content, attachments from detail page."""
    soup = BeautifulSoup(html, 'html.parser')
    
    # 1. Title from <title>
    title_tag = soup.find('title')
    title = title_tag.get_text(strip=True) if title_tag else ""
    
    # 2. Date from sx_Textcontent
    date = ""
    date_match = re.search(r'发布日期[：:]\s*(\d{4})-(\d{2})-(\d{2})', html)
    if date_match:
        date = f"{date_match.group(1)}-{date_match.group(2)}-{date_match.group(3)}"
    if not date:
        date_match = re.search(r'成文日期[：:]\s*(\d{4})-(\d{2})-(\d{2})', html)
        if date_match:
            date = f"{date_match.group(1)}-{date_match.group(2)}-{date_match.group(3)}"
    
    # 3. Content
    content_parts = []
    attachments = []
    
    content_div = soup.find('div', class_='sx_Textcontent')
    if content_div:
        # Attachments
        for a in content_div.find_all('a', href=True):
            href = a['href']
            if any(kw in href.lower() for kw in ['download', '.pdf', '.doc', '.docx', '.xls', '.xlsx']):
                attachments.append({'name': a.get_text(strip=True) or href.split('/')[-1], 
                                   'url': urljoin(page_url, href)})
        
        # Content elements
        for el in content_div.find_all(['p', 'table', 'img']):
            if el.name == 'p':
                if el.find_parent('table'):
                    continue
                txt = el.get_text(strip=True)
                if txt:
                    if title and txt == title:
                        continue
                    content_parts.append(txt)
            elif el.name == 'table':
                rows = el.find_all('tr')
                md_rows = []
                for row in rows:
                    cells = row.find_all(['td', 'th'])
                    cell_texts = []
                    for c in cells:
                        link = c.find('a', href=True)
                        if link and any(kw in link['href'].lower() for kw in ['download', '.pdf', '.doc']):
                            cell_texts.append(f"[{link.get_text(strip=True)}]({urljoin(page_url, link['href'])})")
                        else:
                            cell_texts.append(c.get_text(strip=True))
                    md_rows.append('| ' + ' | '.join(cell_texts) + ' |')
                if md_rows:
                    n_cols = md_rows[0].count('|') - 1
                    md_rows.insert(1, '|' + ' --- |' * n_cols)
                    content_parts.append('\n'.join(md_rows))
            elif el.name == 'img':
                src = el.get('src', '')
                if src:
                    content_parts.append(f'![]({urljoin(page_url, src)})')
    
    # Fallback
    if not content_parts and content_div:
        raw = content_div.get_text(strip=True)
        if raw:
            content_parts.append(raw)
    
    content = '\n\n'.join(content_parts)
    return title, content, attachments, date

async def main():
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True, args=['--no-sandbox'])
        page = await browser.new_page()
        
        all_items = []
        # Page 1 loads the list, check if there's pagination to click through
        try:
            items = await get_article_list(page, 1)
            print(f"Page 1: {len(items)} items")
            all_items.extend(items)
            
            # Try clicking next page up to MAX_PAGES-1 more times
            for pg in range(2, MAX_PAGES + 1):
                clicked = await page.evaluate('''
                    () => {
                        const nextBtn = document.querySelector('.layui-laypage-next');
                        if (nextBtn && !nextBtn.classList.contains('layui-disabled')) {
                            nextBtn.click();
                            return true;
                        }
                        return false;
                    }
                ''')
                if not clicked:
                    print(f"  No more pages after page {pg-1}")
                    break
                
                await asyncio.sleep(2)
                items = await page.evaluate('''
                    () => {
                        const links = document.querySelectorAll('a[href*="/col/col1229709391/art/"]');
                        const seen = new Set();
                        return Array.from(links).map(a => {
                            const href = a.href.split('?')[0];
                            if (!seen.has(href) && a.textContent.trim().length > 5) {
                                seen.add(href);
                                return { href: href, text: a.textContent.trim() };
                            }
                            return null;
                        }).filter(x => x !== null);
                    }
                ''')
                # Only add new items (not already collected)
                existing_hrefs = {x['href'] for x in all_items}
                new_items = [x for x in items if x['href'] not in existing_hrefs]
                print(f"  Page {pg}: {len(new_items)} new items")
                all_items.extend(new_items)
                await asyncio.sleep(0.5)
                
        except Exception as e:
            print(f"  [WARN] Playwright error: {e}")
        
        await browser.close()
        
        # Deduplicate
        seen_hrefs = set()
        unique_items = []
        for item in all_items:
            if item['href'] not in seen_hrefs:
                seen_hrefs.add(item['href'])
                unique_items.append(item)
        
        print(f"\nTotal unique items: {len(unique_items)}")
        
        # Fetch each detail page
        results = []
        for i, item in enumerate(unique_items):
            print(f"[{i+1}/{len(unique_items)}] {item['text'][:50]}...")
            html = fetch(item['href'])
            if not html:
                results.append({
                    'title': item['text'], 'content': '', 'page_url': item['href'],
                    'attachments': [], 'date': '',
                })
                continue
            
            title, content, attachments, date = extract_detail(html, item['href'])
            if not title:
                title = item['text']
            
            results.append({
                'title': title, 'content': content, 'page_url': item['href'],
                'attachments': attachments, 'date': date,
            })
            print(f"  -> {title[:50]} | date={date} | attach={len(attachments)} | content_len={len(content)}")
            time.sleep(0.3)
        
        # Push to DB
        SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
        try:
            import sqlite3
            conn = sqlite3.connect(SEARCH_DB)
            c = conn.cursor()
            
            # Clean old data
            c.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE site_name=?)", (SITE_NAME,))
            c.execute("DELETE FROM gov_raw WHERE site_name=?", (SITE_NAME,))
            
            inserted = 0
            for r in results:
                summary = r['content'][:200] if r['content'] else ''
                attach_json = str(r['attachments']) if r['attachments'] else ''
                has_table = 1 if '| --- |' in r['content'] else 0
                c.execute(
                    """INSERT INTO gov_raw 
                       (title, content, page_url, source_url, site_name, publish_date, summary, attachments, has_table, category)
                       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, '行政许可')""",
                    (r['title'], r['content'], r['page_url'], r['page_url'], SITE_NAME,
                     r['date'], summary, attach_json, has_table)
                )
                inserted += 1
            
            conn.commit()
            conn.close()
            print(f"\n✅ Pushed {inserted} records to search DB")
        except Exception as e:
            print(f"\n[ERROR] DB push failed: {e}")
        
        print(f"\n{'='*50}")
        print(f"站点: {SITE_NAME}")
        print(f"采集页数: {MAX_PAGES}")
        print(f"采集条数: {len(results)}")
        print(f"{'='*50}")

if __name__ == '__main__':
    asyncio.run(main())