#!/usr/bin/env python3
"""
晋江市-建设项目环境影响评价 (Playwright版)
使用Playwright通过eval方式翻页，调用Avalon的toPage直接跳转
"""
import re, sys, json, time, requests, sqlite3
from bs4 import BeautifulSoup
from urllib.parse import urljoin

BASE_URL = "https://www.jinjiang.gov.cn/xxgk/zdxxgk/hjbh/jsxmyxpj/"
SITE_NAME = "jinjiang"
HEADERS = {"User-Agent": "Mozilla/5.0"}
SEARCH_DB = "/root/temp_search.db"

def get_detail_url(url):
    return urljoin(BASE_URL, url)

def extract_inline_data(html):
    items = []
    pattern = r'\{\s*"docreltime":\s*\'([^\']+)\'\s*,\s*"doctitle":\s*\'([^\']+)\'\s*,\s*"docpuburl":\s*\'([^\']+)\'\s*\}'
    matches = re.findall(pattern, html)
    seen = set()
    for docreltime, doctitle, docpuburl in matches:
        url = get_detail_url(docpuburl.strip())
        if url not in seen:
            seen.add(url)
            items.append({"date": docreltime.strip(), "title": doctitle.strip(), "url": url})
    return items

def extract_dom_data(page):
    """直接从DOM提取列表数据"""
    items = []
    try:
        links = page.query_selector_all('ul[ms-if="list.length>0"] li a')
        if links:
            print(f"  DOM找到 {len(links)} 个链接")
            return None  # 从页面JS提取
    except:
        pass
    return items

def main():
    print(f"===== 晋江市-建设项目环境影响评价 =====")
    print(f"目标: {BASE_URL}")
    
    from playwright.sync_api import sync_playwright
    
    with sync_playwright() as pw:
        browser = pw.chromium.launch(headless=True)
        context = browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            viewport={"width": 1920, "height": 1080}
        )
        page = context.new_page()
        
        # 拦截XHR/Fetch请求
        xhr_urls = set()
        page.on("request", lambda r: xhr_urls.add(r.url) if r.resource_type in ("xhr", "fetch") else None)
        
        page.goto(BASE_URL, wait_until="networkidle", timeout=60000)
        time.sleep(3)
        
        print(f"\n=== XHR请求: {len(xhr_urls)} ===")
        for url in xhr_urls:
            print(f"  {url[:150]}")
        
        # 尝试用evaluate直接操作Avalon
        print(f"\n=== 尝试Avalon翻页 ===")
        
        # 收集所有页面数据
        all_items = []
        total_pages = 107
        batch_size = 20  # 每次跳转20页
        
        # 先获取当前页(第1页)的inline数据
        page1_items = extract_inline_data(page.content())
        print(f"第1页(从HTML): {len(page1_items)} 条")
        all_items.extend(page1_items)
        
        # 直接跳转到第11页(索引10)
        for page_idx in range(10, total_pages):
            print(f"  跳转第{page_idx+1}页 (0-based: {page_idx})...", end=" ")
            sys.stdout.flush()
            
            try:
                # 通过JS直接设置avalon vm的pageIndex
                result = page.evaluate(f"""
                    (() => {{
                        // 找到list controller
                        var vms = avalon.vmodels;
                        var listVm = null;
                        for(var k in vms) {{
                            if(vms[k].pageIndex !== undefined) {{
                                listVm = vms[k];
                                break;
                            }}
                        }}
                        if(listVm) {{
                            listVm.toPage({page_idx + 1});
                            return "called toPage(" + ({page_idx + 1}) + ")";
                        }}
                        return "no list vm found, keys: " + Object.keys(vms).join(",");
                    }})()
                """)
                print(f"JS: {result[:80]}")
            except Exception as e:
                print(f"evaluate error: {e}")
                break
            
            time.sleep(3)
            page.wait_for_load_state("networkidle", timeout=15000)
            time.sleep(1)
            
            # 从当前页提取数据
            content = page.content()
            items = extract_inline_data(content)
            if not items:
                print(f"  -> 无数据")
                # 检查是否到了空页
                try:
                    no_data = page.query_selector('div.default_page')
                    if no_data:
                        print("  -> 无更多数据！")
                        break
                except:
                    pass
                continue
            
            print(f"  -> {len(items)} 条 [{items[-1]['date']} ~ {items[0]['date']}]")
            all_items.extend(items)
            
            # 每10页提交一次
            if len(all_items) % 150 == 0:
                print(f"  [累计] {len(all_items)} 条")
        
        browser.close()
    
    # 去重
    seen = set()
    unique = []
    for item in all_items:
        if item['url'] not in seen:
            seen.add(item['url'])
            unique.append(item)
    
    print(f"\n共获取: {len(unique)} 条 (去重后)")
    if unique:
        print(f"日期范围: {unique[-1]['date']} ~ {unique[0]['date']}")
    
    # 获取详情
    print(f"\n--- 获取详情页 ---")
    session = requests.Session()
    session.headers.update(HEADERS)
    
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()
    c.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT, content TEXT, source_url TEXT UNIQUE,
        site_name TEXT, publish_date TEXT,
        crawl_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
    )''')
    
    inserted = 0
    short_count = 0
    for idx, item in enumerate(unique):
        print(f"  [{idx+1}/{len(unique)}] {item['date']} {item['title'][:40]}...", end=" ")
        sys.stdout.flush()
        try:
            resp = session.get(item['url'], timeout=30)
            if resp.status_code == 200:
                soup = BeautifulSoup(resp.text, 'html.parser')
                content_div = soup.select_one('div.TRS_Editor')
                if not content_div:
                    content_div = soup.select_one('div.article_content')
                content = str(content_div) if content_div else ""
                
                meta_title = soup.find('meta', attrs={"name": "ArticleTitle"})
                title = meta_title['content'].strip() if meta_title and meta_title.get('content') else item['title']
                
                meta_date = soup.find('meta', attrs={"name": "PubDate"})
                date = meta_date['content'].strip() if meta_date and meta_date.get('content') else item['date']
                
                if not content or len(content.strip()) < 200:
                    print(f"SHORT({len(content.strip())})")
                    short_count += 1
                else:
                    print(f"OK({len(content.strip())})")
            else:
                content = ""
                print(f"HTTP {resp.status_code}")
                title = item['title']
                date = item['date']
        except Exception as e:
            content = ""
            print(f"ERROR: {e}")
            title = item['title']
            date = item['date']
        
        try:
            c.execute("INSERT OR IGNORE INTO gov_raw (title, content, source_url, site_name, publish_date) VALUES (?, ?, ?, ?, ?)",
                      (title, content, item['url'], SITE_NAME, date))
            if c.rowcount > 0:
                inserted += 1
        except Exception as e:
            print(f"    [DB ERROR] {e}")
    
    conn.commit()
    conn.close()
    print(f"\n{'='*50}")
    print(f"完成！总计: {inserted} 条入库, 有内容: {len(unique)-short_count} 条")
    print(f"{'='*50}")

if __name__ == "__main__":
    main()