#!/usr/bin/env python3
"""
crawl_eiacloud_local.py — 环评云(www.eiacloud.com) 本地Mac跑
使用 POST 表单提交分页（非API），避免服务器IP被封
"""
import os, re, sys, time, json, sqlite3
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
import requests
import warnings
warnings.filterwarnings('ignore')

BASE_URL = "https://www.eiacloud.com/gs/list/1"
DETAIL_URL = "https://www.eiacloud.com/gs/detail/1"
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Content-Type": "application/x-www-form-urlencoded",
    "Referer": "https://www.eiacloud.com/gs/list/1",
    "Origin": "https://www.eiacloud.com",
}
SLEEP = 2.0  # seconds between pages to avoid rate limiting

SERVER_SSH = "root@1.94.217.116"
SERVER_SEARCH_DB = "/mnt/data/search.db"

SITE_NAME = "环评云"
CATEGORY = "环评公示"

CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

is_incremental = len(sys.argv) >= 2
cutoff_days = int(sys.argv[1]) if is_incremental else 9999
if is_incremental:
    cutoff = (datetime.now() - timedelta(days=cutoff_days)).strftime("%Y-%m-%d")
else:
    cutoff = CUTOFF

# Optional: max pages limit for testing
MAX_PAGES = int(sys.argv[2]) if len(sys.argv) >= 3 else None

stats = {"new": 0, "skip": 0, "err": 0}

def fetch_page(page_num):
    """Fetch a single page via POST form submission."""
    data = {
        "pageModel.numberNo": page_num,
        "pageModel.totalPage": 4812,
        "pageModel.numberSize": 15,
        "sort": 0,
        "filterType": 0,
        "select": 0,
        "gkClassifyId": 0,
        "gkPlateId": 1,
    }
    try:
        r = requests.post(BASE_URL, headers=HEADERS, data=data, timeout=30, verify=False)
        r.encoding = 'utf-8'
        return r.text
    except Exception as e:
        print(f"  [ERR] page {page_num}: {e}")
        return ""

def parse_list(html):
    """Extract (url, id, title, date) from list page HTML."""
    items = []
    # Find all detail links - only article links (not "新窗口打开")
    for m in re.finditer(
        r'<a\s+href="/gs/detail/1\?id=([^"&]+)"[^>]*>((?![^<]*新窗口打开)[^<]+)</a>',
        html, re.DOTALL
    ):
        item_id = m.group(1)
        title = m.group(2).strip()
        url = f"{DETAIL_URL}?id={item_id}"
        if title and len(title) > 5:
            items.append((url, item_id, title.strip(), ""))

    # Fallback: use title attribute
    if len(items) < 5:
        items = []
        for m in re.finditer(
            r'<a\s+href="/gs/detail/1\?id=([^"&]+)"[^>]*title="([^"]*)"[^>]*>.*?</a>',
            html, re.DOTALL
        ):
            item_id = m.group(1)
            title = m.group(2).strip()
            url = f"{DETAIL_URL}?id={item_id}"
            if title and len(title) > 5:
                items.append((url, item_id, title.strip(), ""))

    # Extract dates - span/td text that looks like a date
    # Dates are in the same row: format YYYY-MM-DD or YYYY/MM/DD
    date_pattern = re.compile(r'(\d{4}-\d{2}-\d{2})\s+\d{2}:\d{2}')
    dates = date_pattern.findall(html)
    
    if len(dates) > 1:
        # Skip first date (the pinned post's date might be different)
        for i, item in enumerate(items):
            if i < len(dates):
                items[i] = (item[0], item[1], item[2], dates[i])
    
    return items

def fetch_detail(url):
    """Fetch detail page and extract content."""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = 'utf-8'
        html = r.text
        
        # Extract content from div.detail_text
        m = re.search(r'<div class="detail_text">(.*?)</div>\s*</div>', html, re.DOTALL)
        if m:
            txt = m.group(1)
        else:
            m = re.search(r'<div class="detail_text">(.*?)</div>', html, re.DOTALL)
            if m:
                txt = m.group(1)
            else:
                return ""
        
        # Clean but preserve table HTML
        txt = re.sub(r'<script[^>]*>.*?</script>', '', txt, flags=re.DOTALL)
        txt = re.sub(r'<style[^>]*>.*?</style>', '', txt, flags=re.DOTALL)
        txt = re.sub(r'<br\s*/?>', '\n', txt)
        txt = re.sub(r'</p>', '\n\n', txt)
        
        table_placeholders = []
        def save_table(m):
            ph = f'__TABLE_{len(table_placeholders)}__'
            table_placeholders.append((ph, m.group(0)))
            return ph
        txt = re.sub(r'<table[^>]*>.*?</table>', save_table, txt, flags=re.DOTALL)
        txt = re.sub(r'<[^>]+>', '', txt)
        txt = re.sub(r'\n{3,}', '\n\n', txt)
        for ph, ht in table_placeholders:
            txt = txt.replace(ph, ht)
        return txt.strip()
    except Exception as e:
        print(f"    [ERR] detail: {e}")
        return ""

def safe_summary(text, max_len=500):
    if len(text) <= max_len:
        return text
    truncated = text[:max_len]
    if '<' in truncated:
        last_open = truncated.rfind('<')
        last_close = truncated.rfind('>')
        if last_open > last_close:
            truncated = truncated[:last_open]
    return truncated

def main():
    # Local temp DB
    local_db = os.path.join(os.path.dirname(os.path.abspath(__file__)), "eiacloud_temp.db")
    conn = sqlite3.connect(local_db, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("""CREATE TABLE IF NOT EXISTS eiacloud_data (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT,
        page_url TEXT UNIQUE,
        content TEXT,
        publish_date TEXT,
        summary TEXT,
        crawled_at TEXT DEFAULT (datetime('now','localtime'))
    )""")
    conn.commit()
    c = conn.cursor()

    # Step 1: Get total pages from page 1
    print("Fetching page 1 to get total pages...")
    html = fetch_page(1)
    if not html:
        print("[FATAL] Cannot fetch page 1")
        return
    
    m = re.search(r'共(\d+)页', html)
    total_pages = int(m.group(1)) if m else 4812
    print(f"Total pages: {total_pages}")
    if MAX_PAGES and MAX_PAGES < total_pages:
        total_pages = MAX_PAGES
        print(f"Limited to {total_pages} pages for this run")

    # Step 2: Iterate all pages
    for page in range(1, total_pages + 1):
        if page == 1:
            pass  # already fetched
        else:
            html = fetch_page(page)
            if not html:
                stats["err"] += 1
                continue

        items = parse_list(html)
        if not items:
            print(f"  Page {page}/{total_pages}: 0 items (skip)")
            continue

        page_new = 0
        for url, item_id, title, date_str in items:
            if not title or len(title) < 5:
                continue
            if not date_str:
                date_str = ""

            # Check if exists
            c.execute("SELECT id FROM eiacloud_data WHERE page_url = ?", (url,))
            if c.fetchone():
                stats["skip"] += 1
                continue

            # Fetch detail
            content = fetch_detail(url)
            if not content or len(content) < 20:
                stats["skip"] += 1
                continue

            summary = safe_summary(content)
            c.execute("""INSERT OR IGNORE INTO eiacloud_data
                         (title, page_url, content, publish_date, summary)
                         VALUES (?, ?, ?, ?, ?)""",
                      (title, url, content, date_str[:10], summary))
            if conn.total_changes > 0:
                page_new += 1
                stats["new"] += 1
            else:
                stats["skip"] += 1

        conn.commit()
        print(f"  Page {page}/{total_pages}: +{page_new} (new={stats['new']}, skip={stats['skip']})")

        # Rate limiting
        if page < total_pages:
            time.sleep(SLEEP)

    conn.close()
    
    total = stats["new"] + stats["skip"]
    print(f"\n=== 完成 ===")
    print(f"新增: {stats['new']}, 跳过: {stats['skip']}, 错误: {stats['err']}")
    print(f"数据文件: {local_db} ({os.path.getsize(local_db)//1024//1024}MB)")

if __name__ == "__main__":
    main()