#!/usr/bin/env python3
"""crawl_wlmq.py - 乌鲁木齐市环境科学学会 (wlmqshjkxxh.cn)
修复版：访问详情页获取HTML正文，支持多种内容格式
"""
import re, time, sys, os
from datetime import datetime, timedelta
import urllib.request, urllib.error

BASE = "http://wlmqshjkxxh.cn"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "乌鲁木齐环科学会"
CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

COLUMNS = {
    "环评公示": 15,
    "清洁生产审核": 17,
    "企业环境信息": 19,
}

HEADERS = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'}


def fetch(url, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers=HEADERS)
            with urllib.request.urlopen(req, timeout=20) as resp:
                return resp.read().decode('utf-8', errors='replace')
        except Exception as e:
            if i < retries - 1:
                time.sleep(2)
            else:
                return None


def get_page_count(html):
    pages = re.findall(r'href="[^"]*\?page=(\d+)"', html)
    return max(int(p) for p in pages) if pages else 1


def parse_list_page(html):
    """从列表页提取标题、日期、详情URL"""
    items = []
    pattern = (r'<a href="http://wlmqshjkxxh\.cn/products/show/(\d+)" '
               r'class="news_box flex-row-sb">(.*?)</a>')
    for m in re.finditer(pattern, html, re.DOTALL):
        item_id = m.group(1)
        block = m.group(2)
        title_m = re.search(r'<div class="text1 text-of-1">(.*?)</div>', block, re.DOTALL)
        title = title_m.group(1).strip() if title_m else ""
        date_m = re.search(r'<div class="text3">(.*?)</div>', block, re.DOTALL)
        date_str = date_m.group(1).strip() if date_m else ""
        if title and date_str:
            page_url = f"{BASE}/products/show/{item_id}"
            items.append((title, page_url, date_str))
    return items


def extract_detail_content(html):
    """从详情页提取正文HTML，支持多种格式"""
    # Method 1: <p class="p" ...> paragraphs (main content format)
    parts = []
    search_from = 0
    while True:
        p_idx = html.find('<p class="p"', search_from)
        if p_idx < 0:
            break
        close_idx = html.find('</p>', p_idx + 10)
        if close_idx < 0:
            break
        p_html = html[p_idx:close_idx + 4]
        parts.append(p_html)
        search_from = close_idx + 4
        if len(parts) > 200:
            break
    
    if parts:
        return "\n".join(parts)
    
    # Method 2: Look for content in news_text div on detail page
    m = re.search(r'class="news_text">(.*?)</div>\s*</div>\s*</div>\s*</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
        # Remove the mask div
        content = re.sub(r'<div class="news_mask.*?</div>\s*', '', content, flags=re.DOTALL)
        # Remove text1 (title) and text3 (date) divs, keep only text2
        text2_m = re.search(r'<div class="text2 text-of-2">(.*?)</div>', content, re.DOTALL)
        if text2_m:
            text2 = text2_m.group(1).strip()
            # Wrap in <p> tags for consistency
            lines = [l.strip() for l in text2.split('\n') if l.strip()]
            return "\n".join(f"<p>{line}</p>" for line in lines)
        return content
    
    # Method 3: Try any <p> tags (generic fallback)
    ps = re.findall(r'<p[^>]*>(.*?)</p>', html, re.DOTALL)
    if ps:
        meaningful = [p.strip() for p in ps if len(p.strip()) > 20]
        if meaningful:
            return "\n".join(f"<p>{p}</p>" for p in meaningful)
    
    return ""


def main():
    print(f"[{datetime.now().isoformat()}] Starting crawl: {SITE_NAME}")
    print(f"Cutoff date: {CUTOFF_DATE}")
    
    all_items = []
    
    for col_name, col_id in COLUMNS.items():
        url = f"{BASE}/products/list/{col_id}"
        print(f"\n--- {col_name} (col={col_id}) ---")
        html = fetch(url)
        if not html:
            print(f"  [ERROR] Failed")
            continue
        
        page_count = get_page_count(html)
        items = parse_list_page(html)
        print(f"  Page 1: {len(items)} items (total pages: {page_count})")
        all_items.extend([(t, u, d, col_name) for t, u, d in items])
        
        for page in range(2, page_count + 1):
            url = f"{BASE}/products/list/{col_id}?page={page}"
            html = fetch(url)
            if not html:
                break
            page_items = parse_list_page(html)
            print(f"  Page {page}: {len(page_items)} items")
            all_items.extend([(t, u, d, col_name) for t, u, d in page_items])
            time.sleep(0.5)
    
    print(f"\n=== Total list items: {len(all_items)} ===")
    
    if not all_items:
        print("No items to process.")
        return
    
    import sqlite3
    
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()
    
    inserted = 0
    skipped = 0
    total = len(all_items)
    
    for i, (title, page_url, date_str, col_name) in enumerate(all_items):
        if date_str < CUTOFF_DATE:
            skipped += 1
            continue
        
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (page_url,))
        if c.fetchone():
            skipped += 1
            continue
        
        print(f"  [{i+1}/{total}] {date_str} | {title[:35]}...", end=" ")
        sys.stdout.flush()
        
        detail_html = fetch(page_url)
        if not detail_html:
            print("[SKIP fetch fail]")
            skipped += 1
            continue
        
        content = extract_detail_content(detail_html)
        if not content:
            print("[SKIP no content]")
            skipped += 1
            continue
        
        try:
            c.execute("""INSERT OR REPLACE INTO gov_raw 
                (site_name, source_url, page_url, title, publish_date, content, status, summary)
                VALUES (?, ?, ?, ?, ?, ?, 'published', ?)""",
                (f"{SITE_NAME}-{col_name}", page_url, page_url, title, date_str, content,
                 content[:200].replace('\n', ' ').strip()))
            inserted += 1
            conn.commit()
            print(f"[OK HTML len={len(content)}]")
        except Exception as e:
            conn.rollback()
            print(f"[ERR] {e}")
            skipped += 1
        
        time.sleep(0.3)
    
    conn.close()
    
    print(f"\n{'='*50}")
    print(f"Completed: {datetime.now().isoformat()}")
    print(f"Total: {total}, Inserted: {inserted}, Skipped: {skipped}")


if __name__ == '__main__':
    main()