#!/usr/bin/env python3
"""crawl_ouryq.py - 企业环保信息公示网 (www.ouryq.com)
4个栏目：环评信息公示/eia、竣工验收公示/epa、固废信息公示/esw、其他公示/other
单页全量加载，详情页HTML正文提取
"""
import re, time, sys, os
import urllib.request, urllib.error
from datetime import datetime

BASE = "https://www.ouryq.com"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "企业环保信息公示网"
CUTOFF_DATE = (datetime.now().replace(year=datetime.now().year - 3)).strftime("%Y-%m-%d")

# 4个栏目配置
CATEGORIES = {
    "环评信息公示": "/eia",
    "竣工验收公示": "/epa",
    "固废信息公示": "/esw",
    "其他公示": "/other",
}

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
}


def fetch(url, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers=HEADERS)
            with urllib.request.urlopen(req, timeout=30) as resp:
                return resp.read().decode('utf-8', errors='replace')
        except Exception as e:
            if i < retries - 1:
                time.sleep(2)
            else:
                print(f"  [WARN] Failed to fetch {url}: {e}")
                return None


def parse_list_page(html, cat_name, cat_path):
    """解析分类列表页表格，返回 [(title, page_url, pub_date), ...]"""
    items = []
    # 提取所有行：检查每行是否有详情页链接
    # 每行结构：<tr>...<td title="企业名称">...</td>...<td><a href="https://..."><img src="check.png"></a></td></tr>
    # 使用正则匹配整行
    row_pattern = r'<tr[^>]*>(.*?)</tr>'
    for tr_match in re.finditer(row_pattern, html, re.DOTALL):
        tr_html = tr_match.group(1)
        # Skip header row
        if 'tbhead' in tr_html or not tr_html.strip():
            continue
        
        # Extract detail page URL
        url_match = re.search(r'href="(https://www\.ouryq\.com' + re.escape(cat_path) + r'/\d+\.html)"', tr_html)
        if not url_match:
            continue
        page_url = url_match.group(1)
        
        # Extract title from 项目名称 column
        title = ""
        title_match = re.search(r'<td[^>]*title="([^"]*)"[^>]*style="text-align:\s*left', tr_html)
        if title_match:
            title = title_match.group(1).strip()
        
        # Extract publish date
        pub_date = ""
        # Find the last few td's - 发布日期 is usually the 7th column
        date_match = re.search(r'<td[^>]*title="(\d{4}年\d{1,2}月\d{1,2}日)"', tr_html)
        if date_match:
            date_str = date_match.group(1)
            # Convert "2026年3月24日" to "2026-03-24"
            try:
                dt = datetime.strptime(date_str, "%Y年%m月%d日")
                pub_date = dt.strftime("%Y-%m-%d")
            except:
                pub_date = date_str
        
        if title and pub_date:
            items.append((title, page_url, pub_date, cat_name))
    
    return items


def extract_detail(html, cat_name):
    """从详情页提取正文"""
    # Content in <div style="text-indent:2em;line-height:30px;">
    content = ""
    m = re.search(r'<div\s+style="text-indent:2em;line-height:30px;">(.*?)</div>\s*</div>\s*</div>\s*</div>', html, re.DOTALL)
    if not m:
        m = re.search(r'<div\s+style="text-indent:2em;line-height:30px;">(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    
    return content


def main():
    print(f"[{datetime.now().isoformat()}] Starting crawl: {SITE_NAME}")
    print(f"Cutoff date: {CUTOFF_DATE}")
    
    all_items = []
    
    # Stage 1: Scrape all category list pages
    for cat_name, cat_path in CATEGORIES.items():
        url = BASE + cat_path
        print(f"\n--- {cat_name} ({url}) ---")
        html = fetch(url)
        if not html:
            print(f"  [ERROR] Failed to fetch {url}")
            continue
        
        items = parse_list_page(html, cat_name, cat_path)
        print(f"  Found {len(items)} items")
        
        # Filter by date
        before = len(items)
        items = [it for it in items if it[2] >= CUTOFF_DATE]
        print(f"  After date filter ({CUTOFF_DATE}): {len(items)}/{before}")
        
        all_items.extend(items)
    
    print(f"\n=== Total list items across all categories: {len(all_items)} ===")
    
    if not all_items:
        print("No items to process. Exiting.")
        return
    
    # Stage 2: Fetch detail pages and save to DB
    import sqlite3
    
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()
    
    inserted = 0
    skipped = 0
    total = len(all_items)
    
    for i, (title, page_url, pub_date, cat_name) in enumerate(all_items):
        # Check if already exists
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (page_url,))
        if c.fetchone():
            skipped += 1
            continue
        
        print(f"  [{i+1}/{total}] {cat_name} | {title[:30]}...", end=" ")
        sys.stdout.flush()
        
        detail_html = fetch(page_url)
        if not detail_html:
            print("[SKIP fetch failed]")
            skipped += 1
            continue
        
        content = extract_detail(detail_html, cat_name)
        if not content:
            print("[SKIP no content]")
            skipped += 1
            continue
        
        # category_name for display in search
        category_tag = f"ouryq-{cat_name}"
        
        try:
            c.execute("""INSERT OR REPLACE INTO gov_raw 
                (site_name, source_url, page_url, title, publish_date, content, status, summary)
                VALUES (?, ?, ?, ?, ?, ?, 'published', ?)""",
                (f"{SITE_NAME}-{cat_name}", page_url, page_url, title, pub_date, content,
                 content[:200].replace('\n', ' ').strip()))
            inserted += 1
            # 每条提交，防止超时丢失数据
            conn.commit()
            print(f"[OK len={len(content)}]")
        except Exception as e:
            conn.rollback()
            print(f"[ERR] {e}")
            skipped += 1
        
        time.sleep(0.3)
    
    conn.close()
    
    print(f"\n{'='*50}")
    print(f"Completed: {datetime.now().isoformat()}")
    print(f"Total items: {total}")
    print(f"Inserted: {inserted}")
    print(f"Skipped (exists/no content): {skipped}")


if __name__ == '__main__':
    main()