#!/usr/bin/env python3
import os
"""crawl_qg.py - 泉州市泉港区人民政府 项目环评"""
import re, time, sys, os
import urllib.request, urllib.error
from datetime import datetime

BASE_URL = "https://www.qg.gov.cn/zwgk/zdxxgk/hjbh/xmhp"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "泉州市泉港区-项目环评"

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}

def fetch(url, retries=3):
    for i in range(retries):
        try:
            req = urllib.request.Request(url, headers=HEADERS)
            with urllib.request.urlopen(req, timeout=30) as resp:
                return resp.read().decode('utf-8', errors='replace')
        except Exception as e:
            if i < retries - 1:
                time.sleep(2)
            else:
                print(f"  [WARN] Failed to fetch {url}: {e}")
                return None

def extract_links(html):
    """Extract list items from a page"""
    items = []
    # Find all list item links
    pattern = r'<a href="\./(\d{6}/t\d+_?\d*\.htm)"[^>]*title="([^"]*)"'
    for m in re.finditer(pattern, html):
        href, title = m.group(1), m.group(2)
        page_url = f"{BASE_URL}/{href}"
        pub_date = href[:4] + '-' + href[4:6] + '-' + '01'
        items.append((title, page_url, pub_date))
    return items

def extract_content(html):
    """Extract content from detail page"""
    content = ""
    m = re.search(r'<div[^>]*class="TRS_Editor"[^>]*>(.*?)</div>', html, re.DOTALL)
    if m:
        content = m.group(1).strip()
    if not content:
        # Fallback: try common containers
        for cls in ['article-content', 'news-content', 'content', 'maintext']:
            m = re.search(r'<div[^>]*class="[^"]*' + cls + r'[^"]*"[^>]*>(.*?)</div>\s*</div>', html, re.DOTALL)
            if m:
                content = m.group(1).strip()
                break
    return content

def extract_date(html, default_date):
    """Extract publish date from detail page"""
    # Look for date in common patterns
    m = re.search(r'(\d{4}-\d{2}-\d{2})', html)
    if m:
        return m.group(1)
    return default_date

def extract_title(html, fallback_title):
    """Extract clean title from detail page"""
    m = re.search(r'<title>(.*?)</title>', html)
    if m:
        title = m.group(1)
        # Clean up TRS title suffixes
        for suffix in ['-项目环评', '-生态环境', '-重点信息公开', '-政务公开', '-泉州市泉港区人民政府']:
            idx = title.rfind(suffix)
            if idx > 0:
                title = title[:idx]
                break
        return title.strip()
    return fallback_title

def get_total_pages():
    """Determine total pages by probing"""
    # Page 1 is the index page
    # Pages 2+ are index_N.htm
    for n in range(30, 1, -1):
        url = f"{BASE_URL}/index_{n}.htm"
        req = urllib.request.Request(url, headers=HEADERS)
        try:
            resp = urllib.request.urlopen(req, timeout=10)
            if resp.status == 200:
                return n
        except:
            continue
    return 2

def main():
    # 页数限制: 支持裸数字 / --pages=N / --pages N (默认0=全量)
    max_pages = 0
    for i, a in enumerate(sys.argv):
        if a.isdigit():
            max_pages = int(a)
        elif a.startswith("--pages="):
            try:
                max_pages = int(a.split("=")[1])
            except ValueError:
                pass
        elif a == "--pages" and i + 1 < len(sys.argv) and sys.argv[i+1].isdigit():
            max_pages = int(sys.argv[i+1])
    print(f"[{datetime.now().isoformat()}] Starting crawl: {SITE_NAME} (max_pages={max_pages})")
    
    # Collect all links from all pages
    all_items = []
    
    # Page 1
    print("Fetching page 1...")
    html = fetch(BASE_URL + '/')
    if html:
        items = extract_links(html)
        print(f"  Found {len(items)} items")
        all_items.extend(items)
    
    # Pages 2+
    page = 2
    while True:
        if max_pages > 0 and page > max_pages:
            break
        url = f"{BASE_URL}/index_{page}.htm"
        print(f"Fetching page {page}...")
        html = fetch(url)
        if not html:
            break
        items = extract_links(html)
        if not items:
            break
        print(f"  Found {len(items)} items")
        all_items.extend(items)
        page += 1
        time.sleep(0.5)
    
    print(f"\nTotal items to process: {len(all_items)}")
    
    # Import into database
    import sqlite3
    
    conn = sqlite3.connect(DB_PATH, timeout=30)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()
    
    inserted = 0
    skipped = 0
    
    for title, page_url, default_date in all_items:
        # Check if already exists
        c.execute("SELECT id FROM gov_raw WHERE page_url=?", (page_url,))
        if c.fetchone():
            skipped += 1
            continue
        
        # Fetch detail page
        detail_html = fetch(page_url)
        if not detail_html:
            skipped += 1
            continue
        
        # Extract content
        content = extract_content(detail_html)
        if not content:
            print(f"  [SKIP] No content: {title[:30]}")
            skipped += 1
            continue
        
        # Extract details
        clean_title = extract_title(detail_html, title)
        pub_date = extract_date(detail_html, default_date)
        
        try:
            c.execute("""INSERT OR REPLACE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, status, summary, script_name) VALUES (?, ?, ?, ?, ?, ?, 'published', ?, 'crawl_qg.py')""",
                (SITE_NAME, BASE_URL, page_url, clean_title, pub_date, content,
                 content[:200].replace('\n', ' ').strip()))
            inserted += 1
        except Exception as e:
            print(f"  [ERR] DB error for {page_url}: {e}")
            skipped += 1
    
    conn.commit()
    conn.close()
    
    print(f"\n=== Completed ===")
    print(f"Inserted: {inserted}")
    print(f"Skipped: {skipped}")
    print(f"Total: {len(all_items)}")

if __name__ == '__main__':
    main()