#!/usr/bin/env python3
"""
十堰经济技术开发区 - 生态环境栏目爬虫
Site: www.sygxq.gov.cn
CMS: TRS (createPageHTML)
List: li.list-group-item.border-0 > a[href], span for date
Detail: meta[ArticleTitle], meta[PubDate], div.view.TRS_UEDITOR
Pagination: index.shtml, index_N.shtml (6 pages total, 25 items/page)
"""

import os
import sys
import re
import json
import subprocess
import urllib.parse
from datetime import datetime, timezone

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "十堰经济技术开发区-生态环境"
COLUMN_NAME = "生态环境"
BASE_URL = "http://www.sygxq.gov.cn/xxgk/fdzdgk/zdmsxx/hjbh/"
CUTOFF_DATE = "2023-06-18"

CURL_OPTS = [
    "curl", "-sL", "--max-time", "30",
    "-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
]

def curl_get(url):
    """Fetch URL with curl"""
    cmd = CURL_OPTS + [url]
    try:
        result = subprocess.run(cmd, capture_output=True, text=True, timeout=35)
        if result.returncode == 0 and result.stdout:
            return result.stdout
        return None
    except:
        return None

def parse_list_page(html):
    """Parse list page HTML, return list of (url, title, date)"""
    items = re.findall(
        r'<li class="list-group-item border-0">.*?<a href="([^"]+)".*?title="([^"]+)".*?<span[^>]*>(\d{4}-\d{2}-\d{2})',
        html, re.DOTALL
    )
    result = []
    for href, title, date in items:
        # Resolve relative URL
        if href.startswith('./'):
            full_url = urllib.parse.urljoin(BASE_URL, href[2:])
        elif href.startswith('/'):
            full_url = urllib.parse.urljoin('http://www.sygxq.gov.cn', href)
        elif href.startswith('http'):
            full_url = href
        else:
            full_url = urllib.parse.urljoin(BASE_URL, href)
        result.append((full_url, title.strip(), date))
    return result

def fetch_detail(url):
    """Fetch detail page and extract title, date, content"""
    html = curl_get(url)
    if not html:
        return None, None, None
    
    # Extract title from meta
    title_m = re.search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
    title = title_m.group(1) if title_m else None
    
    # Extract date from meta
    date_m = re.search(r'<meta name="PubDate" content="([^"]*)"', html)
    date = date_m.group(1).strip() if date_m else None
    
    # Extract content: try TRS_UEDITOR first
    content = None
    ue_match = re.search(
        r'<div class="view TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>\s*</div>',
        html, re.DOTALL
    )
    if ue_match:
        content = ue_match.group(1).strip()
        # Fix relative image URLs
        content = re.sub(r'src="\./', f'src="{urllib.parse.urljoin(url, "./")}', content)
    else:
        # No TRS_UEDITOR - try PDF iframe
        pdf_match = re.search(
            r'<iframe[^>]*src="([^"]+\.pdf)"[^>]*>',
            html, re.DOTALL
        )
        if pdf_match:
            pdf_url = pdf_match.group(1)
            if not pdf_url.startswith('http'):
                pdf_url = urllib.parse.urljoin(url, pdf_url)
            content = f'<p>详细内容请查看PDF文件：</p><p><a href="{pdf_url}" target="_blank">{title or "查看PDF"}</a></p>'
        else:
            # Try extracting text from doc-content div
            dc_match = re.search(
                r'<div class="text-justify fz-18 lh-225">(.*?)</div>\s*<div class="social-share',
                html, re.DOTALL
            )
            if dc_match:
                content = dc_match.group(1).strip()
    
    if not title:
        h1_m = re.search(r'<h1[^>]*>([^<]+)</h1>', html)
        if h1_m:
            title = h1_m.group(1).strip()
        else:
            title_m2 = re.search(r'<title>([^<]+)</title>', html)
            if title_m2:
                title = title_m2.group(1).replace(' - 生态环境', '').strip()
    
    return title, date, content

def save_to_db(items):
    """Save items to search.db"""
    import sqlite3
    
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()
    
    # Truncate date to YYYY-MM-DD
    def truncate_date(d):
        if not d:
            return None
        return d[:10]
    
    new_count = 0
    for page_url, title, publish_date, content in items:
        try:
            c.execute("""
                INSERT OR IGNORE INTO gov_raw (page_url, title, publish_date, content, site_name, summary)
                VALUES (?, ?, ?, ?, ?, ?)
            """, (page_url, title, truncate_date(publish_date), content, SITE_NAME, title))
            if c.rowcount > 0:
                new_count += 1
        except Exception as e:
            print(f"  DB error: {e}", file=sys.stderr)
    
    conn.commit()
    conn.close()
    return new_count

def rebuild_fts():
    """Rebuild FTS index"""
    import sqlite3
    try:
        conn = sqlite3.connect(SEARCH_DB, timeout=60)
        c = conn.cursor()
        c.execute("SELECT 1 /* noop: gov_search 由触发器维护, 无需 rebuild */")
        conn.commit()
        conn.close()
        print("  FTS index rebuilt")
    except Exception as e:
        print(f"  FTS rebuild error: {e}", file=sys.stderr)

def main():
    print(f"Starting crawl: {SITE_NAME} - {COLUMN_NAME}")
    
    total_items = []
    page = 0
    max_pages = 6  # createPageHTML("6", ...)
    
    while page < max_pages:
        if page == 0:
            url = BASE_URL
        else:
            url = f"{BASE_URL}index_{page}.shtml"
        
        print(f"  Fetching page {page}/{max_pages-1}: {url}")
        html = curl_get(url)
        if not html:
            print(f"  Failed to fetch page {page}", file=sys.stderr)
            page += 1
            continue
        
        items = parse_list_page(html)
        print(f"    Found {len(items)} items")
        
        # Filter by date
        recent = [(u, t, d) for u, t, d in items if d >= CUTOFF_DATE]
        older = [(u, t, d) for u, t, d in items if d < CUTOFF_DATE]
        
        if older:
            print(f"    Filtered: {len(older)} older than {CUTOFF_DATE}")
        
        if recent:
            print(f"    Recent items: {len(recent)}")
            for u, t, d in recent:
                print(f"      {d} | {t[:50]}")
                total_items.append((u, t, d))
        
        # If all items on this page are older, stop
        if len(older) == len(items):
            print(f"    All items older than cutoff, stopping pagination")
            break
        
        page += 1
    
    print(f"\nTotal recent items: {len(total_items)}")
    
    if not total_items:
        print("No new items to process.")
        return
    
    # Fetch details for each item
    detail_items = []
    for url, title, date in total_items:
        print(f"  Fetching detail: {title[:40]}...")
        detail_title, detail_date, content = fetch_detail(url)
        if content:
            detail_items.append((url, detail_title or title, detail_date or date, content))
        else:
            print(f"    No content found for {url}", file=sys.stderr)
            detail_items.append((url, title, date, ''))
    
    # Save to DB
    print(f"\nSaving {len(detail_items)} items to DB...")
    new_count = save_to_db(detail_items)
    print(f"  Inserted {new_count} new records")
    
    # Rebuild FTS
    if new_count > 0:
        rebuild_fts()
    
    print("Done!")

if __name__ == "__main__":
    main()