#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""crawl_kmsthjj.py — 昆明市生态环境局-建设项目环境影响评价信息"""

import os, re, sqlite3, time, sys, random, urllib.parse
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed

DB_PATH = os.environ.get('SEARCH_DB', '/root/search.db')
SITE_NAME = '昆明市生态环境局-建设项目环境影响评价信息'
BASE = 'https://sthjj.km.gov.cn'
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')
MAX_WORKERS = 5

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
}

import requests
sess = requests.Session()
sess.headers.update(HEADERS)


def fetch(url, timeout=15):
    full_url = urllib.parse.urljoin(BASE, url) if not url.startswith('http') else url
    for _ in range(3):
        try:
            r = sess.get(full_url, timeout=timeout)
            r.encoding = 'utf-8'
            return r.text
        except Exception as e:
            time.sleep(1)
    return None


def parse_list_page(html):
    """Parse article items from a list page"""
    items = []
    # Each item: <div class="data-table-item clearfix">
    for block in re.finditer(r'<div class="data-table-item clearfix">(.*?)</div>\s*</div>\s*</div>', html, re.DOTALL):
        content = block.group(1)
        a = re.search(r'<a href="([^"]+)">\s*([^<]+)\s*</a>', content)
        date_match = re.search(r'title="(\d{4}-\d{2}-\d{2})', content)
        if a:
            items.append({
                'url': urllib.parse.urljoin(BASE, a.group(1)),
                'title': a.group(2).strip(),
                'date': date_match.group(1) if date_match else '',
            })
    return items


def parse_detail(html):
    """Extract content from detail page"""
    # Title from <meta ArticleTitle>
    title_m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
    title = title_m.group(1) if title_m else ''
    
    if not title:
        h1 = re.search(r'<h1>(.*?)</h1>', html)
        title = h1.group(1).strip() if h1 else ''
    
    # Content from <div class="activity">
    content_m = re.search(r'<div class="activity">(.*?)</div>\s*</div>\s*</div>\s*</div>\s*</div>\s*</div>', html, re.DOTALL)
    if not content_m:
        content_m = re.search(r'<div class="activity">(.*?)</div>\s*</div>\s*</div>', html, re.DOTALL)
    
    content = content_m.group(1).strip() if content_m else ''
    
    # Clean
    content = re.sub(r'<script[^>]*>.*?</script>', '', content, flags=re.DOTALL)
    content = re.sub(r'<style[^>]*>.*?</style>', '', content, flags=re.DOTALL)
    content = re.sub(r'<o:p>\s*</o:p>', '', content)
    content = re.sub(r'<o:p/>', '', content)
    content = re.sub(r'<p[^>]*>\s*(?:<br\s*/?>\s*)*</p>', '', content)
    
    return title, content.strip()


def process_item(item):
    """Fetch detail and return data for insertion"""
    html = fetch(item['url'])
    if not html:
        return None, f'FETCH_ERR: {item["title"][:30]}'
    
    dt, content = parse_detail(html)
    final_title = dt or item['title']
    
    return {
        'site_name': SITE_NAME,
        'title': final_title,
        'page_url': item['url'],
        'publish_date': item['date'],
        'source_url': item['url'],
        'content': content,
    }, None


def main():
    print(f'[{datetime.now().strftime("%H:%M:%S")}] {SITE_NAME}')
    
    # Step 1: Fetch all list pages
    list_urls = [
        'https://sthjj.km.gov.cn/zfxxgk/fdzdgknr/zdlyxxgk/hjbh/jsxmhjyxpjxx/index.shtml',
    ]
    for i in range(2, 6):  # pages 2-5
        list_urls.append(f'https://sthjj.km.gov.cn/zfxxgk/fdzdgknr/zdlyxxgk/hjbh/jsxmhjyxpjxx/index_{i}.shtml')
    
    all_items = []
    for url in list_urls:
        html = fetch(url)
        if html:
            items = parse_list_page(html)
            all_items.extend(items)
            print(f'  List page {list_urls.index(url)+1}: {len(items)} items')
        else:
            print(f'  [WARN] Failed: {url}')
        time.sleep(random.uniform(0.3, 0.5))
    
    # Filter duplicates by URL
    seen = set()
    unique = []
    for item in all_items:
        if item['url'] not in seen:
            seen.add(item['url'])
            unique.append(item)
    
    print(f'\nTotal unique: {len(unique)} (deduped from {len(all_items)})')
    
    # Filter by date cutoff
    recent = [i for i in unique if i['date'] >= CUTOFF]
    print(f'Within 3 years: {len(recent)} (filtered: {len(unique)-len(recent)})')
    
    # Step 2: Check existing in DB
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    existing_urls = set()
    for row in c.execute('SELECT page_url FROM gov_raw WHERE site_name=?', (SITE_NAME,)):
        existing_urls.add(row[0])
    
    to_fetch = [i for i in recent if i['url'] not in existing_urls]
    print(f'New items to fetch: {len(to_fetch)}')
    
    # Step 3: Multi-threaded detail fetch
    new_count = 0
    error_count = 0
    
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        futures = {executor.submit(process_item, item): item for item in to_fetch}
        
        for i, future in enumerate(as_completed(futures), 1):
            item = futures[future]
            result, err = future.result()
            
            if err or result is None:
                print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... {err or "UNKNOWN ERR"}')
                error_count += 1
                continue
            
            try:
                c.execute('''INSERT OR IGNORE INTO gov_raw
                    (site_name, title, page_url, publish_date, source_url, content)
                    VALUES (?,?,?,?,?,?)''',
                    (result['site_name'], result['title'], result['page_url'],
                     result['publish_date'], result['source_url'], result['content']))
                if c.rowcount:
                    new_count += 1
                    print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... OK ({len(result["content"])}B)')
                else:
                    print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... DUP')
            except Exception as e:
                print(f'  [{i}/{len(to_fetch)}] {item["title"][:30]}... ERR: {e}')
                error_count += 1
            
            conn.commit()
    
    conn.close()
    print(f'\n=== Done ===')
    print(f'New: {new_count}, Errors: {error_count}')


if __name__ == '__main__':
    main()