#!/usr/bin/env python3
"""crawl_dmlhq_gsgg.py - 达尔罕茂明安联合旗-公示公告"""
import sys, os, requests, re
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

CUTOFF = (datetime.now() - timedelta(days=365*3)).strftime('%Y-%m-%d')
SITE_NAME = '达茂联合旗-公示公告'
BASE = 'http://dmlhq.gov.cn/ywdt/ywdt_gsgg'

def fetch_urls_from_page(page_num):
    if page_num == 0:
        url = BASE + '/'
    else:
        url = f'{BASE}/index_{page_num}.html'
    try:
        r = requests.get(url, timeout=20)
        r.encoding = 'utf-8'
    except:
        return []
    m = re.search(r'<ul class="list">(.*?)</ul>', r.text, re.DOTALL)
    if not m:
        return []
    items = re.findall(
        r'<a href="([^"]+)"[^>]*>\s*<span>\s*(.*?)\s*</span>\s*<span>\s*(\d{4}-\d{2}-\d{2})\s*</span>',
        m.group(1), re.DOTALL
    )
    results = []
    for href, title_html, pub_date in items:
        if href.startswith('./'):
            full_url = BASE + href[1:]
        else:
            full_url = BASE + '/' + href
        results.append({'url': full_url, 'pub_date': pub_date})
    return results

def fetch_detail(url):
    try:
        r = requests.get(url, timeout=15)
        r.encoding = 'utf-8'
    except:
        return '', ''
    
    title = ''
    m = re.search(r'<div class="title">(.*?)<div class="other">', r.text, re.DOTALL)
    if m:
        title = re.sub(r'<[^>]+>', '', m.group(1))
        title = re.sub(r'\s+', ' ', title).strip()
    
    content = ''
    m = re.search(
        r'<div class="trs_editor_view TRS_UEDITOR[^"]*"[^>]*>(.*?)</div>\s*</div>',
        r.text, re.DOTALL
    )
    if m:
        content = m.group(1).strip()
    
    return title, content

def run(incremental=False):
    # Phase 1: collect URLs from list pages (fast, static HTML)
    all_items = []
    seen_urls = set()
    
    max_pages = 1 if incremental else 70
    for pg in range(max_pages):
        items = fetch_urls_from_page(pg)
        if not items:
            break
        
        for item in items:
            if item['pub_date'] < CUTOFF:
                continue
            if item['url'] not in seen_urls:
                seen_urls.add(item['url'])
                all_items.append(item)
        
        if pg % 10 == 0:
            print(f"  List page {pg+1}: {len(items)} items, {len(all_items)} collected")
    
    print(f"  Phase 1 done: {len(all_items)} items to fetch")
    
    # Phase 2: fetch details with thread pool (concurrent)
    records = []
    done = 0
    
    with ThreadPoolExecutor(max_workers=5) as executor:
        future_map = {executor.submit(fetch_detail, item['url']): item for item in all_items}
        for future in as_completed(future_map):
            item = future_map[future]
            done += 1
            try:
                title, content = future.result()
            except:
                title, content = '', ''
            
            if title and content.strip():
                records.append({
                    'title': title,
                    'url': item['url'],
                    'pub_date': item['pub_date'],
                    'site_name': SITE_NAME,
                    'content': content,
                    'summary': '',
                })
            
            if done % 10 == 0:
                print(f"  Details: {done}/{len(all_items)} done, {len(records)} with content")
    
    print(f"  Total: {len(records)} records with content")
    if records:
        push_to_searchdb(records, "dmlhq_gsgg")
    return len(records)

if __name__ == '__main__':
    inc = '--incremental' in sys.argv
    cnt = run(incremental=inc)
    print(f"Done: {cnt} records")