#!/usr/bin/env python3
"""
Crawl 七台河市茄子河区人民政府 - 公示公告
API: /common/search/{channelId}?page=N&_pageSize=20
"""
import requests, json, sqlite3, sys, time, hashlib
from datetime import datetime, timedelta
import os

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = '茄子河区 - 公示公告'
BASE_URL = 'https://www.hljqzh.gov.cn'
API_URL = 'https://www.hljqzh.gov.cn/common/search/362d996da26d459fade0f1ed3aca449a'
CUTOFF_DATE = (datetime.now() - timedelta(days=3*365)).strftime('%Y-%m-%d')

HEADERS = {'User-Agent': 'Mozilla/5.0'}
session = requests.Session()
session.headers.update(HEADERS)

def fetch_detail(url):
    """Extract title, date, content from detail page."""
    resp = session.get(url, timeout=30)
    resp.encoding = 'utf-8'
    html = resp.text
    
    # Title from meta
    tm = __import__('re').search(r'<meta name="ArticleTitle" content="([^"]*)"', html)
    title = tm.group(1).strip() if tm else ''
    
    # Date from meta
    dm = __import__('re').search(r'<meta name="PubDate" content="([^"]*)"', html)
    date_str = dm.group(1)[:10] if dm else ''
    
    # Content from div.wenzi
    cm = __import__('re').search(r'<div class="wenzi">(.*?)</div>\s*<div class="others', html, __import__('re').DOTALL)
    if cm:
        content = cm.group(1).strip()
        content = __import__('re').sub(r'<script[^>]*>.*?</script>', '', content, flags=__import__('re').DOTALL|__import__('re').I)
        content = __import__('re').sub(r'<style[^>]*>.*?</style>', '', content, flags=__import__('re').DOTALL|__import__('re').I)
        return title, date_str, content
    
    return title, date_str, ''

def main():
    is_incremental = 'incremental' in sys.argv
    print(f'=== {SITE_NAME} ===')
    
    # Get first page to know total
    resp = session.get(f'{API_URL}?_isAgg=false&_isJson=true&_pageSize=20&_template=index&_page=1', timeout=30)
    data = resp.json()
    total = data['data']['total']
    total_pages = (total + 19) // 20
    print(f'Total: {total}, Pages: {total_pages}')
    
    max_pages = 1 if is_incremental else total_pages
    
    conn = sqlite3.connect(DB_PATH, timeout=60)
    c = conn.cursor()
    inserted, skipped, old_total = 0, 0, 0
    consecutive_old = 0
    min_pages = 5
    start_time = time.time()
    
    for pg in range(1, max_pages + 1):
        try:
            resp = session.get(f'{API_URL}?_isAgg=false&_isJson=true&_pageSize=20&_template=index&page={pg}', timeout=30)
            data = resp.json()
            items = data['data']['results']
        except Exception as e:
            print(f'  [ERROR] Page {pg}: {e}')
            time.sleep(2)
            continue
        
        if not items:
            break
        
        # Check if all items are old
        dates = [i.get('publishedTimeStr', '')[:10] for i in items]
        all_old = all(d < CUTOFF_DATE for d in dates if d)
        if all_old:
            consecutive_old += 1
            if consecutive_old >= 2 and pg >= min_pages:
                print(f'  Page {pg}: all old, stopping')
                break
        else:
            consecutive_old = 0
        
        page_new = 0
        for item in items:
            pub_date = (item.get('publishedTimeStr') or '')[:10]
            if pub_date and pub_date < CUTOFF_DATE:
                old_total += 1
                continue
            
            title = (item.get('title') or '').strip()
            rel_url = item.get('url', '')
            page_url = BASE_URL + rel_url if rel_url.startswith('/') else rel_url
            
            has = c.execute('SELECT 1 FROM gov_raw WHERE page_url=? AND content IS NOT NULL AND content!=\'\'', (page_url,)).fetchone()
            if has:
                skipped += 1
                continue
            
            try:
                detail_title, detail_date, content = fetch_detail(page_url)
            except Exception as e:
                print(f'  [ERROR] Detail: {e}')
                time.sleep(1)
                continue
            
            final_title = detail_title or title
            final_date = detail_date or pub_date
            
            # Generate stable ID
            int_id = int(hashlib.md5(page_url.encode()).hexdigest()[:15], 16) % (2**63)
            date_rank = int(final_date.replace('-', '')) if final_date else 0
            
            summary = __import__('re').sub(r'<[^>]+>', '', content)[:200] if content else final_title
            summary = __import__('re').sub(r'\s+', ' ', summary).strip()
            
            try:
                c.execute(
                    'INSERT OR IGNORE INTO gov_raw(id, site_name, source_url, page_url, title, publish_date, content, summary, date_rank) VALUES(?,?,?,?,?,?,?,?,?)',
                    (int_id, SITE_NAME, page_url, page_url, final_title, final_date, content, summary, date_rank)
                )
                if c.rowcount > 0:
                    inserted += 1
                    page_new += 1
                else:
                    skipped += 1
            except Exception as e:
                print(f'  [DB] {e}')
                skipped += 1
            
            time.sleep(0.3)
        
        elapsed = time.time() - start_time
        print(f'  Page {pg}/{max_pages}: +{inserted} (skipped {skipped}, >3y {old_total}) [{elapsed:.0f}s]')
    
    conn.commit()
    
    # Rebuild FTS
    try:
        c2 = conn.cursor()
        c2.execute("DELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE site_name=?)", (SITE_NAME,))
        rows = c2.execute("SELECT id, title, site_name, summary FROM gov_raw WHERE site_name=?", (SITE_NAME,)).fetchall()
        c2.executemany("INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES(?,?,?,?)", rows)
        conn.commit()
        print(f'  FTS: {len(rows)} records rebuilt')
    except Exception as e:
        print(f'  FTS error: {e}')
    
    conn.close()
    elapsed = time.time() - start_time
    print(f'\nDone ({elapsed:.0f}s). Inserted {inserted}, Skipped {skipped}, Old {old_total}')

if __name__ == '__main__':
    main()