#!/usr/bin/env python3
import os
"""Resume detail fetching for chiping items not yet in DB"""
import sys, re, json, os, requests, sqlite3
from datetime import datetime, timedelta
from playwright.sync_api import sync_playwright

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "茌平区-通知公示"
BASE_URL = "http://www.chiping.gov.cn/channel_x_0_5747/"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def get_existing_urls():
    conn = sqlite3.connect(SEARCH_DB)
    c = conn.cursor()
    c.execute('SELECT source_url FROM gov_raw WHERE site_name = ?', (SITE_NAME,))
    urls = set(r[0] for r in c.fetchall())
    conn.close()
    return urls

def get_all_list_items():
    existing = get_existing_urls()
    new_items = []
    
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True, args=['--disable-dev-shm-usage'])
        context = browser.new_context(user_agent=HEADERS["User-Agent"])
        page = context.new_page()
        page.goto(BASE_URL, wait_until='networkidle', timeout=30000)
        
        extracted = set(existing)  # Skip already-saved items
        while True:
            lis = page.query_selector_all('div.list li')
            for li in lis:
                a = li.query_selector('a')
                time_el = li.query_selector('.time')
                if a:
                    url = a.get_attribute('href')
                    full_url = url if url.startswith('http') else f'http://www.chiping.gov.cn{url}'
                    if full_url not in extracted:
                        extracted.add(full_url)
                        title = a.get_attribute('title') or a.inner_text().strip()
                        date = time_el.inner_text().strip() if time_el else ""
                        if date >= CUTOFF:
                            new_items.append({'title': title[:200], 'url': full_url, 'date': date})
            
            if new_items and new_items[-1]['date'] < CUTOFF:
                break
            next_btn = page.query_selector('a.layui-laypage-next:not(.layui-disabled)')
            if not next_btn:
                break
            next_btn.click()
            page.wait_for_timeout(300)
        
        browser.close()
    
    return new_items

def save_items(items):
    conn = sqlite3.connect(SEARCH_DB)
    conn.execute('PRAGMA journal_mode=WAL')
    c = conn.cursor()
    
    new_count = 0
    for i, item in enumerate(items):
        try:
            r = requests.get(item['url'], headers=HEADERS, timeout=30)
            r.encoding = 'utf-8'
            html = r.text
            title = (re.search(r'<title>(.*?)</title>', html, re.DOTALL) or ['']).group(1).strip()[:200]
            content = ''
            for pat in [r'<div class="content"[^>]*>(.*?)</div>\s*</div>', r'<div class="detail"[^>]*>(.*?)</div>\s*</div>']:
                m = re.search(pat, html, re.DOTALL)
                if m:
                    c2 = re.sub(r'<script[^>]*>.*?</script>', '', m.group(1), flags=re.DOTALL|re.I)
                    c2 = re.sub(r'<style[^>]*>.*?</style>', '', c2, flags=re.DOTALL|re.I)
                    if len(c2.strip()) > 50:
                        content = c2.strip()
                        break
            date = item['date']
            for pat in [r'发布日[期期]\s*[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})']:
                m = re.search(pat, html)
                if m: date = m.group(1).replace('/', '-'); break
            
            c.execute('''INSERT OR IGNORE INTO gov_raw 
                (title, content, summary, source_url, site_name, publish_date, category)
                VALUES (?,?,?,?,?,?,?)''',
                (title, content, title[:200], item['url'], SITE_NAME, date, ''))
            if c.rowcount > 0: new_count += 1
        except Exception as e:
            print(f"  Error: {e}", flush=True)
        
        if (i+1) % 30 == 0:
            conn.commit()
            print(f"  {i+1}/{len(items)} ({new_count} new)", flush=True)
    
    conn.commit()
    conn.close()
    return new_count

def update_fts():
    conn = sqlite3.connect(SEARCH_DB)
    c = conn.cursor()
    c.execute('''INSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary)
        SELECT r.id, r.title, r.site_name, r.summary
        FROM gov_raw r LEFT JOIN gov_search s ON r.id = s.rowid
        WHERE s.rowid IS NULL AND r.site_name = ?''', (SITE_NAME,))
    conn.commit()
    print(f"FTS: {c.rowcount} new", flush=True)
    conn.close()

def main():
    existing = get_existing_urls()
    print(f"Existing in DB: {len(existing)}", flush=True)
    
    print("Phase 1: List pages...", flush=True)
    items = get_all_list_items()
    print(f"New items to fetch: {len(items)}", flush=True)
    
    if items:
        print("Phase 2: Details...", flush=True)
        new = save_items(items)
        print(f"New: {new}", flush=True)
        
        print("Phase 3: FTS...", flush=True)
        update_fts()
    
    # Final count
    existing2 = get_existing_urls()
    print(f"\nFinal: {len(existing2)} items in DB", flush=True)

if __name__ == '__main__':
    main()