#!/usr/bin/env python3
import os
"""Crawl chiping using Playwright only for first page, then requests for rest"""
import sys, re, json, os, requests
from datetime import datetime, timedelta
import sqlite3
from playwright.sync_api import sync_playwright

SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "茌平区-通知公示"
BASE_URL = "http://www.chiping.gov.cn/channel_x_0_5747/"
CUTOFF = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def get_all_items_from_playwright():
    """Open page in Playwright, click through ALL pages, collect items"""
    all_items = []
    extracted = set()
    
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True, args=['--disable-dev-shm-usage'])
        context = browser.new_context(user_agent=HEADERS["User-Agent"])
        page = context.new_page()
        page.goto(BASE_URL, wait_until='networkidle', timeout=30000)
        
        while True:
            # Extract items from current page
            lis = page.query_selector_all('div.list li')
            for li in lis:
                a = li.query_selector('a')
                time_el = li.query_selector('.time')
                if a:
                    url = a.get_attribute('href')
                    if url and url not in extracted:
                        extracted.add(url)
                        full_url = url if url.startswith('http') else f'http://www.chiping.gov.cn{url}'
                        title = a.get_attribute('title') or a.inner_text().strip()
                        date = time_el.inner_text().strip() if time_el else ""
                        all_items.append({'title': title[:200], 'url': full_url, 'date': date})
            
            # Check cutoff on last item
            if all_items and all_items[-1]['date'] < CUTOFF:
                break
            
            # Next page
            next_btn = page.query_selector('a.layui-laypage-next:not(.layui-disabled)')
            if not next_btn:
                break
            next_btn.click()
            page.wait_for_timeout(800)
        
        browser.close()
    
    return all_items

def fetch_detail(url):
    try:
        r = requests.get(url, headers=HEADERS, timeout=30)
        r.encoding = 'utf-8'
        html = r.text
        # Title: prefer ArticleTitle meta tag (clean, no prefix)
        title = ''
        ar = re.search(r'<meta name="ArticleTitle" content="(.*?)"', html, re.DOTALL)
        if ar:
            title = ar.group(1).strip()[:200]
        else:
            t = re.search(r'<title>(.*?)</title>', html, re.DOTALL)
            if t:
                title = t.group(1).strip()[:200]
                title = re.sub(r'^.*? - [^\-]+? - ', '', title)
        content = ''
        for pat in [
            r'<div class="content-text"[^>]*>(.*?)</div>\s*</div>',
            r'<div class="content"[^>]*>(.*?)</div>\s*</div>',
            r'<div class="detail"[^>]*>(.*?)</div>\s*</div>',
        ]:
            m = re.search(pat, html, re.DOTALL)
            if m:
                c = re.sub(r'<script[^>]*>.*?</script>', '', m.group(1), flags=re.DOTALL|re.I)
                c = re.sub(r'<style[^>]*>.*?</style>', '', c, flags=re.DOTALL|re.I)
                if len(c.strip()) > 50:
                    content = c.strip()
                    break
        date = ''
        for pat in [r'发布日[期期]\s*[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})', r'(\d{4}[-/]\d{1,2}[-/]\d{1,2})']:
            m = re.search(pat, html)
            if m:
                date = m.group(1).replace('/', '-')
                break
        return title, date, content
    except:
        return "", "", ""

def save_to_db(items):
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    conn.execute('PRAGMA journal_mode=WAL')
    c = conn.cursor()
    c.execute('''CREATE TABLE IF NOT EXISTS gov_raw (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        title TEXT, content TEXT, summary TEXT,
        source_url TEXT UNIQUE, site_name TEXT,
        publish_date TEXT, category TEXT
    )''')
    
    new_count = 0
    for i, item in enumerate(items):
        c.execute('SELECT 1 FROM gov_raw WHERE source_url = ?', (item['url'],))
        if c.fetchone():
            continue
        
        title, date, content = fetch_detail(item['url'])
        title = title or item['title']
        date = date or item['date']
        if not content or not content.strip():
            content = '<p>无法获取正文</p>'

        
        c.execute('''INSERT OR REPLACE INTO gov_raw (title, content, summary, source_url, site_name, publish_date, category, script_name) VALUES (?,?,?,?,?,?,?, \'crawl_chiping.py\')''',
            (title, content, title[:200], item['url'], SITE_NAME, date, ''))
        if c.rowcount > 0:
            new_count += 1
        
        if (i + 1) % 20 == 0:
            conn.commit()
            print(f"  Saved {new_count}/{i+1} items...", flush=True)
    
    conn.commit()
    conn.close()
    return new_count

def update_fts():
    conn = sqlite3.connect(SEARCH_DB, timeout=60)
    c = conn.cursor()
    try:
        c.execute('''INSERT OR REPLACE INTO gov_search (rowid, title, site_name, summary)
            SELECT r.id, r.title, r.site_name, r.summary
            FROM gov_raw r LEFT JOIN gov_search s ON r.id = s.rowid
            WHERE s.rowid IS NULL''')
        conn.commit()
        print(f"  FTS: {c.rowcount} new rows", flush=True)
    except Exception as e:
        print(f"  FTS error: {e}", flush=True)
    conn.close()

def main():
    print(f"Starting: {SITE_NAME}", flush=True)
    
    # Step 1: Get all items from list pages via Playwright
    print("Phase 1: Crawling list pages...", flush=True)
    items = get_all_items_from_playwright()
    items = [i for i in items if i['date'] >= CUTOFF]
    print(f"  Collected {len(items)} items (within 3 years)", flush=True)
    
    # Step 2: Fetch details
    print("Phase 2: Fetching details...", flush=True)
    new = save_to_db(items)
    print(f"  New items saved: {new}", flush=True)
    
    # Step 3: Update FTS
    print("Phase 3: Updating FTS...", flush=True)
    update_fts()
    
    print("Done!", flush=True)

if __name__ == '__main__':
    main()