#!/usr/bin/env python3
"""
Crawl 九江市政府 - 公告公示 (jiujiang.gov.cn/zwzx/gggs/)
Static HTML with TRS pagination: index.html, index_2.html, ...
"""

import sys, re, time
import requests
from bs4 import BeautifulSoup
from datetime import datetime, timedelta
import os

BASE_URL = "https://www.jiujiang.gov.cn"
LIST_PATH = "/zwzx/gggs"
DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE_NAME = "九江市政府 - 公告公示"

session = requests.Session()
session.headers.update({
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
})

THREE_YEARS_AGO = (datetime.now() - timedelta(days=3*365)).strftime("%Y-%m-%d")

def parse_list_page(html_text):
    """Parse list page, return list of dicts"""
    records = []
    soup = BeautifulSoup(html_text, 'html.parser')
    content = soup.find('div', class_='content')
    if not content:
        return records
    
    for ul in content.find_all('ul'):
        for li in ul.find_all('li'):
            a = li.find('a')
            if not a:
                continue
            
            href = a.get('href', '')
            title = a.get_text(strip=True)
            if not href or not title:
                continue
            
            # Make absolute URL
            if href.startswith('//'):
                href = 'https:' + href
            elif href.startswith('/'):
                href = BASE_URL + href
            elif not href.startswith('http'):
                href = BASE_URL + '/' + href.lstrip('/')
            
            # Date from span
            span = li.find('span')
            date_str = span.get_text(strip=True) if span else ''
            
            records.append({
                "title": title,
                "url": href,
                "date": date_str,
            })
    
    return records

def fetch_detail(url):
    """Fetch detail page, return (content_html, date_str)"""
    try:
        resp = session.get(url, timeout=30)
        resp.encoding = 'utf-8'
    except Exception:
        return "", ""
    
    soup = BeautifulSoup(resp.text, 'html.parser')
    
    # Find content area - try main-body-wrap or the article body
    content_div = soup.find('div', class_='main-body-wrap')
    if not content_div:
        content_div = soup.find('div', class_=lambda c: c and ('content' in str(c).lower() or 'article' in str(c).lower()))
    
    content_html = ""
    if content_div:
        content_html = str(content_div)
    
    # Date from meta tag
    date_str = ""
    meta = soup.find('meta', attrs={'name': 'PubDate'})
    if meta and meta.get('content'):
        date_str = meta['content'].strip()[:10]
    
    return content_html, date_str

def insert_to_db(records, conn):
    """Insert records into gov_raw"""
    cursor = conn.cursor()
    inserted = 0
    skipped = 0
    
    for rec in records:
        if rec["date"] and rec["date"] < THREE_YEARS_AGO:
            skipped += 1
            continue
        
        try:
            cursor.execute("""
                INSERT OR IGNORE INTO gov_raw 
                (page_url, source_url, title, summary, content, publish_date, site_name)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (
                rec["url"],
                BASE_URL,
                rec["title"],
                "",
                rec.get("content", ""),
                rec.get("date", ""),
                SITE_NAME,
            ))
            if cursor.rowcount > 0:
                inserted += 1
        except Exception as e:
            print(f"  DB error: {e}")
    
    conn.commit()
    return inserted, skipped

def get_pagination_info(html_text):
    """Extract total pages from createPageHTML"""
    m = re.search(r'createPageHTML\((\d+),\s*(\d+)', html_text)
    if m:
        total_records = int(m.group(1))
        total_pages = int(m.group(2))
        return total_pages, total_records
    return 1, 0

def main():
    incremental = 'incremental' in sys.argv or sys.argv[-1] == '1'
    
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    
    if incremental:
        pages_to_crawl = 1
        print("Mode: incremental (page 1 only)")
    else:
        pages_to_crawl = 5
    
    # Fetch page 1
    url1 = f"{BASE_URL}{LIST_PATH}/index.html"
    print(f"Fetching page 1...")
    resp = session.get(url1, timeout=30)
    resp.encoding = 'utf-8'
    
    all_records = parse_list_page(resp.text)
    print(f"  Page 1: {len(all_records)} records")
    
    if not incremental:
        total_pages, total_records = get_pagination_info(resp.text)
        if total_pages > 0 and total_pages < pages_to_crawl:
            pages_to_crawl = total_pages
        print(f"  Total: {total_records} records, {total_pages} pages, crawling {pages_to_crawl}")
    
    # Fetch remaining pages
    for page in range(2, pages_to_crawl + 1):
        page_url = f"{BASE_URL}{LIST_PATH}/index_{page}.html"
        try:
            resp = session.get(page_url, timeout=30)
            resp.encoding = 'utf-8'
            if resp.status_code != 200:
                print(f"  Page {page}: HTTP {resp.status_code} - stopping")
                break
            records = parse_list_page(resp.text)
            if not records:
                print(f"  Page {page}: empty - stopping")
                break
            print(f"  Page {page}: {len(records)} records")
            all_records.extend(records)
            time.sleep(0.3)
        except Exception as e:
            print(f"  Page {page} error: {e}")
            break
    
    print(f"\nTotal records collected: {len(all_records)}")
    
    # Fetch detail pages
    print("Fetching detail pages...")
    for i, rec in enumerate(all_records):
        if i > 0 and i % 10 == 0:
            print(f"  {i}/{len(all_records)}", flush=True)
        
        content, date_from_detail = fetch_detail(rec["url"])
        if content:
            rec["content"] = content
        if date_from_detail:
            rec["date"] = date_from_detail
    
    # Insert to DB
    inserted, skipped = insert_to_db(all_records, conn)
    conn.close()
    
    print(f"\n=== SUMMARY ===")
    print(f"Inserted: {inserted}, Skipped (old): {skipped}")

if __name__ == "__main__":
    main()