#!/usr/bin/env python3
"""Crawler for www.fjql.gov.cn - 公告公示
TRS WCM API for list + detail page for full HTML content
"""
import requests
import sqlite3
import json
import time
import re
import urllib3
from datetime import datetime, timedelta

API_URL = "https://www.fjql.gov.cn/fjdzapp/data"
DB_PATH = "/root/search.db"
MAX_PAGES = 5
ITEMS_PER_PAGE = 15
SITE_NAME = "www.fjql.gov.cn-公告公示"
CUTOFF_DATE = (datetime.now() - timedelta(days=3 * 365)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Accept": "application/json, text/plain, */*",
    "Content-Type": "application/json",
}
DETAIL_HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


def fetch_page(page):
    """Fetch one page from TRS WCM API (POST JSON)."""
    payload = {
        "channelid": 100000,
        "classsql": "chnlid=6272",
        "sortfield": "-docorderpri,-docreltime",
        "prepage": ITEMS_PER_PAGE,
        "page": page,
    }
    resp = requests.post(API_URL, json=payload, headers=HEADERS, verify=False, timeout=20)
    data = resp.json()
    if data.get("error"):
        print(f"  [ERR] API error: {data.get('msg')}")
        return [], 0
    return data.get("data", []), data.get("recordCount", 0)


def fetch_detail_content(url):
    """Fetch detail page and extract content from div.TRS_Editor using BeautifulSoup."""
    try:
        resp = requests.get(url, headers=DETAIL_HEADERS, verify=False, timeout=20)
        resp.encoding = "utf-8"
        html = resp.text
        
        from bs4 import BeautifulSoup
        soup = BeautifulSoup(html, "html.parser")
        
        # Find TRS_Editor div (supports class="TRS_Editor" and class=TRS_Editor)
        editor = soup.find("div", class_="TRS_Editor")
        if editor:
            return str(editor)
        
        # Fallback by searching class attribute
        for div in soup.find_all("div"):
            cls = div.get("class", [])
            if cls and any("TRS_Editor" in c for c in cls):
                return str(div)
        
        # Last fallback: smgb-article
        article = soup.find("div", class_="smgb-article")
        if article:
            # Skip the video placeholder, get the main content
            inner = article.find("div", class_="TRS_Editor") or article.find("div", recursive=False)
            if inner:
                return str(inner)
            return str(article)
        
        return ""
    except Exception as e:
        print(f"  [ERR] detail fetch: {url[:60]}: {e}")
        return ""


def convert_date(datestr):
    try:
        dt = datetime.strptime(datestr.strip(), "%Y.%m.%d %H:%M:%S")
        return dt.strftime("%Y-%m-%d")
    except:
        return datestr[:10].replace('.', '-')


def main():
    all_items = []
    total_records = 0
    
    print(f"[{SITE_NAME}] Fetching {MAX_PAGES} pages from API...")
    for page in range(1, MAX_PAGES + 1):
        try:
            items, total = fetch_page(page)
            if not total_records and total:
                total_records = total
            print(f"  Page {page}: {len(items)} items")
            all_items.extend(items)
        except Exception as e:
            print(f"  [ERR] Page {page}: {e}")
        time.sleep(0.5)
    
    # Dedup by URL
    seen = set()
    unique_items = []
    for item in all_items:
        url = item.get("docpuburl") or item.get("url") or ""
        if url and url not in seen:
            seen.add(url)
            unique_items.append(item)
    
    print(f"\n[{SITE_NAME}] Total unique: {len(unique_items)}")
    
    # Filter by 3 years
    filtered = []
    for item in unique_items:
        reltime = (item.get("docreltime") or "").strip()
        if reltime:
            d = convert_date(reltime)
            if d >= CUTOFF_DATE:
                filtered.append(item)
    
    print(f"[{SITE_NAME}] In 3 years: {len(filtered)}")
    
    # Connect DB
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    
    # Clean old records first
    c.execute("DELETE FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    c.execute("DELETE FROM gov_search WHERE site_name=?", (SITE_NAME,))
    conn.commit()
    print(f"[{SITE_NAME}] Cleaned old records")
    
    # Process each item - fetch detail page for full content
    new_count = 0
    error_count = 0
    
    for i, item in enumerate(filtered):
        page_url = item.get("docpuburl") or item.get("url") or ""
        if not page_url:
            continue
        
        title = (item.get("doctitle") or "").strip()
        date_part = convert_date(item.get("docreltime", ""))
        
        # Fetch detail page for full HTML content
        print(f"  [{i+1}/{len(filtered)}] Fetching detail: {title[:40]}...")
        content_html = fetch_detail_content(page_url)
        
        if not content_html or len(content_html) < 20:
            print(f"    [WARN] Empty/short content, retrying once...")
            time.sleep(2)
            content_html = fetch_detail_content(page_url)
        
        if not content_html or len(content_html) < 20:
            print(f"    [SKIP] No content after retry")
            error_count += 1
            continue
        
        c.execute(
            "INSERT OR IGNORE INTO gov_raw (site_name, title, content, page_url, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)",
            (SITE_NAME, title, content_html, page_url, date_part, title[:200])
        )
        conn.commit()
        new_count += 1
        time.sleep(0.8)  # Be gentle to the server
    
    print(f"\n[{SITE_NAME}] Done: {new_count} new, {error_count} errors")
    
    c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    total = c.fetchone()[0]
    print(f"[{SITE_NAME}] Total in DB: {total}")
    
    # Show a sample
    if total > 0:
        c.execute("SELECT title, length(content) FROM gov_raw WHERE site_name=? LIMIT 3", (SITE_NAME,))
        for t, l in c.fetchall():
            print(f"  {t[:40]}: {l} chars")
    
    conn.close()
    
    summary = {"site": SITE_NAME, "new": new_count, "errors": error_count, "total": total}
    print(f"\nJSON_OUTPUT:{json.dumps(summary, ensure_ascii=False)}")


if __name__ == "__main__":
    main()