#!/usr/bin/env python3
"""Crawler for www.aspd.gov.cn - 意见征集 (TRS IGI API)"""

import requests
import sqlite3
import json
import time
import sys
import urllib3
from datetime import datetime, timedelta

API_URL = "https://www.aspd.gov.cn/IGI/opinion/web/list"
BASE_URL = "https://www.aspd.gov.cn"
DB_PATH = "/root/search.db"
MAX_PAGES = 5  # latest ~75 records
PAGE_SIZE = 15
SITE_NAME = "www.aspd.gov.cn-意见征集"
CUTOFF_DATE = (datetime.now() - timedelta(days=3 * 365)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Referer": "https://www.aspd.gov.cn/hdjl/yjzj/",
}

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


def fetch_page(page_index):
    """Fetch one page of data from the TRS IGI API."""
    params = {
        "pageIndex": page_index,
        "pageSize": PAGE_SIZE,
        "siteId": 502345,
        "orderby": "endTime_desc",
    }
    resp = requests.get(API_URL, params=params, headers=HEADERS, verify=False, timeout=20)
    resp.encoding = "utf-8"
    data = resp.json()
    items = data.get("datas", {}).get("data", [])
    page_info = data.get("datas", {}).get("pageInfo", {})
    return items, page_info


def ms_to_date(ms_str):
    """Convert millisecond timestamp string to YYYY-MM-DD."""
    try:
        ms = int(ms_str)
        return datetime.fromtimestamp(ms / 1000).strftime("%Y-%m-%d")
    except:
        return ""


def main():
    all_items = []
    
    print(f"[{SITE_NAME}] Fetching pages...")
    for page in range(1, MAX_PAGES + 1):
        try:
            items, page_info = fetch_page(page)
            print(f"  Page {page}: {len(items)} items")
            all_items.extend(items)
        except Exception as e:
            print(f"  [ERR] Page {page}: {e}")
        time.sleep(0.5)
    
    # Get total from page_info if available
    total_available = page_info.get("totalResults", 0) if page_info else 0
    print(f"\n[{SITE_NAME}] Total fetched: {len(all_items)} (of {total_available} available)")
    
    # Filter to 3-year window by beginTime
    filtered = []
    cutoff_ts = int((datetime.now() - timedelta(days=3 * 365)).timestamp() * 1000)
    
    for item in all_items:
        try:
            begin_ms = int(item.get("beginTime", 0))
            if begin_ms >= cutoff_ts:
                filtered.append(item)
        except:
            pass
    
    print(f"[{SITE_NAME}] Items within 3 years: {len(filtered)}")
    
    # Connect DB
    conn = sqlite3.connect(DB_PATH)
    c = conn.cursor()
    
    # Check existing
    c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    existing_count = c.fetchone()[0]
    print(f"[{SITE_NAME}] Existing records: {existing_count}")
    
    # Process each item
    new_count = 0
    skip_count = 0
    error_count = 0
    
    for item in filtered:
        page_url = item.get("publishUrl") or item.get("exLink") or ""
        if not page_url:
            skip_count += 1
            continue
        
        # Check if already exists
        c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url = ?", (page_url,))
        if c.fetchone()[0] > 0:
            skip_count += 1
            continue
        
        title = (item.get("theme") or "").strip()
        content_html = (item.get("htmlContent") or "").strip()
        begin_date = ms_to_date(item.get("beginTime", "0"))
        
        if not content_html:
            print(f"  [SKIP] Empty content: {title[:40]}")
            skip_count += 1
            continue
        
        if not title:
            title = "意见征集"
        
        # Clean content - extract inner text from TRS_UEDITOR div
        # The htmlContent already has full HTML, just clean it up
        # Remove the enclosing div.view.TRS_UEDITOR wrapper if present for cleaner storage
        if content_html.startswith('<div class="view TRS_UEDITOR'):
            # Strip the outer wrapper, keep inner content
            import re
            m = re.search(r'<div class="view TRS_UEDITOR[^>]*>(.*)</div>\s*$', content_html, re.DOTALL)
            if m:
                content_html = m.group(1).strip()
        
        # Extract images/attachments from content
        attachments = []
        soup_html = content_html
        for ext in ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.zip', '.rar']:
            for a_tag in content_html.split('<a '):
                if ext in a_tag.lower() and 'href=' in a_tag:
                    import re
                    href_m = re.search(r'href=[\'"]([^\'"]+)', a_tag)
                    if href_m:
                        href = href_m.group(1)
                        if href.startswith('http'):
                            attachments.append(href)
                        elif href.startswith('/'):
                            attachments.append(BASE_URL + href)
        
        attachments_json = json.dumps(attachments, ensure_ascii=False) if attachments else ""
        
        # Find publish_time from item
        pub_date = begin_date
        
        # Insert
        c.execute(
            "INSERT OR IGNORE INTO gov_raw (site_name, title, content, page_url, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)",
            (SITE_NAME, title, content_html, page_url, pub_date, "" if len(title) <= 200 else title[:200])
        )
        conn.commit()
        new_count += 1
    
    print(f"\n[{SITE_NAME}] Done: {new_count} new, {skip_count} skipped, {error_count} errors")
    
    # Summary
    c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name = ?", (SITE_NAME,))
    total = c.fetchone()[0]
    print(f"[{SITE_NAME}] Total in DB: {total}")
    
    conn.close()
    
    summary = {
        "site": SITE_NAME,
        "new": new_count,
        "skipped": skip_count,
        "errors": error_count,
        "total": total,
        "pages_scanned": MAX_PAGES,
    }
    print(f"\nJSON_OUTPUT:{json.dumps(summary, ensure_ascii=False)}")


if __name__ == "__main__":
    main()