#!/usr/bin/env python3
"""Crawler for www.xam.gov.cn - 措施及实施情况 (Huilan CMS)"""

import requests
import sqlite3
import json
import time
import urllib3
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from datetime import datetime, timedelta

BASE_URL = "https://www.xam.gov.cn"
LIST_BASE = "https://www.xam.gov.cn/xam/zwgk73/zfxxgk31/fdzdgknr69/zdlyxx/sthj/csjssqk78"
DB_PATH = "/root/search.db"
MAX_PAGES = 5
SITE_NAME = "www.xam.gov.cn-措施及实施情况"
CUTOFF_DATE = (datetime.now() - timedelta(days=3 * 365)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
}

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)


def get_list_urls(num_pages):
    """Generate list page URLs."""
    urls = [f"{LIST_BASE}/index.html"]
    for i in range(2, num_pages + 1):
        urls.append(f"{LIST_BASE}/f48ec2f5-{i}.html")
    return urls


def parse_list(html):
    """Parse list page, return list of (title, url, date)."""
    soup = BeautifulSoup(html, "html.parser")
    ul = soup.find("ul", class_="xxgkList")
    if not ul:
        return []
    items = []
    for li in ul.find_all("li", recursive=False):
        a = li.find("a")
        span = li.find("span")
        if not a or not span:
            continue
        href = a.get("href", "")
        if not href or href.startswith("javascript"):
            continue
        full_url = urljoin(BASE_URL, href)
        # Use title attribute for full title (link text may be truncated)
        title = (a.get("title") or a.get_text(strip=True)).strip()
        date = span.get_text(strip=True)
        items.append((title, full_url, date))
    return items


def fetch_detail_content(url):
    """Fetch detail page and extract content from div#mainText."""
    try:
        resp = requests.get(url, headers=HEADERS, verify=False, timeout=20)
        resp.encoding = "utf-8"
        soup = BeautifulSoup(resp.text, "html.parser")
        
        # Content is in div#mainText
        main_text = soup.find("div", id="mainText")
        if main_text:
            return str(main_text)
        
        # Fallback: detail_txt
        txt = soup.find("div", class_="detail_txt")
        if txt:
            return str(txt)
        
        # Last fallback: detail_con
        con = soup.find("div", class_="detail_con")
        if con:
            return str(con)
        
        return ""
    except Exception as e:
        print(f"  [ERR] detail: {url[:60]}: {e}")
        return ""


def extract_attachments(html, page_url):
    """Extract attachment links from content HTML."""
    soup = BeautifulSoup(html, "html.parser")
    attachments = []
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if any(ext in href.lower() for ext in ['.pdf', '.doc', '.docx', '.xls', '.xlsx', '.zip', '.rar']):
            if href.startswith("http"):
                attachments.append(href)
            elif href.startswith("/"):
                attachments.append(urljoin(BASE_URL, href))
    return attachments


def main():
    page_urls = get_list_urls(MAX_PAGES)
    all_items = []
    
    print(f"[{SITE_NAME}] Scanning {len(page_urls)} pages...")
    for pu in page_urls:
        try:
            resp = requests.get(pu, headers=HEADERS, verify=False, timeout=20)
            resp.encoding = "utf-8"
            items = parse_list(resp.text)
            print(f"  {pu.split('/')[-1]} -> {len(items)} items")
            all_items.extend(items)
        except Exception as e:
            print(f"  [ERR] {pu}: {e}")
        time.sleep(0.5)
    
    # Dedup
    seen = set()
    unique = []
    for t, u, d in all_items:
        if u not in seen:
            seen.add(u)
            unique.append((t, u, d))
    
    print(f"\n[{SITE_NAME}] Total unique: {len(unique)}")
    
    # Filter 3 years
    filtered = [(t, u, d) for t, u, d in unique if d >= CUTOFF_DATE]
    print(f"[{SITE_NAME}] In 3 years (>{CUTOFF_DATE}): {len(filtered)}")
    
    # Connect DB
    conn = sqlite3.connect(DB_PATH, timeout=30)
    c = conn.cursor()
    
    # Check existing
    c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    existing = c.fetchone()[0]
    print(f"[{SITE_NAME}] Existing: {existing}")
    
    new_count = skip_count = 0
    
    for title, url, date in filtered:
        c.execute("SELECT COUNT(*) FROM gov_raw WHERE page_url=?", (url,))
        if c.fetchone()[0] > 0:
            skip_count += 1
            continue
        
        content_html = fetch_detail_content(url)
        
        if not content_html or len(content_html) < 20:
            print(f"  [WARN] Short content for {title[:30]}")
            time.sleep(2)
            content_html = fetch_detail_content(url)
        
        if not content_html or len(content_html) < 20:
            skip_count += 1
            continue
        
        c.execute(
            "INSERT OR IGNORE INTO gov_raw (site_name, title, content, page_url, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)",
            (SITE_NAME, title, content_html, url, date, title[:200])
        )
        conn.commit()
        new_count += 1
        time.sleep(0.8)
    
    print(f"\n[{SITE_NAME}] Done: {new_count} new, {skip_count} skipped")
    
    c.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (SITE_NAME,))
    total = c.fetchone()[0]
    print(f"[{SITE_NAME}] Total in DB: {total}")
    
    if total > 0:
        c.execute("SELECT title, length(content) FROM gov_raw WHERE site_name=? LIMIT 3", (SITE_NAME,))
        for t, l in c.fetchall():
            print(f"  {t[:35]}: {l} chars")
    
    conn.close()
    
    summary = {"site": SITE_NAME, "new": new_count, "skipped": skip_count, "total": total}
    print(f"\nJSON_OUTPUT:{json.dumps(summary, ensure_ascii=False)}")


if __name__ == "__main__":
    main()