#!/usr/bin/env python3
"""Crawl aax.gov.cn - 昂昂溪区公示公告 (API + concurrent detail extraction)"""
import sys, re, json, sqlite3, urllib.request, urllib.error, traceback, ssl
from datetime import datetime
from urllib.parse import urljoin
from concurrent.futures import ThreadPoolExecutor, as_completed

ssl._create_default_https_context = ssl._create_unverified_context

BASE = "https://www.aax.gov.cn"
API_URL = "https://www.aax.gov.cn/search/1cc659753cd94d769c4692ff6fa7ca61"
SITE_NAME = "aax_tzgg"
DB_PATH = "/root/search.db"
PAGE_SIZE = 20
MAX_PAGES = 5

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": BASE + "/aax/c102254/list.shtml",
}


def fetch_api(page):
    url = f"{API_URL}?_isAgg=false&_isJson=true&_pageSize={PAGE_SIZE}&_template=index&_rangeTimeGte=&_channelName=&page={page}"
    req = urllib.request.Request(url, headers=headers)
    resp = urllib.request.urlopen(req, timeout=15)
    data = json.loads(resp.read().decode("utf-8", errors="replace"))
    return data.get("data", {})


def fetch_detail(url):
    """Fetch detail page and extract attachment links."""
    attachments = []
    try:
        req = urllib.request.Request(url, headers=headers)
        resp = urllib.request.urlopen(req, timeout=15)
        html = resp.read().decode("utf-8", errors="replace")

        ext_pattern = r'(?:pdf|doc|docx|xls|xlsx|zip|rar)'
        href_pattern = r'href=["\']([^"\']+\.' + ext_pattern + r')["\']'
        for href in re.findall(href_pattern, html, re.I):
            full_url = urljoin(url, href)
            a_match = re.search(r'<a[^>]*href=["\'][^"\']*' + re.escape(href) + r'["\'][^>]*>(.*?)</a>', html, re.I | re.DOTALL)
            text = ""
            if a_match:
                text = re.sub(r'<[^>]+>', '', a_match.group(1)).strip()
            if not text:
                text = href.split('/')[-1].rsplit('.', 1)[0]
            attachments.append(f"[{text}]({full_url})")
    except Exception as e:
        pass  # Silently skip if detail page fails
    return attachments


def process_entry(entry):
    """Process a single entry: extract metadata + fetch detail attachments."""
    title = entry.get("title", "").strip()
    content = entry.get("content", "").strip()
    url = entry.get("url", "")
    if url and not url.startswith("http"):
        url = "http://" + url.lstrip("http://")
    pub_time = entry.get("publishedTimeStr", "")
    date = pub_time[:10] if pub_time else ""

    if not title:
        return None

    # Fetch detail page for attachments
    if url:
        attachments = fetch_detail(url)
        if attachments:
            content = content.rstrip()
            content += "\n\n**附件：**\n" + "\n".join(attachments)

    return {"title": title, "url": url, "date": date, "content": content}


def save_to_db(records):
    if not records:
        return 0
    conn = sqlite3.connect(DB_PATH)
    conn.execute("PRAGMA busy_timeout=30000")
    c = conn.cursor()
    count = 0
    for r in records:
        if c.execute("SELECT id FROM gov_raw WHERE source_url=?", (r["url"],)).fetchone():
            continue
        c.execute("""INSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, content)
            VALUES (?,?,?,?,?,?,?,?)""",
                  (SITE_NAME, r["url"], r["url"], r["title"], r["date"],
                   int(datetime.strptime(r["date"], "%Y-%m-%d").timestamp()) if r["date"] else 0,
                   r["content"][:200] if r["content"] else "", r["content"]))
        conn.commit()
        count += 1
    conn.close()
    return count


def main():
    print(f"=== Crawling {SITE_NAME} ===")

    # Fetch all API pages (fast - batch)
    data = fetch_api(1)
    total = data.get("total", 0)
    results = data.get("results", data.get("list", []))
    if not results:
        print("ERROR: No results from API")
        return
    print(f"Total: {total} articles")

    all_entries = list(results)
    for page in range(2, MAX_PAGES + 1):
        try:
            data = fetch_api(page)
            results = data.get("results", data.get("list", []))
            if not results:
                break
            all_entries.extend(results)
            if len(all_entries) >= total:
                break
        except Exception as e:
            print(f"Page {page}: error - {e}")
            break

    print(f"Total entries: {len(all_entries)}")

    # Deduplicate by URL
    seen = set()
    unique = [e for e in all_entries if not (e.get("url", "") in seen or seen.add(e.get("url", "")))]
    print(f"Unique: {len(unique)}")

    # Concurrently process entries (fetch detail + extract attachments)
    processed = []
    with ThreadPoolExecutor(max_workers=10) as executor:
        futures = {executor.submit(process_entry, entry): i for i, entry in enumerate(unique)}
        for future in as_completed(futures):
            idx = futures[future]
            result = future.result()
            if result:
                processed.append(result)
                print(f"[{idx+1}/{len(unique)}] {result['title'][:50]}")
            else:
                print(f"[{idx+1}/{len(unique)}] SKIP")

    # Sort by original order for deterministic output
    processed.sort(key=lambda r: unique.index(next(e for e in unique if e.get("url") == r["url"])) if any(e.get("url") == r["url"] for e in unique) else 0)

    # Save all
    saved = save_to_db(processed)
    print(f"\n=== Complete: {saved}/{len(processed)} new records ===")


if __name__ == "__main__":
    main()