#!/usr/bin/env python3
"""
Crawler: 湖州市生态环境局长兴分局 (zjcx.gov.cn JPAAS)
"""
import sys, os, json, re, time, requests
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from urllib.parse import urljoin

DB_PATH = os.getenv("SEARCH_DB", "/root/search.db")
SITE = "长兴县生态环境分局"
COLUMN = "信息公开"
BASE_URL = "https://www.zjcx.gov.cn"
API_URL = f"{BASE_URL}/api-gateway/jpaas-publish-server/front/page/build/unit"
API_PARAMS = {
    "parseType": "bulidstatic",
    "webId": "3645",
    "pageId": "8kDHhxYDq8SLXt9vSDiY9",
    "pageType": "column",
    "tagId": "\u7ec4\u914d\u5206\u7c7blist",
    "tplSetId": "QIrUapMnq9Avhahnnyp8M",
}
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
ATTACH_EXTS = ('.doc', '.docx', '.pdf', '.xls', '.xlsx', '.ppt', '.pptx', '.zip', '.rar', '.7z')


def log(msg):
    print(f"[{SITE}] {msg}", flush=True)


def fetch_list():
    """Fetch all items from JPAAS API (returns all sub-categories at once)."""
    r = requests.get(API_URL, params=API_PARAMS, headers=HEADERS, timeout=60, verify=False)
    data = r.json()
    html = data["data"]["html"]

    items = []
    seen_urls = set()
    for ul in re.finditer(r'<ul class="ajax-ul">(.*?)</ul>', html, re.DOTALL):
        ul_html = ul.group(1)
        for li in re.finditer(r'<li[^>]*class="cf"[^>]*>(.*?)</li>', ul_html, re.DOTALL):
            li_html = li.group(1)
            a_m = re.search(r'<a[^>]*href="([^"]*)"[^>]*title="([^"]*)"', li_html)
            d_m = re.search(r'<span class="fr">(\d{4}-\d{2}-\d{2})', li_html)
            if a_m:
                href = a_m.group(1)
                title = a_m.group(2).strip()
                date_str = d_m.group(1) if d_m else ""
                if href not in seen_urls:
                    seen_urls.add(href)
                    items.append({"title": title, "url": href, "date": date_str})
    return items


def fetch_detail(url):
    """Fetch detail page, extract content from div.zhengw."""
    try:
        r = requests.get(url, headers=HEADERS, timeout=30, verify=False)
        r.encoding = "utf-8"
        soup = BeautifulSoup(r.text, "html.parser")
    except Exception:
        return "", "", ""

    # Title from h1 or title
    title = ""
    h1 = soup.find("h1")
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        t = soup.find("title")
        if t:
            title = t.get_text(strip=True)

    # Date from meta PubDate
    pub_date = ""
    meta = soup.find("meta", attrs={"name": "PubDate"})
    if meta and meta.get("content"):
        pub_date = meta["content"][:10]

    # Content from div.zhengw
    content_html = ""
    zw = soup.find("div", class_="zhengw")
    if zw:
        content_html = str(zw)

    return title, content_html, pub_date


def extract_content(content_html):
    """Clean HTML content to text with attachment links."""
    if not content_html:
        return ""
    soup = BeautifulSoup(content_html, "html.parser")

    # Replace links with markdown
    for a_tag in soup.find_all("a", href=True):
        href = a_tag.get("href", "")
        text = a_tag.get_text(strip=True) or "附件"
        full_url = urljoin(BASE_URL, href)
        md = f"[{text}]({full_url})"
        a_tag.replace_with(md)

    # Unwrap inline tags
    for tag in soup.find_all(['span', 'b', 'strong', 'font', 'em', 'i', 'u', 's']):
        tag.unwrap()

    paragraphs = []
    for p in soup.find_all('p'):
        t = p.get_text(separator='', strip=True)
        if t:
            paragraphs.append(t)

    return '\n\n'.join(paragraphs) if paragraphs else content_html.strip()


def store_items(items):
    import sqlite3
    conn = sqlite3.connect(DB_PATH, timeout=60)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA busy_timeout=10000")
    c = conn.cursor()
    new_count = 0
    skip_count = 0
    for item in items:
        try:
            c.execute(
                "INSERT OR IGNORE INTO gov_raw "
                "(site_name, source_url, page_url, title, publish_date, content, summary, category)"
                " VALUES (?,?,?,?,?,?,?,?)",
                (
                    f"{SITE}-{COLUMN}",
                    item["url"], item["url"],
                    item["title"], item["date"],
                    item["content"],
                    (item["content"] or "")[:500],
                    COLUMN,
                )
            )
            if c.rowcount > 0:
                new_count += 1
            else:
                skip_count += 1
        except Exception as e:
            log(f"DB error: {e}")
            skip_count += 1
    conn.commit()
    conn.close()
    return new_count, skip_count


def crawl_all(months_back=36):
    cutoff = (datetime.now() - timedelta(days=months_back * 30)).strftime("%Y-%m-%d")
    log(f"Full crawl (cutoff: {cutoff})")

    items = fetch_list()
    log(f"List API returned {len(items)} items")

    # Filter by cutoff
    filtered = []
    for item in items:
        if item["date"] and item["date"] >= cutoff:
            filtered.append(item)
    log(f"After cutoff filter: {len(filtered)} items")

    # Fetch details
    for idx, item in enumerate(filtered, 1):
        title, content_html, date_str = fetch_detail(item["url"])
        if title:
            item["title"] = title
        if date_str:
            item["date"] = date_str
        item["content"] = extract_content(content_html)
        if idx % 10 == 0:
            log(f"  Progress: {idx}/{len(filtered)}")
        time.sleep(0.5)

    new, skip = store_items(filtered)
    log(f"Result: {new} new, {skip} skipped")


def crawl_incremental():
    cutoff = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d")
    log(f"Incremental (last 7 days)")

    items = fetch_list()
    filtered = [it for it in items if it["date"] and it["date"] >= cutoff]
    log(f"{len(filtered)} items since {cutoff}")

    for idx, item in enumerate(filtered, 1):
        title, content_html, date_str = fetch_detail(item["url"])
        if title: item["title"] = title
        if date_str: item["date"] = date_str
        item["content"] = extract_content(content_html)
        time.sleep(0.5)

    new, skip = store_items(filtered)
    log(f"Result: {new} new, {skip} skipped")


if __name__ == "__main__":
    mode = sys.argv[1] if len(sys.argv) > 1 else "incremental"
    if mode == "incremental":
        crawl_incremental()
    elif mode == "full":
        crawl_all()
    elif mode == "list":
        items = fetch_list()
        log(f"Total: {len(items)} items")
        for it in items[:5]:
            print(f"  {it['date']} | {it['title'][:50]}")