#!/usr/bin/env python3
"""Crawler for 察布查尔县人民政府网 - 公示公告
CMS: Visual SiteBuilder 9 (VSB), static .shtml pages
Site: www.xjcbcr.gov.cn
"""

import requests
import re
import json
import sys
import os
import tempfile
from datetime import datetime, timedelta
from bs4 import BeautifulSoup

BASE_URL = "http://www.xjcbcr.gov.cn"
CHANNEL = "c114052"
LIST_URL = f"{BASE_URL}/xjcbcr/{CHANNEL}/list.shtml"
SITE_NAME = "察布查尔县人民政府-公示公告"
CATEGORY = "公示公告"
INCREMENTAL_DAYS = 7

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TIMEOUT = 20
MAX_PAGES = 5

session = requests.Session()
session.headers.update(HEADERS)


def fetch(url, encoding="utf-8"):
    resp = session.get(url, timeout=TIMEOUT)
    resp.encoding = encoding
    return resp.text


def parse_list(html):
    """Parse list page, return list of (title, url, date) tuples."""
    soup = BeautifulSoup(html, "html.parser")
    items = []
    for tr in soup.find_all("tr", id=re.compile(r"line1457_")):
        a_tag = tr.find("a", class_="c1457")
        if not a_tag:
            continue
        href = a_tag.get("href", "").strip()
        # Use title attribute for complete title (no ellipsis!)
        title = a_tag.get("title") or a_tag.get_text(strip=True)
        if not title or not href:
            continue
        if not href.startswith("http"):
            href = BASE_URL + href
        date_span = tr.find("span", class_="timestyle1457")
        date = date_span.get_text(strip=True) if date_span else ""
        items.append((title, href, date))
    return items


def parse_detail(html, url):
    """Parse detail page, return dict with title, content, date, attachments."""
    soup = BeautifulSoup(html, "html.parser")

    # Title from h1
    h1 = soup.find("h1")
    title = h1.get_text(strip=True) if h1 else ""

    # Date from meta or text
    pub_date = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        pub_date = meta_date["content"][:10]

    # Content from vsb_content > v_news_content
    content_html = ""
    content_div = soup.find("div", id="vsb_content")
    if content_div:
        news_div = content_div.find("div", class_="v_news_content")
        if news_div:
            # Process: keep <table>, <p>, <br>; strip unwanted attributes
            for tag in news_div.find_all(True):
                # Keep only essential tags
                if tag.name in ("table", "tr", "td", "th", "thead", "tbody", "tfoot"):
                    continue  # keep as-is
                elif tag.name == "a":
                    # Keep attachment links
                    href = tag.get("href", "")
                    if href and not href.startswith("#") and not href.startswith("javascript"):
                        tag.attrs = {"href": href}
                        if tag.get("target"):
                            tag.attrs["target"] = tag["target"]
                    else:
                        tag.unwrap()
                elif tag.name in ("p", "div", "span", "br", "img", "strong", "em", "u", "h1", "h2", "h3", "h4", "h5", "h6"):
                    # Keep with minimal attributes
                    attrs_to_keep = {}
                    if tag.name == "img" and tag.get("src"):
                        attrs_to_keep["src"] = tag["src"]
                    if tag.name == "a" and tag.get("href"):
                        attrs_to_keep["href"] = tag["href"]
                    tag.attrs = attrs_to_keep
                else:
                    tag.unwrap()
            content_html = str(news_div)

    # Attachments from .xgfj .fujian
    attachments = []
    fj_div = soup.find("div", class_="xgfj")
    if fj_div:
        for a in fj_div.find_all("a", href=True):
            ahref = a["href"].strip()
            if any(ext in ahref.lower() for ext in [".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar"]):
                if not ahref.startswith("http"):
                    ahref = BASE_URL + ahref
                attachments.append({
                    "title": a.get_text(strip=True) or "附件",
                    "url": ahref
                })

    # Fallback: find attachment links in content
    if not attachments and content_html:
        content_soup = BeautifulSoup(content_html, "html.parser")
        for a in content_soup.find_all("a", href=True):
            ahref = a["href"].strip().lower()
            if any(ext in ahref for ext in [".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar"]):
                attachments.append({
                    "title": a.get_text(strip=True) or "附件",
                    "url": a["href"] if a["href"].startswith("http") else BASE_URL + a["href"]
                })

    return {
        "title": title,
        "date": pub_date,
        "content": content_html,
        "attachments": attachments,
        "source_url": url,
    }


def render_to_markdown(content_html, title, url):
    """Convert content HTML to readable markdown-like text with tables."""
    if not content_html or len(content_html.strip()) < 50:
        return f"[{title}]({url})"

    soup = BeautifulSoup(content_html, "html.parser")
    parts = []

    for el in soup.children:
        if el.name is None:
            text = el.strip()
            if text:
                parts.append(text)
        elif el.name == "p":
            text = el.get_text(" ", strip=True)
            # Check for images
            imgs = el.find_all("img")
            img_info = ""
            for img in imgs:
                src = img.get("src", "")
                if src:
                    if not src.startswith("http"):
                        src = BASE_URL + src
                    img_info += f"\n![图片]({src})"
            if text:
                parts.append(text)
            if img_info:
                parts.append(img_info)
            else:
                parts.append("")
        elif el.name == "table":
            # Convert to markdown table
            rows = el.find_all("tr")
            md_rows = []
            for ri, row in enumerate(rows):
                cells = row.find_all(["td", "th"])
                md_cells = []
                for cell in cells:
                    cell_text = cell.get_text(" ", strip=True).replace("|", "\\|")
                    md_cells.append(cell_text)
                md_rows.append("| " + " | ".join(md_cells) + " |")
                if ri == 0:
                    # Header separator
                    md_rows.append("| " + " | ".join(["---"] * len(md_cells)) + " |")
            parts.append("\n".join(md_rows))
        elif el.name == "br":
            parts.append("")
        elif el.name in ("h1", "h2", "h3", "h4", "h5", "h6"):
            text = el.get_text(" ", strip=True)
            if text:
                parts.append(f"\n## {text}")
        elif el.name == "img":
            src = el.get("src", "")
            if src:
                if not src.startswith("http"):
                    src = BASE_URL + src
                parts.append(f"\n![图片]({src})")
        elif el.name == "div":
            text = el.get_text(" ", strip=True)
            if text:
                parts.append(text)

    result = "\n\n".join(p for p in parts if p is not None).strip()
    result = re.sub(r"\n{3,}", "\n\n", result)

    if len(result.strip()) < 20:
        return f"[{title}]({url})"

    return result


def main():
    incremental = len(sys.argv) > 1 and sys.argv[1] == "incremental"
    print(f"[{SITE_NAME}] Starting crawl (incremental={incremental})")

    # Fetch list pages
    all_items = []
    for page in range(1, MAX_PAGES + 1):
        if page == 1:
            page_url = LIST_URL
        else:
            page_url = f"{BASE_URL}/xjcbcr/{CHANNEL}/list_{page}.shtml"

        print(f"  Fetching page {page}: {page_url}")
        try:
            html = fetch(page_url)
        except Exception as e:
            print(f"    ERROR: {e}")
            continue

        items = parse_list(html)
        print(f"    Found {len(items)} items")
        if not items:
            print(f"    No more items, stopping")
            break
        all_items.extend(items)

    print(f"\n  Total items from list: {len(all_items)}")

    if not all_items:
        print("  No items found, exiting")
        return

    # Incremental filter
    cutoff_date = None
    if incremental:
        cutoff_date = datetime.now() - timedelta(days=INCREMENTAL_DAYS)
        print(f"  Incremental mode: cutoff = {cutoff_date.date()}")
        filtered = []
        for title, url, date in all_items:
            if date:
                try:
                    item_date = datetime.strptime(date, "%Y-%m-%d")
                    if item_date >= cutoff_date:
                        filtered.append((title, url, date))
                except ValueError:
                    filtered.append((title, url, date))
            else:
                filtered.append((title, url, date))
        all_items = filtered
        print(f"  After incremental filter: {len(all_items)} items")

    # Fetch detail pages
    results = []
    for idx, (title, url, date_from_list) in enumerate(all_items):
        print(f"  [{idx+1}/{len(all_items)}] Fetching: {title[:40]}...")
        try:
            html = fetch(url)
        except Exception as e:
            print(f"    ERROR fetching detail: {e}")
            # Still save with list info
            results.append({
                "title": title,
                "date": date_from_list,
                "content": f"[{title}]({url})",
                "attachments": [],
                "source_url": url,
            })
            continue

        detail = parse_detail(html, url)

        # Fill missing title from list
        if not detail["title"]:
            detail["title"] = title
        if not detail["date"]:
            detail["date"] = date_from_list

        # Render content
        content = render_to_markdown(detail["content"], detail["title"], url)
        detail["content"] = content
        results.append(detail)

    print(f"\n  Total valid items: {len(results)}")
    if not results:
        print("  No items to import, exiting")
        return

    # Check for attachments in content
    for r in results:
        if r.get("attachments"):
            continue
        # Scan content for file links
        file_urls = re.findall(r'(https?://[^\s"]+\.(?:pdf|doc|docx|xls|xlsx|zip|rar))', r.get("content", ""), re.I)
        if file_urls:
            r["attachments"] = [{"title": "附件", "url": u} for u in file_urls]

    # Write JSONL
    tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False, dir="/tmp")
    label = f"{SITE_NAME} ({len(results)}条)"
    for r in results:
        line = {
            "title": r["title"],
            "page_url": r["source_url"],
            "content": r["content"],
            "publish_date": r["date"],
            "site_name": SITE_NAME,
            "source_url": r["source_url"],
            "attachments": r.get("attachments", []),
        }
        tmp.write(json.dumps(line, ensure_ascii=False) + "\n")
    tmp.close()
    print(f"  JSONL written: {tmp.name}")

    # Import
    ret = os.system(f"python3 /root/gov_crawler/import_jsonl.py {tmp.name} '{label}'")
    if ret == 0:
        print(f"  OK Import complete")
    else:
        print(f"  FAIL Import failed (exit={ret})")
        sys.exit(1)

    # Cleanup
    os.unlink(tmp.name)


if __name__ == "__main__":
    main()