#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""莘县人民政府 - 通知公告爬虫"""
import sys, re, os, json, argparse
sys.path.insert(0, "/root")
sys.path.insert(0, "/root/gov_crawler")
from urllib.parse import urljoin
from datetime import datetime
import urllib.request, ssl

SITE_NAME = "莘县通知公告"
BASE_URL = "http://www.sdsx.gov.cn"
LIST_URL = "http://www.sdsx.gov.cn/channel_t_273_15736/"

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE


def fetch_page(page_url):
    req = urllib.request.Request(page_url, headers={"User-Agent": "Mozilla/5.0"})
    resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
    return resp.read().decode("utf-8")


def parse_list_page(html):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")
    items = []

    for a in soup.find_all("a", href=re.compile(r"doc_")):
        href = a["href"]
        title = a.get_text(strip=True)
        if not title or len(title) < 5:
            continue
        # Remove date suffix from title (appended in HTML)
        title_clean = re.sub(r"\d{4}-\d{2}-\d{2}$", "", title).strip()

        if href.startswith("/"):
            href = BASE_URL + href
        elif not href.startswith("http"):
            href = BASE_URL + "/" + href.lstrip("/")

        # Find date - look in parent span or li text
        date_str = ""
        parent = a.find_parent()
        if parent:
            full = parent.get_text(" ", strip=True)
            m = re.search(r"(\d{4}-\d{2}-\d{2})", full)
            if m:
                # Check if it's the date from the title
                if m.group(1) != title[-10:] or len(title) < 15:
                    date_str = m.group(1)

        items.append({
            "url": href,
            "title": title_clean,
            "date": date_str,
        })
    return items


def fetch_detail(url):
    try:
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
        html = resp.read().decode("utf-8", errors="replace")
    except Exception:
        return "", "", [], ""

    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")

    # Title
    title = ""
    m = re.search(r"<title>(.*?)</title>", html)
    if m:
        t = m.group(1).replace("|莘县人民政府", "").replace("| 莘县人民政府", "").strip()
        if t:
            title = t

    # Date
    date_str = ""
    for pat in [
        r"发布时间[：:]\s*(\d{4}[-/.]\d{1,2}[-/.]\d{1,2})",
        r"(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})",
        r"(\d{4}-\d{2}-\d{2})",
    ]:
        m = re.search(pat, html)
        if m:
            date_str = m.group(1).replace("/", "-").replace(".", "-")[:10]
            break

    # Content
    content_div = soup.find("div", class_="sxNewsDetailContent")
    if not content_div:
        content_div = soup.find("div", class_=lambda x: x and "content" in str(x).lower())

    content = ""
    attachments = []

    if content_div:
        for tag in content_div.find_all(["script", "style"]):
            tag.decompose()

        parts = []
        for el in content_div.find_all(["p", "table", "img"]):
            if el.name == "p" and not el.find_parent("table"):
                txt = el.get_text(" ", strip=True)
                txt = re.sub(r"\s+", " ", txt)
                txt = re.sub(r"(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])", "", txt)
                if txt:
                    parts.append(txt)
            elif el.name == "table":
                rows = el.find_all("tr")
                md_rows = []
                col_count = None
                for i, tr in enumerate(rows):
                    cells = tr.find_all(["td", "th"])
                    if col_count is None:
                        col_count = len(cells)
                    row_cells = [re.sub(r"\s+", " ", re.sub(r"(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])", "", c.get_text(" ", strip=True))) for c in cells]
                    md_rows.append("| " + " | ".join(row_cells) + " |")
                    if i == 0 and col_count and col_count > 1:
                        md_rows.append("| " + " | ".join(["---"] * col_count) + " |")
                parts.append("\n".join(md_rows))
            elif el.name == "img":
                src = el.get("src", "")
                alt = el.get("alt", "图片")
                if src and not src.startswith("data:"):
                    parts.append("![" + alt + "](" + urljoin(url, src) + ")")

        content = "\n\n".join(parts)

        for a_tag in content_div.find_all("a", href=True):
            href = a_tag["href"]
            if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", href, re.I):
                full_url = urljoin(url, href)
                name = a_tag.get_text(strip=True) or os.path.basename(href)
                attachments.append({"name": name, "url": full_url})

    if attachments:
        if content:
            content += "\n\n"
        att_lines = ["[" + att["name"] + "](" + att["url"] + ")" for att in attachments]
        content += "\n".join(att_lines)

    return title, date_str, attachments, content


def main():
    parser = argparse.ArgumentParser(description="莘县通知公告爬虫")
    parser.add_argument("--pages", type=int, default=5)
    args = parser.parse_args()

    pages = args.pages
    print("=== " + SITE_NAME + " ===")
    print("  pages: " + str(pages))

    all_items = []
    for page in range(1, pages + 1):
        url = LIST_URL + "?page=" + str(page)
        print("  [Page " + str(page) + "/" + str(pages) + "] fetching...", end=" ")
        try:
            html = fetch_page(url)
            items = parse_list_page(html)
            print(str(len(items)) + " items")
            all_items.extend(items)
        except Exception as e:
            print("ERROR: " + str(e))
            break

    print("\n=== Total " + str(len(all_items)) + " items ===")

    seen = set()
    unique = []
    for item in all_items:
        if item["url"] not in seen:
            seen.add(item["url"])
            unique.append(item)
    print("Unique: " + str(len(unique)))

    db_items = []
    for i, item in enumerate(unique):
        print("  [" + str(i + 1) + "/" + str(len(unique)) + "] " + item["title"][:40] + "...", end=" ")
        title, date_str, attachments, content = fetch_detail(item["url"])
        use_title = title or item["title"]
        use_date = date_str or item.get("date", "")
        summary = re.sub(r"\s+", "", content)[:200] if content else ""

        db_items.append({
            "site_name": SITE_NAME,
            "source_url": item["url"],
            "url": item["url"],
            "title": use_title,
            "pub_date": use_date,
            "summary": summary,
            "content": content,
            "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
        })
        print("ok [" + use_date + "]")

    try:
        from crawler_lib import push_to_searchdb
        push_to_searchdb(db_items, SITE_NAME)
        print("\nOK: " + str(len(db_items)))
    except Exception as e:
        print("FAIL: " + str(e))
        import traceback
        traceback.print_exc()


if __name__ == "__main__":
    main()