#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""邵武市金塘工业园 - 园区动态爬虫"""
import sys, re, os, json, argparse
sys.path.insert(0, "/root")
sys.path.insert(0, "/root/gov_crawler")
from urllib.parse import urljoin
from datetime import datetime
import urllib.request, ssl

SITE_NAME = "邵武金塘工业园-园区动态"
BASE_URL = "http://www.fjjtchemzone.com"
LIST_URL = "http://www.fjjtchemzone.com/news/typeid/1.html"

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE

NL = chr(10)


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)

def fetch_page(page_url):
    req = urllib.request.Request(page_url, headers={"User-Agent": "Mozilla/5.0"})
    resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
    return resp.read().decode("utf-8")


def parse_list_page(html):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")
    for ul in soup.find_all("ul"):
        lis = ul.find_all("li")
        news_items = []
        for li in lis:
            a = li.find("a")
            if not a:
                continue
            href = a.get("href", "")
            if "new_detail" not in href:
                continue
            title = re.sub(r"\s+", " ", a.get_text(strip=True)).strip()
            span = li.find("span")
            date_str = span.get_text(strip=True) if span else ""
            news_items.append({
                "url": href if href.startswith("http") else urljoin(BASE_URL, href),
                "title": title,
                "date": date_str,
            })
        if len(news_items) >= 5:
            return news_items
    return []


def get_next_page(html):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")
    for a in soup.find_all("a"):
        if a.get_text(strip=True) == chr(19979) + chr(19968) + chr(39029) and a.get("href"):
            return urljoin(LIST_URL, a["href"])
    return None


def fetch_detail(url):
    try:
        req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
        resp = urllib.request.urlopen(req, timeout=30, context=ssl_ctx)
        html = resp.read().decode("utf-8", errors="replace")
    except Exception:
        return "", "", [], ""

    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html, "html.parser")

    title = ""
    h1 = soup.find("h1")
    if h1:
        title = re.sub(r"\s+", " ", h1.get_text(strip=True)).strip()
    if not title:
        m = re.search(r"<title>(.*?)</title>", html)
        if m:
            title = re.sub(r"\s+", " ", m.group(1)).strip()
            title = re.sub(r"^.*?--", "", title).strip()

    date_str = ""
    for pat in [
        chr(21457) + chr(24067) + chr(26102) + chr(38388) + "[：:]\\s*(\\d{4}[-/.]\\d{1,2}[-/.]\\d{1,2})",
        chr(21457) + chr(24067) + chr(26085) + chr(26399) + "[：:]\\s*(\\d{4}[-/.]\\d{1,2}[-/.]\\d{1,2})",
        "(\\d{4}-\\d{2}-\\d{2})",
    ]:
        m = re.search(pat, html)
        if m:
            date_str = m.group(1).replace("/", "-").replace(".", "-")
            break

    div = soup.find("div", class_="news_ny") or soup.find("div", class_="content")
    content = ""
    attachments = []

    if div:
        for tag in div.find_all(["script", "style"]):
            tag.decompose()

        parts = []
        for el in div.find_all(["p", "table", "img"]):
            if el.name == "p" and not el.find_parent("table") and not el.find("table") and not el.find("p"):
                txt = el.get_text(" ", strip=True)
                txt = re.sub(r"\s+", " ", txt)
                txt = re.sub(r"(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=[\u4e00-\u9fff])", "", txt)
                if txt:
                    parts.append(txt)
            if el.name == 'table':
                parts.append(html_table_to_html(el, url))
            elif el.name == "img":
                src = el.get("src", "")
                alt = el.get("alt", chr(22270) + chr(29255))
                if src and not src.startswith("data:"):
                    parts.append("![" + alt + "](" + urljoin(url, src) + ")")

        content = NL * 2 + (NL * 2).join(parts)

        for a_tag in div.find_all("a", href=True):
            href = a_tag["href"]
            if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar)$", href, re.I):
                full_url = urljoin(url, href)
                name = a_tag.get_text(strip=True) or os.path.basename(href)
                attachments.append({"name": name, "url": full_url})

    if attachments:
        if content:
            content += NL * 2
        att_lines = ["[" + att["name"] + "](" + att["url"] + ")" for att in attachments]
        content += NL.join(att_lines)

    return "", date_str, attachments, content


def main():
    parser = argparse.ArgumentParser(description="jingtang crawler")
    parser.add_argument("--pages", type=int, default=5)
    parser.add_argument("--incremental", action="store_true")
    args = parser.parse_args()

    pages = args.pages
    print("=== " + SITE_NAME + " ===")
    print("  pages: " + str(pages))

    url = LIST_URL
    page_num = 0
    all_items = []

    while url and page_num < pages:
        page_num += 1
        print("  [Page " + str(page_num) + "/" + str(pages) + "] fetching...", end=" ")
        try:
            html = fetch_page(url)
            items = parse_list_page(html)
            print(str(len(items)) + " items")
            all_items.extend(items)
            url = get_next_page(html)
        except Exception as e:
            print("ERROR: " + str(e))
            break

    print("\n=== Total " + str(len(all_items)) + " items ===")

    seen = set()
    unique = []
    for item in all_items:
        if item["url"] not in seen:
            seen.add(item["url"])
            unique.append(item)
    print("Unique: " + str(len(unique)))

    db_items = []
    for i, item in enumerate(unique):
        print("  [" + str(i + 1) + "/" + str(len(unique)) + "] " + item["title"][:40] + "...", end=" ")
        title, date_str, attachments, content = fetch_detail(item["url"])
        use_title = title or item["title"]
        use_date = date_str or item.get("date", "")
        summary = re.sub(r"\s+", "", content)[:200] if content else ""

        db_items.append({
            "site_name": SITE_NAME,
            "source_url": item["url"],
            "url": item["url"],
            "title": use_title,
            "pub_date": use_date,
            "summary": summary,
            "content": content,
            "attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
        })
        print("ok [" + use_date + "]")

    try:
        from crawler_lib import push_to_searchdb
        push_to_searchdb(db_items, SITE_NAME)
        print("\nOK: " + str(len(db_items)))
    except Exception as e:
        print("FAIL: " + str(e))
        import traceback
        traceback.print_exc()
        backup = "/root/jingtang_" + datetime.now().strftime("%Y%m%d_%H%M%S") + ".json"
        with open(backup, "w", encoding="utf-8") as f:
            json.dump(db_items, f, ensure_ascii=False, indent=2)
        print("  Saved to " + backup)


if __name__ == "__main__":
    main()