#!/usr/bin/env python3
"""
宿迁市生态环境局 - 建设项目信息公开
https://sthj.suqian.gov.cn/shbj/jsxm/xxgk_list.shtml
Pagination: static (createPageHTML), 173 pages, 16/page
Detail: UCAPCONTENT content, meta ArticleTitle, meta PubDate
注意: DNS 已写入 /etc/hosts (116.198.151.205)
"""

import urllib.request, json, re, sys, ssl, time, os
from bs4 import BeautifulSoup

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE_URL = "https://sthj.suqian.gov.cn"
LIST_PATH = "/shbj/jsxm/xxgk_list"
TOTAL_PAGES = 173
SITE_NAME = "宿迁市生态环境局-建设项目"

# === 自动页数控制 ===
_MAX_PG = None
for i, a in enumerate(sys.argv):
    if a == '--pages' and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break
if _MAX_PG is not None:
    print("[AutoPg] max_pages=" + str(_MAX_PG))

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}


def fetch_url(url, timeout=30):
    req = urllib.request.Request(url, headers=HEADERS)
    try:
        resp = urllib.request.urlopen(req, timeout=timeout, context=ssl_ctx)
        return resp.read().decode("utf-8", errors="replace")
    except Exception as e:
        print(f"  [WARN] {url[:60]}: {e}")
        return None


def extract_list_items(html):
    items = []
    m = re.search(r'<ul\s+class="listContent">(.*?)</ul>', html, re.DOTALL)
    if not m:
        return items

    lis = re.findall(r'<li>(.*?)</li>', m.group(1), re.DOTALL)
    for li in lis:
        am = re.search(r'<a\s+href="([^"]*)"', li)
        if not am:
            continue

        href = am.group(1).strip()
        if href.startswith("/"):
            url = BASE_URL + href
        elif not href.startswith("http"):
            url = BASE_URL + "/shbj/jsxm/" + href
        else:
            url = href

        title = None
        tm = re.search(r'title="([^"]*)"', li)
        if tm:
            title = tm.group(1).strip()
        else:
            tm = re.search(r"title='([^']*)'", li)
            if tm:
                title = tm.group(1).strip()

        dm = re.search(r'<span>(\d{4}-\d{2}-\d{2})</span>', li)
        publish_date = dm.group(1).strip() if dm else ""

        if title and url:
            items.append({"url": url, "title": title, "publish_date": publish_date})

    return items


def resolve_url(page_url, src):
    if src.startswith("http"):
        return src
    if src.startswith("/"):
        return BASE_URL + src
    return page_url.rsplit("/", 1)[0] + "/" + src


def extract_content(raw, page_url):
    soup = BeautifulSoup(raw, "html.parser")
    for t in soup.find_all(["script", "style"]):
        t.decompose()

    parts = []
    for child in soup.children:
        if not child.name:
            continue
        if child.name == "table" and len(child.get_text(strip=True)) >= 15:
            parts.append(str(child))
        elif child.name == "img":
            src = child.get("src", "")
            if src:
                parts.append("![](" + resolve_url(page_url, src) + ")")
        elif child.name in ["p", "div", "span", "section"]:
            # Skip if nested inside a table (already handled as raw HTML)
            if child.find_parent("table"):
                continue
            text = child.get_text(" ", strip=True)
            # Remove span-fragment spaces between Chinese chars
            text = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=[\u4e00-\u9fff])', '', text)
            text = re.sub(r'(?<=[\u4e00-\u9fff])\s+(?=\d)', '', text)
            text = re.sub(r'(?<=\d)\s+(?=[\u4e00-\u9fff])', '', text)
            if text:
                parts.append(text)

    result = "\n\n".join(parts)
    result = re.sub(r"\n{3,}", "\n\n", result)
    return result


def extract_attachments(html, page_url):
    atts = []
    p = re.compile(
        r'<a\s+[^>]*href="([^"]*\.(?:doc|docx|pdf|xls|xlsx|zip|rar))"[^>]*>([^<]*)</a>', re.I
    )
    for m in p.finditer(html):
        atts.append({"name": m.group(2).strip(), "url": resolve_url(page_url, m.group(1).strip())})
    return atts


def fetch_detail(detail_url):
    html = fetch_url(detail_url)
    if not html:
        return None, None, None, []

    title = None
    m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
    if m:
        title = re.sub(r"\s+", " ", m.group(1)).strip()

    publish_date = None
    m = re.search(r'<meta\s+name="PubDate"\s+content="(\d{4}-\d{2}-\d{2})', html)
    if m:
        publish_date = m.group(1)

    content = None
    m = re.search(r"<UCAPCONTENT>(.*?)</UCAPCONTENT>", html, re.DOTALL)
    if m:
        content = extract_content(m.group(1), detail_url)

    attachments = extract_attachments(html, detail_url)
    return title, publish_date, content, attachments


def main():
    pages_to_fetch = min(TOTAL_PAGES, _MAX_PG) if _MAX_PG else TOTAL_PAGES

    print(f"=== {SITE_NAME} ===")
    print(f"  总页数: {TOTAL_PAGES}, 本次: {pages_to_fetch}")

    all_records = []
    empty = total = 0
    last_page = 0

    for page in range(1, pages_to_fetch + 1):
        list_url = f"{LIST_PATH}.shtml" if page == 1 else f"{LIST_PATH}_{page}.shtml"
        print(f"  第 {page}/{pages_to_fetch} 页: {BASE_URL + list_url}")

        html = fetch_url(BASE_URL + list_url)
        if not html:
            print(f"    无法获取, 停止")
            break

        items = extract_list_items(html)
        if not items:
            if page > 1:
                print(f"    空列表, 停止")
                break
            print(f"    空列表")
            continue

        last_page = page
        total += len(items)
        print(f"    找到 {len(items)} 条")

        for item in items:
            time.sleep(0.3)
            title, publish_date, content, attachments = fetch_detail(item["url"])
            if title:
                item["title"] = title
            if publish_date:
                item["publish_date"] = publish_date
            item["content"] = content or ""
            item["attachments"] = attachments
            if not content or len(content.strip()) < 20:
                empty += 1
            all_records.append(item)

    print(f"\n  列表总计: {total}, 入库准备: {len(all_records)}, 空正文: {empty}")

    if all_records:
        # 转换格式为 push_to_searchdb 所需
        results = []
        for item in all_records:
            summary = re.sub(r'<[^>]+>', '', item.get("content", ""))[:200].strip()
            if not summary:
                summary = item["title"][:200]
            results.append({
                "site_name": SITE_NAME,
                "title": item["title"],
                "url": item["url"],
                "content": item.get("content", ""),
                "summary": summary,
                "pub_date": item.get("publish_date", ""),
                "source_url": item["url"],
            })
        push_to_searchdb(results, "suqian_jsxm")

    print("完成")


if __name__ == "__main__":
    main()