#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
柳北区人民政府 - 建设项目环境影响评价审批 爬虫
http://www.lbq.gov.cn/xxgk/fdzdgknr/hjbh/jsxmhjyxpjsp/
TRS CMS, 单页静态列表(20条), cmsArticleContent正文
"""

import json, re, sys, urllib.request, urllib.parse, ssl
from html import unescape

ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE

SITE_NAME = "柳北区建设项目环境影响评价审批"
BASE_URL = "http://www.lbq.gov.cn"
LIST_URL = BASE_URL + "/xxgk/fdzdgknr/hjbh/jsxmhjyxpjsp/lists.shtml"
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}

def fetch_url(url, timeout=15):
    req = urllib.request.Request(url, headers=HEADERS)
    resp = urllib.request.urlopen(req, timeout=timeout, context=ssl_ctx)
    return resp.read().decode("utf-8", errors="replace")

def fetch_list():
    html = fetch_url(LIST_URL)
    items = []
    # Match PC list items
    for m in re.finditer(
        r'<li class="PC-SHOW">\s*<a href="([^"]+)"[^>]*title="([^"]*)"[^>]*>.*?</a><span class="time">\s*(\S+)',
        html, re.DOTALL
    ):
        href = m.group(1).strip()
        title = m.group(2).strip()
        date = m.group(3).strip()
        if href.startswith("./"):
            href = BASE_URL + "/xxgk/fdzdgknr/hjbh/jsxmhjyxpjsp" + href[1:]
        elif href.startswith("http"):
            pass
        else:
            href = BASE_URL + href if href.startswith("/") else BASE_URL + "/" + href
        items.append({"title": title, "url": href, "date": date})
    return items

def fetch_detail(url):
    try:
        html = fetch_url(url)
    except Exception as e:
        print(f"  [ERROR] {url}: {e}", file=sys.stderr)
        return {"content": "", "attachments": []}

    # Title from meta
    title = ""
    m = re.search(r'<meta\s+name="ArticleTitle"\s+content="([^"]*)"', html)
    if m:
        title = m.group(1).strip()

    # Date from meta
    pub_date = ""
    m = re.search(r'<meta\s+name="PubDate"\s+content="([^"]*)"', html)
    if m:
        pub_date = m.group(1).strip()[:10]

    # Content from cmsArticleContent
    content = ""
    idx = html.find('<div id="cmsArticleContent">')
    if idx >= 0:
        end = html.find('</div>', idx)
        if end < 0:
            end = idx + 20000
        inner = html[idx + len('<div id="cmsArticleContent">'):end]
        content = extract_content(inner)
    else:
        # Try trs_editor_view
        idx = html.find('class="trs_editor_view')
        if idx >= 0:
            # Find parent div
            start = html.rfind('<div', 0, idx)
            end = html.find('</div>', idx)
            if start >= 0 and end > start:
                inner = html[start:end]
                content = extract_content(inner)

    # Attachments
    attachments = []
    for m in re.finditer(r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>', html, re.DOTALL | re.I):
        href = m.group(1)
        atext = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        # Match .pdf, .doc, .docx or download/attached patterns
        if re.search(r'\.(pdf|doc|docx|xls|xlsx|zip|rar)$', href, re.I) or \
           re.search(r'(/download|attached|upload)', href, re.I):
            if href.startswith("/"):
                href = BASE_URL + href
            elif not href.startswith("http"):
                href = url.rsplit("/", 1)[0] + "/" + href
            attachments.append({"name": atext, "url": href})

    return {"content": content, "attachments": attachments, "date": pub_date, "title": title}

def extract_content(html):
    parts = []
    # Process p, table, img elements
    for m in re.finditer(r'<p[^>]*>(.*?)</p\s*>|<table[^>]*>(.*?)</table\s*>|<img[^>]*src="([^"]+)"[^>]*>', html, re.DOTALL | re.I):
        tag = m.group(0)
        if tag.startswith("<p") or tag.startswith("<P"):
            p_text = re.sub(r'<[^>]+>', '', m.group(1) or '').strip()
            p_text = unescape(p_text).strip()
            if p_text and not re.match(r'^[\s\xa0;]*$', p_text):
                # Skip if it's just a style wrapper with no real text
                if len(p_text) > 3:
                    parts.append(p_text)
        elif tag.startswith("<table") or tag.startswith("<TABLE"):
            table_md = html_table_to_markdown(m.group(0))
            if table_md:
                parts.append(table_md)
        elif tag.startswith("<img") or tag.startswith("<IMG"):
            src = m.group(3) or ""
            if src and not src.startswith("data:"):
                alt = re.search(r'alt="([^"]*)"', tag)
                alt_text = alt.group(1) if alt else ""
                if not src.startswith("http"):
                    src = BASE_URL + "/" + src.lstrip("/")
                parts.append("![{}]({})".format(alt_text, src))

    if not parts:
        text = re.sub(r'<[^>]+>', ' ', html)
        text = unescape(text).strip()
        text = re.sub(r'\s+', ' ', text)
        if text:
            parts.append(text)

    return "\n\n".join(parts)

def html_table_to_markdown(table_html):
    lines = []
    rows = re.findall(r'<tr[^>]*>(.*?)</tr\s*>', table_html, re.DOTALL | re.I)
    if not rows:
        return ""
    for ri, row in enumerate(rows):
        cells = re.findall(r'<t[dh][^>]*>(.*?)</t[dh]\s*>', row, re.DOTALL | re.I)
        cell_texts = []
        for c in cells:
            ct = re.sub(r'<[^>]+>', '', c).strip()
            ct = unescape(ct)
            ct = re.sub(r'\s+', ' ', ct)
            cell_texts.append(ct)
        if ri == 0:
            lines.append("| " + " | ".join(cell_texts) + " |")
            lines.append("|---" * len(cell_texts) + "|")
        else:
            lines.append("| " + " | ".join(cell_texts) + " |")
    return "\n".join(lines)

def main():
    items = fetch_list()
    print("[INFO] 列表共 {} 条".format(len(items)), file=sys.stderr)

    results = []
    for item in items:
        print("[INFO]  详情: {}...".format(item["title"][:30]), file=sys.stderr)
        detail = fetch_detail(item["url"])
        content = detail.get("content", "")
        pub_date = detail.get("date", "")

        attachments_str = ""
        if detail.get("attachments"):
            att_list = ["[{}]({})".format(a["name"], a["url"]) for a in detail["attachments"]]
            attachments_str = "; ".join(att_list)

        results.append({
            "title": item["title"],
            "source_url": item["url"],
            "url": item["url"],
            "content": content,
            "site_name": SITE_NAME,
            "pub_date": pub_date or item["date"],
            "summary": content[:200],
            "attachments": attachments_str,
        })

    print(json.dumps(results, ensure_ascii=False, indent=2))

if __name__ == "__main__":
    main()