#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
格尔木市人民政府 - 工程建设类通知公告 (www.geermu.gov.cn)
CMS: 自定义PHP (嵌入式开发)
列表: /newsList?columnId=...&currentPage=N (N=1~24, ~20条/页, 共471条)
详情: /details?id=hex_id
详情标题: div.xwxqy_titlt
详情正文: div.detailcontainer_content_html
详情日期: meta[name="PubDate"]
网络: 可直接访问
"""

import sys, os, re, time, json
from datetime import datetime, timezone, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

import requests
from bs4 import BeautifulSoup
import urllib.parse

# === 自动页数控制 ===
_MAX_PG = None
for i, a in enumerate(sys.argv):
    if a == '--pages' and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break
if _MAX_PG is not None:
    print(f"[AutoPg] max_pages={_MAX_PG}")

SITE_NAME = "格尔木市人民政府-工程建设类通知公告"
DOMAIN = "www.geermu.gov.cn"
BASE_URL = f"https://{DOMAIN}"
COLUMN_ID = "bb5cf2df6c22fda6016c2be704740082"
TOTAL_PAGES = 24
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365*3)).strftime("%Y-%m-%d")

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "Referer": f"{BASE_URL}/",
}


def html_table_to_html(table, base_url=""):
    """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
    from bs4 import BeautifulSoup
    tbl = BeautifulSoup(str(table), 'html.parser')
    for a in tbl.find_all('a'):
        href = a.get('href', '')
        if href and not href.startswith(('http', 'javascript', '#')):
            a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
    for img in tbl.find_all('img'):
        src = img.get('src', '')
        if src and not src.startswith(('http', '//', 'data:')):
            img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
    return str(tbl)

def fetch(url, timeout=20):
    for attempt in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=timeout, verify=False)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            if attempt < 2:
                time.sleep(2)
            else:
                print(f"  [WARN] {url}: {e}")
                return None


def make_list_url(page):
    return BASE_URL + f"/newsList?columnId={COLUMN_ID}&currentPage={page}"


def parse_list_page(html):
    items = []
    # 每个条目: div[href^="/details?"] + class="flex-row items-center justify-between"
    # 标题: a[href^="/details?"] span.link_index_con
    # 日期: 同级div [YYYY/MM/DD]
    for m in re.finditer(
        r'href="(/details\?id=[^"]+)"[^>]*class="[^"]*\bflex-row\b[^"]*\bjustify-between\b[^"]*">.*?'
        r'class="link_index_con[^"]*">(.*?)</span>.*?'
        r'\[?(\d{4}/\d{2}/\d{2})\]?',
        html, re.DOTALL
    ):
        href = m.group(1).strip()
        title = re.sub(r'<[^>]+>', '', m.group(2)).strip()
        # Clean &nbsp; and whitespace
        title = re.sub(r'\s+', ' ', title).strip()
        date = m.group(3).strip().replace('/', '-')

        if href.startswith("/"):
            href = BASE_URL + href
        elif not href.startswith("http"):
            href = BASE_URL + "/" + href

        if title and href:
            items.append((title, href, date))

    return items


def _clean_span_spaces(text):
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=[\u4e00-\u9fff])', '', text)
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=\d)', '', text)
    text = re.sub(r'(?<=\d)[ \t\u3000]+(?=[\u4e00-\u9fff])', '', text)
    text = re.sub(r'\s+([，。、；：？！)】」》])', r'\1', text)
    text = re.sub(r'([（【「《])\s+', r'\1', text)
    text = re.sub(r'(?<=\d)\s+(?=\d)', '', text)
    return re.sub(r'\s+', ' ', text).strip()


def extract_content(content_div, page_url):
    parts = []
    for child in content_div.children:
        name = getattr(child, 'name', None)
        if not name:
            continue

        if child.name == 'table':
            parts.append(html_table_to_html(child, page_url))
        elif name == 'div':
            inner_content = extract_content(child, page_url)
            if inner_content:
                parts.append(inner_content)
                parts.append("\n\n")

        elif name == 'p':
            if child.find_parent("table"):
                continue

            # 附件链接
            for a_tag in child.find_all("a", href=True):
                href = a_tag["href"]
                if any(href.lower().endswith(e) for e in ('.pdf', '.doc', '.docx', '.xls', '.xlsx', '.zip', '.rar', '.wps')):
                    a_text = a_tag.get_text(" ", strip=True)
                    if href.startswith("/"):
                        full_href = BASE_URL + href
                    elif not href.startswith("http"):
                        full_href = BASE_URL + "/" + href
                    else:
                        full_href = href
                    if a_text:
                        parts.append(f"[{a_text}]({full_href})\n\n")

            txt = child.get_text(" ", strip=True)
            txt = _clean_span_spaces(txt)
            txt = re.sub(r'&#\d+;', '', txt)
            if txt and len(txt) > 3:
                parts.append(txt)
                parts.append("\n\n")

    result = ''.join(parts).strip()
    result = re.sub(r'\n{4,}', '\n\n', result)
    return result


def parse_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None

    soup = BeautifulSoup(html, "html.parser")

    # Title
    title = ""
    title_div = soup.find("div", class_="xwxqy_titlt")
    if title_div:
        title = title_div.get_text(" ", strip=True)
    if not title:
        title_tag = soup.find("title")
        if title_tag:
            title = title_tag.get_text(strip=True)
            title = re.sub(r'_格尔木市人民政府网站.*', '', title).strip()
            title = re.sub(r'格尔木市人民政府网站.*', '', title).strip()

    # Date
    date = ""
    meta_d = soup.find("meta", attrs={"name": "PubDate"})
    if meta_d:
        d = meta_d.get("content", "")
        m = re.search(r'(\d{4}-\d{2}-\d{2})', d)
        if m:
            date = m.group(1)
    if not date:
        m = re.search(r'发布时间：(\d{4}/\d{2}/\d{2})', html)
        if m:
            date = m.group(1).replace('/', '-')

    # Content
    content = ""
    content_div = soup.find("div", class_="detailcontainer_content_html")
    if content_div:
        content = extract_content(content_div, url)
    if not content or len(content.strip()) < 10:
        # Fallback: get all text from detailcontainer_content_html
        content_div = soup.find("div", class_="detailcontainer_content_html")
        if content_div:
            content = content_div.get_text(" ", strip=True)

    return title, content, date


def main():
    pages_to_fetch = min(TOTAL_PAGES, _MAX_PG) if _MAX_PG else TOTAL_PAGES

    print(f"=== {SITE_NAME} ===")
    print(f"  总页数: {TOTAL_PAGES}, 本次: {pages_to_fetch}")

    all_items = []
    for page in range(1, pages_to_fetch + 1):
        list_url = make_list_url(page)
        print(f"  第 {page}/{pages_to_fetch} 页: {list_url[:80]}...")

        html = fetch(list_url)
        if not html:
            print(f"    无法获取, 停止")
            break

        items = parse_list_page(html)
        if not items:
            print(f"    空列表, 停止")
            break

        print(f"    找到 {len(items)} 条")
        all_items.extend(items)
        time.sleep(0.3)

    print(f"\n  列表总计: {len(all_items)} 条\n")

    if not all_items:
        print("  无数据")
        return

    results = []
    for idx, (title, url, date) in enumerate(all_items):
        print(f"  [{idx+1}/{len(all_items)}] {title[:40]}...", end=" ", flush=True)

        dt, content, dd = parse_detail(url)
        final_title = dt or title
        final_date = date or dd

        if not content or len(content.strip()) < 10:
            print("⚠ 空")
            continue

        summary = re.sub(r'<[^>]+>', '', content)[:200].strip()
        if not summary:
            summary = final_title[:200]

        results.append({
            "site_name": SITE_NAME,
            "title": final_title,
            "url": url,
            "content": content,
            "summary": summary,
            "pub_date": final_date,
        })
        print(f"✅ ({len(content)}字)")

        if (idx + 1) % 20 == 0:
            print(f"  进度: {idx+1}/{len(all_items)}, 已入库: {len(results)}")

    if results:
        print(f"\n  入库 {len(results)} 条")
        push_to_searchdb(results, "geermu_gcjs")

    print("完成")


if __name__ == "__main__":
    main()