#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
寿光市人民政府 — 公示公告 (gsgg)
静态HTML分页 index_N.html, --resolve DNS
站点: https://www.shouguang.gov.cn/news/gsgg/
"""
import requests
import re
import json
import sys
import os
import argparse
from bs4 import BeautifulSoup

sys.path.insert(0, "/root/gov_crawler")
from crawler_lib import push_to_searchdb

SITE_NAME = "寿光市人民政府-公示公告"
DOMAIN = "shouguang.gov.cn"
IP = "221.1.82.250"
BASE = "https://shouguang.gov.cn"
LIST_URL = "/news/gsgg/"
GROUP = "山东省"
INDUSTRY = "公示公告"

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
MAX_PAGES = 5


def safe_get(url):
    """GET with --resolve (IP + Host header)."""
    s = requests.Session()
    s.headers.update(HEADERS)
    s.headers["Host"] = DOMAIN
    actual_url = url.replace(DOMAIN, IP)
    r = s.get(actual_url, timeout=30, verify=False)
    r.encoding = "utf-8"
    return r


def fetch_list(page_no):
    """Fetch list page. page_no is 0-indexed (0=index.html)."""
    if page_no == 0:
        url = BASE + LIST_URL
    else:
        url = BASE + f"/news/gsgg/index_{page_no}.html"
    r = safe_get(url)
    if r.status_code != 200:
        return []
    soup = BeautifulSoup(r.text, "html.parser")
    items = []
    for li in soup.select("div.list ul li"):
        a = li.find("a")
        span = li.find("span")
        if a and span:
            href = a.get("href", "")
            title = a.get("title", a.get_text(strip=True))
            date = span.get_text(strip=True)
            if href:
                full_url = BASE + href if href.startswith("/") else BASE + "/news/gsgg/" + href.lstrip("./")
                items.append({"url": full_url, "title": title.strip(), "date": date.strip()})
    return items


def fetch_detail(url):
    """Fetch detail page."""
    r = safe_get(url)
    if r.status_code != 200:
        return None
    soup = BeautifulSoup(r.text, "html.parser")

    # Title from h1 or meta
    title = ""
    h1 = soup.select_one("h1")
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        meta = soup.find("meta", attrs={"name": "ArticleTitle"})
        if meta and meta.get("content"):
            title = meta["content"].strip()

    # Date from meta
    date = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        raw = meta_date["content"].strip()
        m = re.match(r"(\d{4})-(\d{1,2})-(\d{1,2})", raw)
        if m:
            date = f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"

    # Body from div.content
    body_text = ""
    attachments = ""
    content = soup.select_one("div.content")
    if content:
        for s in content.find_all("script"):
            s.decompose()
        parts = []
        for p in content.find_all("p"):
            t = p.get_text(strip=True)
            if t:
                parts.append(t)
        body_text = "\n".join(parts)
        # 正文为图片时提取图片链接
        imgs = content.find_all("img")
        if not body_text and imgs:
            img_parts = []
            for img in imgs:
                src = img.get("src", "")
                if src:
                    if src.startswith("."):
                        # 相对路径: 基于详情页目录解析
                        from urllib.parse import urljoin
                        src = urljoin(url, src)
                    img_parts.append(f"[图片: {src}]")
            attachments = "\n".join(img_parts)
            body_text = attachments

    return {"title": title, "date": date, "body_text": body_text, "attachments": attachments}


def main():
    parser = argparse.ArgumentParser(description="寿光市公示公告爬虫")
    parser.add_argument("--pages", type=int, default=MAX_PAGES, help="爬取页数")
    parser.add_argument("--push", action="store_true", default=False)
    args = parser.parse_args()

    import urllib3
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

    print(f"{SITE_NAME} 公示公告 — 爬取 {args.pages} 页", flush=True)

    all_items = []
    for page_no in range(args.pages):
        items = fetch_list(page_no)
        if not items:
            print(f"第{page_no+1}页: 无数据", flush=True)
            break
        print(f"第{page_no+1}页: {len(items)} 条", flush=True)
        all_items.extend(items)

    print(f"\n共获取 {len(all_items)} 条列表项", flush=True)

    details = []
    for idx, item in enumerate(all_items, 1):
        print(f"  [{idx}/{len(all_items)}] {item['title'][:35]}...", end=" ", flush=True)
        detail = fetch_detail(item["url"])
        if not detail:
            print("❌", flush=True)
            continue
        details.append({
            "site_name": SITE_NAME,
            "source_url": item["url"],
            "url": item["url"],
            "title": detail["title"] or item["title"],
            "pub_date": detail["date"],
            "summary": (detail["body_text"] or "")[:500],
            "content": detail["body_text"] or "",
            "category": GROUP,
            "tags": INDUSTRY,
            "attachments": detail.get("attachments", ""),
        })
        print(f"✅ {len(detail['body_text'])}字", flush=True)

    saved = len(details)
    skipped = len(all_items) - saved
    print(f"\n爬取完成: {saved} 条, {skipped} 条失败", flush=True)

    if args.push and details:
        push_to_searchdb(details, batch_label=f"shouguang_gsgg_{args.pages}p")

    output = {"saved": saved, "skipped": skipped, "total": len(all_items), "push": args.push}
    print(f"\n---STATS---\n{json.dumps(output)}")


if __name__ == "__main__":
    main()