#!/usr/bin/env python3
"""
寿光市人民政府 — 环评公示
静态HTML分页 index_N.html, --resolve DNS
"""
import requests
import re
import json
import sys
import os
import argparse
from bs4 import BeautifulSoup

sys.path.insert(0, "/root/gov_crawler")
from crawler_lib import push_to_searchdb

SITE_NAME = "寿光市人民政府"
DOMAIN = "shouguang.gov.cn"
IP = "221.1.82.250"
BASE = "https://shouguang.gov.cn"
LIST_URL = "/news/hpgs/"
GROUP = "山东省"
INDUSTRY = "环评公示"

HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
MAX_PAGES = 10


def safe_get(url):
    """GET with --resolve (IP + Host header)."""
    s = requests.Session()
    s.headers.update(HEADERS)
    s.headers["Host"] = DOMAIN
    actual_url = url.replace(DOMAIN, IP)
    r = s.get(actual_url, timeout=30, verify=False)
    r.encoding = "utf-8"
    return r


def fetch_list(page_no):
    """Fetch list page. page_no is 0-indexed (0=index.html)."""
    if page_no == 0:
        url = BASE + LIST_URL
    else:
        url = BASE + f"/news/hpgs/index_{page_no}.html"
    r = safe_get(url)
    if r.status_code != 200:
        return []
    soup = BeautifulSoup(r.text, "html.parser")
    items = []
    for li in soup.select("div.list ul li"):
        a = li.find("a")
        span = li.find("span")
        if a and span:
            href = a.get("href", "")
            title = a.get("title", a.get_text(strip=True))
            date = span.get_text(strip=True)
            if href:
                full_url = BASE + href if href.startswith("/") else BASE + "/news/hpgs/" + href.lstrip("./")
                items.append({"url": full_url, "title": title.strip(), "date": date.strip()})
    return items


def fetch_detail(url):
    """Fetch detail page."""
    r = safe_get(url)
    if r.status_code != 200:
        return None
    soup = BeautifulSoup(r.text, "html.parser")

    # Title from h1 or meta
    title = ""
    h1 = soup.select_one("h1")
    if h1:
        title = h1.get_text(strip=True)
    if not title:
        meta = soup.find("meta", attrs={"name": "ArticleTitle"})
        if meta and meta.get("content"):
            title = meta["content"].strip()

    # Date from meta
    date = ""
    meta_date = soup.find("meta", attrs={"name": "PubDate"})
    if meta_date and meta_date.get("content"):
        raw = meta_date["content"].strip()
        m = re.match(r"(\d{4})-(\d{1,2})-(\d{1,2})", raw)
        if m:
            date = f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2)}"

    # Body from div.content
    body_text = ""
    content = soup.select_one("div.content")
    if content:
        for s in content.find_all("script"):
            s.decompose()
        parts = []
        for p in content.find_all("p"):
            t = p.get_text(strip=True)
            if t:
                parts.append(t)
        body_text = "\n".join(parts)

    return {"title": title, "date": date, "body_text": body_text}


def main():
    parser = argparse.ArgumentParser(description="寿光市环评公示爬虫")
    parser.add_argument("--pages", type=int, default=MAX_PAGES, help="爬取页数")
    parser.add_argument("--push", action="store_true", default=False)
    args = parser.parse_args()

    import urllib3
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

    print(f"{SITE_NAME} 环评公示 — 爬取 {args.pages} 页", flush=True)

    all_items = []
    for page_no in range(args.pages):
        items = fetch_list(page_no)
        if not items:
            print(f"第{page_no+1}页: 无数据", flush=True)
            break
        print(f"第{page_no+1}页: {len(items)} 条", flush=True)
        all_items.extend(items)

    print(f"\n共获取 {len(all_items)} 条列表项", flush=True)

    details = []
    for idx, item in enumerate(all_items, 1):
        print(f"  [{idx}/{len(all_items)}] {item['title'][:35]}...", end=" ", flush=True)
        detail = fetch_detail(item["url"])
        if not detail:
            print("❌", flush=True)
            continue
        details.append({
            "site_name": SITE_NAME,
            "source_url": item["url"],
            "url": item["url"],
            "title": detail["title"] or item["title"],
            "pub_date": detail["date"],
            "summary": (detail["body_text"] or "")[:500],
            "content": detail["body_text"] or "",
            "category": GROUP,
            "tags": INDUSTRY,
            "attachments": "",
        })
        print(f"✅ {len(detail['body_text'])}字", flush=True)

    saved = len(details)
    skipped = len(all_items) - saved
    print(f"\n爬取完成: {saved} 条, {skipped} 条失败", flush=True)

    if args.push and details:
        push_to_searchdb(details, batch_label=f"shouguang_hpgs_{args.pages}p")

    output = {"saved": saved, "skipped": skipped, "total": len(all_items), "push": args.push}
    print(f"\n---STATS---\n{json.dumps(output)}")


if __name__ == "__main__":
    main()