#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
crawl_zhoukou_xzsp.py - 周口市生态环境局-行政审批 (xzsp 栏目)
URL: https://sthb.zhoukou.gov.cn/sitesources/sthjj/page_pc/xxgk/fdzdgknr/xzsp/list1.html
CMS: HTML-Import CMS (同 hjyxpj 站群), list{N}.html 分页
列表: ul.picboxcon > li.colRightOneXXGK > span.date + a[title]
详情: /sitesources/sthjj/page_pc/xxgk/fdzdgknr/xzsp/article{32hex}.html
注意: IP 221.14.138.84 + Host header (同 crawl_zhoukou_hjyxpj.py)
详情容器: div.article-detail#articleDetail
"""
import sys, os, re, time
from datetime import datetime, timezone, timedelta

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb
import requests
from bs4 import BeautifulSoup

_MAX_PG = 5
for i, a in enumerate(sys.argv):
    if a == '--pages' and i + 1 < len(sys.argv):
        _MAX_PG = int(sys.argv[i + 1])
        break

SITE_NAME = "周口市生态环境局-行政审批"
BASE_IP = "221.14.138.84"
DOMAIN = "sthb.zhoukou.gov.cn"
BASE_URL = f"http://{BASE_IP}"
LIST_PATH = "/sitesources/sthjj/page_pc/xxgk/fdzdgknr/xzsp"
CUTOFF = (datetime.now(timezone.utc) - timedelta(days=365 * 3)).strftime("%Y-%m-%d")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0 Safari/537.36",
    "Host": DOMAIN,
}

LIST_RE = re.compile(r'<li class="colRightOneXXGK">\s*<span class="date">(\d{4}-\d{2}-\d{2})</span>\s*<a href="([^"]+)"[^>]*title="([^"]*)"', re.S)


def fetch(url, timeout=25):
    for _ in range(3):
        try:
            r = requests.get(url, headers=HEADERS, timeout=timeout)
            r.encoding = "utf-8"
            return r.text
        except Exception as e:
            time.sleep(2)
    return None


def make_list_url(page):
    return BASE_URL + LIST_PATH + f"/list{page}.html"


def parse_list(html):
    items = []
    for m in LIST_RE.finditer(html):
        date, href, title = m.group(1).strip(), m.group(2).strip(), m.group(3).strip()
        if href.startswith("/"):
            full = "http://" + DOMAIN + href
        elif href.startswith("http"):
            full = href
        else:
            full = "http://" + DOMAIN + "/" + href
        if title and date:
            items.append((title, full, date))
    return items


def table_abs(tbl, base_url):
    tb = BeautifulSoup(str(tbl), "html.parser")
    for a in tb.find_all("a", href=True):
        h = a["href"]
        if h and not h.startswith(("http", "javascript", "#")):
            a["href"] = "http://" + DOMAIN + h if h.startswith("/") else h
    for im in tb.find_all("img", src=True):
        s = im["src"]
        if s and not s.startswith(("http", "data:")):
            im["src"] = "http://" + DOMAIN + s if s.startswith("/") else s
    return str(tb)


def clean_span(text):
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=[\u4e00-\u9fff])', '', text)
    text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t\u3000]+(?=\d)', '', text)
    text = re.sub(r'(?<=\d)[ \t\u3000]+(?=[\u4e00-\u9fff])', '', text)
    text = re.sub(r'\s+([，。、；：？！)】」》])', r'\1', text)
    text = re.sub(r'([（【「《])\s+', r'\1', text)
    return re.sub(r'\s+', ' ', text).strip()


def extract_content(detail_div, page_url):
    parts = []
    for child in detail_div.children:
        name = getattr(child, 'name', None)
        if not name:
            continue
        if name == 'table':
            parts.append(table_abs(child, page_url))
        elif name in ('p', 'div', 'section', 'ul', 'ol', 'h2', 'h3', 'h4'):
            if child.find_parent('table'):
                continue
            att_anchors = []
            for a_tag in child.find_all('a', href=True):
                h = a_tag['href'].lower()
                if any(h.endswith(e) for e in ('.pdf', '.doc', '.docx', '.xls', '.xlsx', '.zip', '.rar', '.wps')):
                    atext = a_tag.get_text(' ', strip=True) or '附件'
                    href_full = a_tag['href']
                    if href_full.startswith('/'):
                        href_full = "http://" + DOMAIN + href_full
                    att_anchors.append(f'<p><a href="{href_full}">{atext}</a></p>')
            txt = clean_span(child.get_text('', strip=True))
            txt = re.sub(r'&#\d+;', '', txt)
            if txt and len(txt) > 2:
                parts.append(f'<p>{txt}</p>')
            parts.extend(att_anchors)
    return '\n'.join(parts)


def parse_detail(url):
    html = fetch(url)
    if not html:
        return None, None, None
    soup = BeautifulSoup(html, "html.parser")
    title = ""
    mt = soup.find("meta", attrs={"name": re.compile("ArticleTitle", re.I)})
    if mt and mt.get("content"):
        title = mt["content"].strip()
    if not title:
        h1 = soup.find("h1")
        if h1:
            title = h1.get_text(strip=True)
    date = ""
    pd = soup.find("meta", attrs={"name": re.compile("PubDate", re.I)})
    if pd and pd.get("content"):
        m = re.search(r'(\d{4}-\d{2}-\d{2})', pd["content"])
        if m:
            date = m.group(1)
    content = ""
    detail_div = soup.find("div", class_="article-detail", id="articleDetail") or soup.select_one("div.article-detail")
    if detail_div:
        content = extract_content(detail_div, url)
    return title, content, date


def main():
    print(f"=== {SITE_NAME} ===")
    all_items = []
    for page in range(_MAX_PG):
        url = make_list_url(page + 1)
        html = fetch(url)
        if not html:
            print(f"  p{page+1} FAIL"); continue
        items = parse_list(html)
        if not items:
            print(f"  p{page+1} empty, stop"); break
        all_items.extend(items)
        print(f"  p{page+1}: {len(items)}")
        time.sleep(0.5)
    seen, filtered = set(), []
    for title, url, date in all_items:
        if url in seen:
            continue
        seen.add(url)
        if date and date < CUTOFF:
            continue
        filtered.append((title, url, date))
    print(f"  去重/3yr后: {len(filtered)}")
    results = []
    for idx, (title, url, date) in enumerate(filtered):
        dt, content, dd = parse_detail(url)
        ft = dt or title
        fd = date or dd
        if not content or len(content.strip()) < 10:
            print(f"  [{idx+1}] ⚠空 {title[:30]}")
            continue
        summary = re.sub(r'<[^>]+>', '', content)[:200].strip() or ft[:200]
        results.append({"site_name": SITE_NAME, "title": ft, "url": url,
                        "content": content, "summary": summary, "pub_date": fd})
        print(f"  [{idx+1}] ✅ {ft[:35]} ({len(content)}B)")
        time.sleep(0.3)
    if results:
        push_to_searchdb(results, "zhoukou_xzsp")
        print(f"入库 {len(results)}")
    print("完成")


if __name__ == "__main__":
    main()