#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
湖北省发展和改革委员会 - 审批 爬虫
栏目: https://fgw.hubei.gov.cn/fbjd/xxgkml/zdjsxm/pzjgxx/sp/index.shtml
      (「审批」总栏目, 含 jnscyj 等全部子栏目; 原为 /sp/jnscyj 单个子栏目)
内容: 省发改委审批结果 (项目建议书/可研/初步设计/节能审查/免税确认等批复)
CMS: TRS WCM + 瑞数4代 JS challenge WAF (412)
WAF 方案: playwright chromium headless 渲染列表页拿瑞数 cookie → requests 带 cookie 直抓
  ⚠️ 服务器 DNS SERVFAIL → chromium --host-resolver-rules=MAP fgw.hubei.gov.cn 223.75.52.243
  ⚠️ cookie 可能过期 (412) → 自动重新 playwright 刷 cookie 重试
列表: TRS 静态分页 createPageHTML(39,...) → index_N.shtml (N=0起, index_1=第2页)
  <a href="../../../../../xkfw/xzxkjg/xmspqk/YYYYMM/tYYYYMMDD_ID.shtml" title="完整标题">标题<span>YYYY-MM-DD</span></a>
  ⚠️ 显示文本截断带省略号 → 用 title 属性 (完整)
  ⚠️ href 相对路径须 urljoin(列表页URL) 保留完整路径段 (fbjd/xxgkml/xkfw/...)
详情: https://fgw.hubei.gov.cn/fbjd/xxgkml/xkfw/xzxkjg/xmspqk/YYYYMM/tYYYYMMDD_ID.shtml
  meta ArticleTitle/PubDate; 正文 id="zwnr" class="zwblock" > div.view.TRS_UEDITOR (>p 段落)
  附件: //fgw.hubei.gov.cn/pdfconvert/download/{ID}.zip?fileName=标题&template=zcwj (urljoin 绝对化)
CUTOFF 3年: 2023-08-11  (新 /sp/ 共 125 页 createPageHTML(125,...), 全量约 2500 条;
  原 /sp/jnscyj 仅 39 页 → 上提一级后覆盖更全)
用法:
  python3 crawl_hbfgw_jnscyj.py --pages=1
  python3 crawl_hbfgw_jnscyj.py --pages=11
"""
import os, sys, re, time, json, html as html_mod, subprocess
import requests
from urllib.parse import urljoin, quote

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE_URL = "https://fgw.hubei.gov.cn"
LIST_DIR = "/fbjd/xxgkml/zdjsxm/pzjgxx/sp"
LIST_URL = BASE_URL + LIST_DIR + "/index.shtml"
SITE_NAME = "湖北省发改委-审批"
CATEGORY = "审批"
CUTOFF = "2023-08-11"   # 3 年窗口
IPV4 = "223.75.52.243"  # 服务器 DNS SERVFAIL, 强制映射 (楚天云 CNAME)

UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
TIMEOUT = 30

# 瑞数 cookie (playwright 初始化获取)
RUI_COOKIE = {"str": ""}


def parse_args():
    pages = 0
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            pages = int(a.split("=")[1])
        elif a.startswith("--pages") and len(sys.argv) > sys.argv.index(a) + 1:
            pages = int(sys.argv[sys.argv.index(a) + 1])
    return pages


def clean_title(t):
    t = html_mod.unescape(t or "")
    t = re.sub(r"[\u200b\u200e\u200f\ufeff\xa0]", "", t)
    t = re.sub(r"\s+", " ", t)
    t = re.sub(r"^[•·\-—]\s*", "", t)
    return t.strip()


def refresh_ruishu_cookie():
    """playwright 渲染列表页拿瑞数 cookie (服务器 DNS SERVFAIL 须 host-resolver-rules 映射)"""
    from playwright.sync_api import sync_playwright
    with sync_playwright() as p:
        browser = p.chromium.launch(
            headless=True,
            args=[
                "--disable-blink-features=AutomationControlled",
                f"--host-resolver-rules=MAP fgw.hubei.gov.cn {IPV4}",
            ],
        )
        ctx = browser.new_context(user_agent=UA, locale="zh-CN")
        page = ctx.new_page()
        page.goto(LIST_URL, wait_until="load", timeout=60000)
        page.wait_for_timeout(4000)
        cookies = ctx.cookies()
        browser.close()
    RUI_COOKIE["str"] = "; ".join([f"{c['name']}={c['value']}" for c in cookies])
    print(f"  [cookie] 刷新成功 {len(cookies)} 个 cookie")


def http_get(url, referer=None, retry_waf=True):
    """requests 带瑞数 cookie 抓取; 412 时自动刷 cookie 重试"""
    for attempt in range(4):
        if not RUI_COOKIE["str"]:
            refresh_ruishu_cookie()
        headers = {
            "User-Agent": UA,
            "Cookie": RUI_COOKIE["str"],
            "Referer": referer or LIST_URL,
            "Accept-Language": "zh-CN,zh;q=0.9",
        }
        try:
            r = requests.get(url, headers=headers, timeout=TIMEOUT)
            if r.status_code == 200:
                if "_$ss" in r.text[:2000] or "Environment Checking" in r.text[:2000]:
                    if retry_waf and attempt < 3:
                        print(f"  [WARN] WAF 页, 刷 cookie 重试 ({attempt+1})", file=sys.stderr)
                        refresh_ruishu_cookie()
                        continue
                    return None
                r.encoding = "utf-8"
                return r.text
            if r.status_code == 412 and retry_waf and attempt < 3:
                print(f"  [WARN] 412, 刷 cookie 重试 ({attempt+1})", file=sys.stderr)
                refresh_ruishu_cookie()
                continue
        except Exception as e:
            if attempt == 3:
                print(f"  [WARN] GET failed {url}: {e}", file=sys.stderr)
                return None
        time.sleep(1.0)
    return None


def _abs_url(u, page_url):
    if u.startswith(("http://", "https://", "javascript:", "mailto:", "tel:", "data:", "#")):
        return u
    if u.startswith("//"):
        return "https:" + u
    return urljoin(page_url, u)


def clean_content(raw_html, page_url):
    """正文清洗: 保留 <p> 段落 HTML, 剥 span/font 内联样式, 附件/图片绝对化"""
    if not raw_html:
        return ""
    raw_html = re.sub(r"<script[\s\S]*?</script>", "", raw_html)
    raw_html = re.sub(r"<style[\s\S]*?</style>", "", raw_html)
    raw_html = re.sub(r"<iframe[\s\S]*?</iframe>", "", raw_html)
    raw_html = re.sub(r'<a\s+href="([^"]+)"', lambda m: f'<a href="{_abs_url(m.group(1), page_url)}"', raw_html)
    raw_html = re.sub(r'<img\s+src="([^"]+)"', lambda m: f'<img src="{_abs_url(m.group(1), page_url)}"', raw_html)
    raw_html = re.sub(r"<p[^>]*>", "<p>", raw_html)
    raw_html = re.sub(r"</?span[^>]*>", "", raw_html)
    raw_html = re.sub(r"<(table|tbody|thead|tr|td|th|br)([^>]*)>", lambda m: f"<{m.group(1)}>", raw_html)
    raw_html = re.sub(r"</(table|tbody|thead|tr|td|th|br)>", lambda m: f"</{m.group(1)}>", raw_html)
    raw_html = re.sub(r"</?(?:font|b|strong|em|u|sup|sub)\b[^>]*>", "", raw_html, flags=re.I)
    raw_html = re.sub(r"[\u3000\xa0]", " ", raw_html)
    raw_html = re.sub(r"<p>\s*(?:&nbsp;)?\s*</p>", "", raw_html)
    raw_html = re.sub(r"\r\n", "\n", raw_html)
    return raw_html.strip()


def extract_attachments(content, page_url):
    """从正文提取附件链接 (pdfconvert/下载链接)"""
    atts = []
    seen = set()
    for am in re.finditer(r'<a\s+href="([^"]+)"[^>]*>(.*?)</a>', content, re.S):
        ahref, atext = am.group(1), re.sub(r"<[^>]+>", "", am.group(2)).strip()
        if re.search(r"\.(pdf|doc|docx|xls|xlsx|zip|rar|wps|7z|txt)(\?|$)", ahref, re.I) or "download" in ahref.lower() or "pdfconvert" in ahref.lower():
            u = _abs_url(ahref, page_url)
            if u not in seen:
                seen.add(u)
                # pdfconvert 附件名取 fileName 参数
                fm = re.search(r"[?&]fileName=([^&]+)", u)
                name = atext
                if not name and fm:
                    name = urllib_parse_unquote(fm.group(1))[:60]
                atts.append({"name": name or u.split("/")[-1], "url": u})
    return json.dumps(atts, ensure_ascii=False) if atts else "[]"


def urllib_parse_unquote(s):
    from urllib.parse import unquote
    return unquote(s)


def fetch_detail(url):
    """抓详情页: (正文, 附件列表)"""
    html_text = http_get(url)
    if not html_text:
        return "", []
    atts = []
    # 正文: id="zwnr" (TRS_UEDITOR)
    body = ""
    m = re.search(r'<div[^>]*id="zwnr"[^>]*>([\s\S]*?)</div>\s*(?:<div[^>]*id="fjtp"|<div[^>]*class="[^"]*appendix|<div[^>]*class="[^"]*article_qrcode|<!--\s*content 结束|$)', html_text)
    if not m:
        m = re.search(r'id="zwnr"[^>]*>([\s\S]*?)</div>', html_text)
    if m:
        body = clean_content(m.group(1), url)
    # 附件: 全文找 pdfconvert/download 链接 (正文外也可能有)
    for am in re.finditer(r'href="([^"]*pdfconvert/download/[^"]+)"', html_text, re.I):
        ahref = am.group(1)
        u = _abs_url(ahref, url)
        fm = re.search(r"[?&]fileName=([^&]+)", u)
        name = urllib_parse_unquote(fm.group(1))[:80] if fm else u.split("/")[-1]
        if not any(a["url"] == u for a in atts):
            atts.append({"name": name, "url": u})
    return body, atts


def main():
    pages = parse_args()
    if not pages:
        pages = 5

    all_items = []
    seen = set()
    for pg in range(1, pages + 1):
        if pg == 1:
            list_page_url = LIST_URL
        else:
            list_page_url = f"{BASE_URL}{LIST_DIR}/index_{pg-1}.shtml"
        html_text = http_get(list_page_url, referer=LIST_URL)
        if not html_text:
            print(f"[PAGE {pg}] fetch failed, stopping")
            break
        items = re.findall(
            r'<a[^>]*href="([^"]*t20\d{6}_\d+\.shtml)"[^>]*title="([^"]*)"[^>]*>[\s\S]*?<span>(\d{4}-\d{2}-\d{2})</span>',
            html_text)
        if not items:
            print(f"[PAGE {pg}] 0 items (分页边界), stopping")
            break
        # 目录越级 href 只出现在绝对路径里时不重复; 相对 href 一律以列表页 URL 为基址
        for href, title_raw, date_raw in items:
            detail_url = urljoin(list_page_url, href)   # ⚠️ 不要复用/覆盖列表页 URL 变量
            detail_url = re.sub(r"(?:/xkfw)+/", "/xkfw/", detail_url)   # 兜底: 折叠重复路径段
            if detail_url in seen:
                continue
            seen.add(detail_url)
            title = clean_title(title_raw)
            if not title:
                continue
            pub_date = date_raw.strip()
            if pub_date and pub_date < CUTOFF:
                continue
            body, atts = fetch_detail(detail_url)
            content = body
            if not content or (len(re.sub(r"<[^>]*>?", "", content or "").strip()) < 10 and "<img" not in content):
                content = f"<p>{title}</p>"
            attachments = json.dumps(atts, ensure_ascii=False) if atts else "[]"
            all_items.append({
                "site_name": SITE_NAME,
                "title": title,
                "pub_date": pub_date,
                "content": content,
                "source_url": detail_url,
                "url": detail_url,
                "attachments": attachments,
                "industry": "",
            })
        print(f"[PAGE {pg}] +{len(items)} items (total {len(all_items)})")
        time.sleep(0.5)

    print(f"\n有效 {len(all_items)} 条, 入库...")
    push_to_searchdb(all_items, CATEGORY)
    print(f"完成! 新增/更新 {len(all_items)} 条")


if __name__ == "__main__":
    main()