#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
安徽省 重大建设项目批准和实施领域「批准结果信息」爬虫
列表: https://zwgk.ah.gov.cn/Zdxm/showList/568/page_N.html   (20 条/页)
      <pagination currentpage="1" pagesize="20" pagecount="80" total="1589">  ← pagecount 直接给总页数
条目: <li><span class="fr">2026-09-17</span><a href="绝对URL" title="完整标题" target="_blank">标题</a><i></i></li>
      ⚠️ 本栏目是**跨厅局聚合页**：条目链接指向 sthjt / jtt / fzggw / zrzyt / slt 等不同厅局域名
      ⚠️ 左侧子栏目菜单也是 li>a（href=/Zdxm/showList/586/、/Zdxm/sgxk/），**必须按 netloc 过滤**
         —— 只保留「非 zwgk.ah.gov.cn」的链接；否则会混入「取水许可审批结果…」这类导航条目
      ⚠️ 日期在 <span class="fr"> 里、在 <a> **前面**（非 TRS 常见的 a+span 顺序）
详情: 各厅局模板完全不同，**没有统一容器** → 正文用「最密 <p> 簇」通用算法：
        max_p = 全页 <p> 最多的容器数 → 在同 p 数的容器里取**文本最短**的（= 最内层包裹，避开页头页脚）
      实测有效容器：sthjt=div.con_main/div.lmcontain.wenzhang(15p) / jtt=div.j-fontContent.gkwz_contnet(43p)
                    zrzyt=div.ls-content-con(13p)
      ⚠️ 特例 fzggw.ah.gov.cn/site/tpl/<tpl>?id=<n>：**壳页**，静态 HTML 里 <p>=0、正文只有 208 字导航文本。
         真数据在龙讯 label 接口：GET /site/label/8888?labelName=amdManage&isJson=true&siteId=49631471&id=<n>
         → JSON 的 data 是「行政许可决定书文号/许可类别/许可内容/行政相对人名称/统一社会信用代码…」结构化字段
用法:
  python3 crawl_ah_zdxm.py --pages=1
  python3 crawl_ah_zdxm.py --pages=5
"""
import os
import re
import sys
import time
import json
import html as html_mod
from urllib.parse import urljoin, urlparse

import requests
from bs4 import BeautifulSoup

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from crawler_lib import push_to_searchdb

BASE_URL = "https://zwgk.ah.gov.cn"
LIST_DIR = "/Zdxm/showList/568"
SELF_HOST = "zwgk.ah.gov.cn"          # 本站自身（栏目导航页）
SITE_NAME = "安徽省-重大建设项目批准结果信息"
CATEGORY = "审批公示"
GROUP = "安徽"
CUTOFF = "2023-09-18"
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36")
TIMEOUT = 45

# fzggw 龙讯 label 接口
FZGGW_SITE_ID = 49631471
FZGGW_LABEL = "amdManage"
FZGGW_FIELDS = [
    ("manageFiledNum", "行政许可决定书文号"),
    ("manageTitle", "行政许可决定文书名称"),
    ("manageClass", "许可类别"),
    ("manageContent", "许可内容"),
    ("manageContentName", "行政相对人名称"),
    ("manageRemarkC", "行政相对人类别"),
    ("manageCode", "统一社会信用代码"),
    ("manageOrgan", "组织机构代码"),
    ("manageBusinessCode", "工商注册号"),
    ("manageRegisterNum", "税务登记号"),
    ("manageUnitCode", "事业单位证书号"),
    ("manageSocietyCode", "社会组织登记证号"),
    ("manageFrCode", "法定代表人姓名"),
    ("manageSource", "许可决定日期"),
    ("manageStartDate", "许可有效期自"),
    ("manageEndDate", "许可有效期至"),
    ("manageOrganName", "许可机关"),
    ("manageOrganCode", "许可机关统一社会信用代码"),
    ("manageStatus", "当前状态"),
    ("manageDataFrom", "数据来源单位"),
    ("manageAddressCode", "住所"),
]

SESSION = requests.Session()
SESSION.headers.update({"User-Agent": UA, "Accept-Language": "zh-CN,zh;q=0.9"})
try:
    requests.packages.urllib3.disable_warnings()
except Exception:
    pass


def parse_args():
    pages = 0
    for a in sys.argv[1:]:
        if a.startswith("--pages="):
            pages = int(a.split("=")[1])
        elif a == "--pages" and len(sys.argv) > sys.argv.index(a) + 1:
            pages = int(sys.argv[sys.argv.index(a) + 1])
    return pages


def clean_title(t):
    t = html_mod.unescape(t or "")
    t = re.sub(r"[\u200b\u200e\u200f\ufeff\xa0]", "", t)
    t = re.sub(r"\s+", " ", t)
    t = re.sub(r"^[•·\-—]\s*", "", t)
    return t.strip()


def http_get(url, referer=None):
    for attempt in range(3):
        try:
            r = SESSION.get(url, headers={"Referer": referer or BASE_URL + "/"},
                            timeout=TIMEOUT, verify=False)
            if r.status_code == 200:
                if not r.encoding or r.encoding.lower() in ("iso-8859-1", "latin-1", "ascii", "cp1252"):
                    r.encoding = r.apparent_encoding or "utf-8"
                return r.text
            print("  [WARN] HTTP %s %s" % (r.status_code, url[:90]), file=sys.stderr)
            return None
        except Exception as e:
            if attempt == 2:
                print("  [WARN] GET failed %s: %s" % (url[:80], str(e)[:80]), file=sys.stderr)
                return None
            time.sleep(1.0 * (attempt + 1))
    return None


def parse_list(html_text):
    """返回 [(url, title, date), ...]；排除本站栏目导航链接"""
    soup = BeautifulSoup(html_text, "html.parser")
    out = []
    seen = set()
    for li in soup.find_all("li"):
        a = li.find("a", href=True)
        if not a:
            continue
        href = a["href"].strip()
        if not href or href.startswith(("javascript:", "#", "mailto:", "tel:")):
            continue
        if href.startswith("http"):
            u = href
        elif href.startswith("//"):
            u = "https:" + href
        elif href.startswith("/"):
            u = urljoin(BASE_URL, href)
        else:
            u = urljoin(BASE_URL + LIST_DIR + "/page_1.html", href)
        # ⚠️ 关键过滤：本栏目是跨厅局聚合页，本站自身的链接都是左侧子栏目导航
        if urlparse(u).netloc.endswith(SELF_HOST):
            continue
        # ⚠️ 跨厅局的「栏目列表页」也不是文章：slt.ah.gov.cn/public/column/21731?type=4&action=list&catId=…
        if "action=list" in u or re.search(r"/public/column/\d+", u):
            continue
        if u in seen:
            continue
        seen.add(u)
        title = clean_title(a.get("title") or a.get_text(" ", strip=True))
        date = ""
        sp = li.find("span")
        if sp:
            m = re.search(r"(\d{4}-\d{2}-\d{2})", sp.get_text(" ", strip=True))
            if m:
                date = m.group(1)
        if not date:
            m = re.search(r"(\d{4}-\d{2}-\d{2})", li.get_text(" ", strip=True))
            if m:
                date = m.group(1)
        if title:
            out.append((u, title, date))
    return out


def total_pages(html_text):
    m = re.search(r'pagecount="(\d+)"', html_text)
    return int(m.group(1)) if m else 0


def absolutize(soup, page_url):
    for tag, attr in (("a", "href"), ("img", "src")):
        for t in soup.find_all(tag):
            v = (t.get(attr) or "").strip()
            if not v or v.startswith(("javascript:", "mailto:", "tel:", "#", "data:")):
                continue
            if v.startswith("//"):
                t[attr] = "https:" + v
            elif not v.startswith("http"):
                t[attr] = urljoin(page_url, v)


def pick_content_node(soup):
    """最密 <p> 簇：<p> 最多的容器中，取文本最短的（= 最内层，避开页头/页脚/导航）"""
    cands = []
    for el in soup.find_all(["div", "td", "article", "section"]):
        np_ = len(el.find_all("p"))
        if np_ < 2:
            continue
        txt = el.get_text(" ", strip=True)
        if len(txt) < 120:
            continue
        cands.append((np_, len(txt), el))
    if not cands:
        return None
    maxp = max(c[0] for c in cands)
    tight = [c for c in cands if c[0] == maxp]
    tight.sort(key=lambda c: c[1])
    return tight[0][2]


def clean_content(node, page_url):
    if node is None:
        return ""
    for bad in node.find_all(["script", "style", "iframe", "noscript"]):
        bad.decompose()
    for b in node.select("div.article-share-group, div.article-share-more, div.favoriteContainer, "
                         "div.div_table_suoyin, div.xz, div.article-inf"):
        b.decompose()
    absolutize(node, page_url)

    parts = []
    for el in node.find_all(["p", "table", "h1", "h2", "h3", "h4"], recursive=True):
        if el.name == "table":
            txt = str(el)
        else:
            txt = clean_title(el.get_text(" ", strip=True))
        if txt:
            parts.append(txt)
    if not parts:
        t = clean_title(node.get_text("\n", strip=True))
        if t:
            parts = [t]

    body = "\n".join(p if p.strip().startswith("<table") else "<p>%s</p>" % p for p in parts)
    body = re.sub(r"<p>\s*</p>", "", body)
    return body


def extract_atts(html_text, page_url):
    atts = []
    seen = set()
    for m in re.finditer(r'href="([^"]+\.(?:pdf|docx?|xlsx?|zip|rar|wps|7z|txt))"', html_text, re.I):
        u = m.group(1).strip()
        if u.startswith("//"):
            u = "https:" + u
        elif not u.startswith("http"):
            u = urljoin(page_url, u)
        if u in seen:
            continue
        seen.add(u)
        atts.append({"name": os.path.basename(u.split("?")[0])[:120], "url": u})
    return atts


def fetch_fzggw_manage(url):
    """fzggw 壳页：走龙讯 label 接口取行政许可结构化数据"""
    from urllib.parse import parse_qs
    qs = parse_qs(urlparse(url).query)
    rid = (qs.get("id") or [""])[0]
    if not rid:
        return ""
    try:
        r = SESSION.get("https://fzggw.ah.gov.cn/site/label/8888",
                        params={"labelName": FZGGW_LABEL, "isJson": "true",
                                "siteId": FZGGW_SITE_ID, "id": rid},
                        headers={"Referer": url, "X-Requested-With": "XMLHttpRequest"},
                        timeout=TIMEOUT, verify=False)
        r.encoding = "utf-8"
        j = r.json()
        data = j.get("data") or {}
    except Exception as e:
        print("  [WARN] fzggw label 接口失败 id=%s: %s" % (rid, str(e)[:70]), file=sys.stderr)
        return ""
    if not isinstance(data, dict) or not data:
        return ""
    rows = []
    for k, label in FZGGW_FIELDS:
        v = data.get(k)
        if v in (None, "", "null"):
            continue
        rows.append("<tr><td>%s</td><td>%s</td></tr>" % (label, clean_title(str(v))))
    return "<table>%s</table>" % "".join(rows) if rows else ""


def fetch_detail(url, list_title, list_date):
    html_text = http_get(url)
    if not html_text:
        return list_title, list_date, "", []
    soup = BeautifulSoup(html_text, "html.parser")

    title = list_title
    mt = soup.find("meta", {"name": "ArticleTitle"})
    if mt and mt.get("content"):
        title = clean_title(mt["content"]) or title

    date = list_date
    mp = soup.find("meta", {"name": "PubDate"})
    if mp and mp.get("content"):
        m = re.search(r"(\d{4})[-/年](\d{1,2})[-/月](\d{1,2})", mp["content"])
        if m:
            date = "%04d-%02d-%02d" % (int(m.group(1)), int(m.group(2)), int(m.group(3)))

    node = pick_content_node(soup)
    body = clean_content(node, url)

    # fzggw 壳页：正文过短则走 label 接口
    netloc = urlparse(url).netloc
    plain_len = len(re.sub(r"<[^>]*>?", "", body).strip())
    if netloc.endswith("fzggw.ah.gov.cn") and plain_len < 200:
        alt = fetch_fzggw_manage(url)
        if alt:
            body = alt

    return title, date, body, extract_atts(html_text, url)


def main():
    pages = parse_args()
    if not pages:
        pages = 5

    all_items = []
    seen = set()
    for pg in range(1, pages + 1):
        list_page_url = "%s%s/page_%d.html" % (BASE_URL, LIST_DIR, pg)
        html_text = http_get(list_page_url, referer=BASE_URL + LIST_DIR + "/page_1.html")
        if not html_text:
            print("[PAGE %d] fetch failed, stopping" % pg)
            break
        if pg == 1:
            tp = total_pages(html_text)
            if tp:
                print("[INFO] 栏目共 %s 页" % tp)
        items = parse_list(html_text)
        if not items:
            print("[PAGE %d] 0 items (分页边界), stopping" % pg)
            break
        n_new = 0
        for detail_url, title, date in items:
            if detail_url in seen:
                continue
            seen.add(detail_url)
            if date and date < CUTOFF:
                continue
            d_title, d_date, body, atts = fetch_detail(detail_url, title, date)
            content = body
            if atts:
                seg = "\n".join('<p><a href="%s">%s</a></p>' % (a["url"], a["name"]) for a in atts)
                content = (content + "\n" + seg).strip() if content else seg
            if not content or len(re.sub(r"<[^>]*>?", "", content).strip()) < 10:
                content = "<p>%s</p>" % (d_title or title)
            all_items.append({
                "site_name": SITE_NAME,
                "title": d_title or title,
                "pub_date": d_date or date,
                "content": content,
                "source_url": detail_url,
                "url": detail_url,
                "attachments": json.dumps(atts, ensure_ascii=False) if atts else "[]",
                "group_name": GROUP,
                "industry": "",
            })
            n_new += 1
            time.sleep(0.25)
        print("[PAGE %d] +%d items (total %d)" % (pg, n_new, len(all_items)))

    print("\n有效 %d 条, 入库..." % len(all_items))
    push_to_searchdb(all_items, CATEGORY)
    print("完成! 新增/更新 %d 条" % len(all_items))


if __name__ == "__main__":
    main()