，
导致同一内容重复（容器那份常还被 get_text(strip=True) 拍平）。
判据：先按值去重，再剔除被其它段完全包含的段。保护：剔除后为空则返回去重结果。
"""
if not parts:
return parts
ps = [p for p in parts if isinstance(p, str)]
if len(ps) != len(parts):
return parts
seen, uniq = set(), []
for p in parts:
if p not in seen:
seen.add(p); uniq.append(p)
keep = [a for a in uniq
if not (len(a) >= 40 and any(b is not a and b and b in a for b in uniq))]
return keep if keep else uniq
def body_text(el):
"""块级边界出换行、行内节点直接拼接、 , 与全站 ct-html 渲染一致
html_parts = []
for p in parts:
if p.startswith("<"): # 表格/附件链接等已是 HTML
html_parts.append(p)
else:
html_parts.append(" " + p + "
出换行（≈ 浏览器看到的换行结构）。"""
if el is None:
return ''
import re as _re
from bs4 import NavigableString
out = []
def walk(node):
for ch in node.children:
if isinstance(ch, NavigableString):
out.append(str(ch))
elif getattr(ch, 'name', None) == 'br':
out.append('\n')
elif getattr(ch, 'name', None) in _BLOCK_TAGS:
out.append('\n')
walk(ch)
out.append('\n')
else:
walk(ch)
walk(el)
t = ''.join(out)
t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
t = _re.sub(r'\n{3,}', '\n\n', t)
return t.strip()
def load_cookies():
cookies = {}
if not os.path.exists(COOKIE_FILE):
print(f"[FATAL] cookie 文件不存在: {COOKIE_FILE}", file=sys.stderr)
sys.exit(1)
for line in open(COOKIE_FILE, encoding="utf-8"):
line = line.strip()
if not line or line.startswith("#"):
continue
parts = line.split("\t")
if len(parts) >= 7:
cookies[parts[5]] = parts[6]
elif "=" in line:
k, v = line.split("=", 1)
cookies[k] = v
return cookies
def http_get(url, cookies, referer=None):
headers = {
"User-Agent": UA,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9",
"Cookie": "; ".join(f"{k}={v}" for k, v in cookies.items()),
}
if referer:
headers["Referer"] = referer
req = urllib.request.Request(url, headers=headers)
try:
resp = urllib.request.urlopen(req, timeout=30, context=_SSL_CTX)
return resp.read().decode("utf-8", errors="replace")
except Exception as e:
print(f" [GET fail] {url[:70]} {e}", file=sys.stderr)
return None
def http_get_json(url, cookies, referer=None):
"""JSON 接口请求 (getIndexData 需要 XHR 头, 否则 500/空)"""
headers = {
"User-Agent": UA,
"Cookie": "; ".join(f"{k}={v}" for k, v in cookies.items()),
"Referer": referer or BASE_URL + "/",
"Accept": "application/json, text/javascript, */*; q=0.01",
"X-Requested-With": "XMLHttpRequest",
}
req = urllib.request.Request(url, headers=headers)
try:
resp = urllib.request.urlopen(req, timeout=30, context=_SSL_CTX)
return json.loads(resp.read().decode("utf-8", errors="replace"))
except Exception as e:
print(f" [GETJSON fail] {url[:70]} {e}", file=sys.stderr)
return None
def parse_date_cn(s):
"""'2026年08月17日' -> '2026-08-17'"""
m = re.search(r"(\d{4})年(\d{1,2})月(\d{1,2})日", s or "")
if m:
return f"{m.group(1)}-{int(m.group(2)):02d}-{int(m.group(3)):02d}"
return ""
def fetch_index(cookies):
"""GET 首页建立会话 + getIndexData 全量"""
http_get(BASE_URL + "/", cookies)
time.sleep(random.uniform(LIST_DELAY_MIN, LIST_DELAY_MAX))
d = http_get_json(BASE_URL + "/getIndexData", cookies)
if not d:
return None
return d
def collect_window_items(d, cutoff, today):
"""从 getIndexData 收集窗口内条目"""
items = []
# project 6 组
for k, (tn, typeid) in GROUPS.items():
for it in d.get(k, []):
ts = int(it.get("create_time", "0"))
dt = datetime.fromtimestamp(ts, tz=timezone(timedelta(hours=8))).strftime("%Y-%m-%d")
if cutoff <= dt <= today:
items.append({
"kind": "project", "typename": tn, "typeid": typeid,
"proid": it.get("proid"), "title": it.get("title", ""), "date": dt,
})
# pubother (pubtype 1-6)
for it in d.get("pubother", []):
ts = int(it.get("create_time", "0"))
dt = datetime.fromtimestamp(ts, tz=timezone(timedelta(hours=8))).strftime("%Y-%m-%d")
if cutoff <= dt <= today:
pt = it.get("pubtype", 4)
try:
pti = int(pt)
except (TypeError, ValueError):
pti = 4
items.append({
"kind": "pubother", "typename": PUBTYPE_MAP.get(pti, "其他环境信息"),
"typeid": pti, "pubid": it.get("id"), "title": it.get("title", ""), "date": dt,
})
return items
def fetch_detail(cookies, item):
"""抓详情页, 返回 HTML"""
if item["kind"] == "project":
list_url = f"{BASE_URL}/project/list?typeid={item['typeid']}"
http_get(list_url, cookies, list_url)
time.sleep(random.uniform(LIST_DELAY_MIN, LIST_DELAY_MAX))
url = f"{BASE_URL}/project/detail?type={item['typeid']}&proid={item['proid']}"
html = http_get(url, cookies, list_url)
else:
# pubother: 直接访问 (实测无需前置, 但保底 GET publist)
list_url = f"{BASE_URL}/publist?typeid={item['typeid']}"
http_get(list_url, cookies, list_url)
time.sleep(random.uniform(LIST_DELAY_MIN, LIST_DELAY_MAX))
url = f"{BASE_URL}/pubother/detail?pubid={item['pubid']}"
html = http_get(url, cookies, list_url)
if not html or "登录须知" in html:
return None
return html
def parse_detail(html, item):
"""从详情页提取标题/日期/正文/附件 (bs4 分段版)"""
title = ""
m = re.search(r'