名称内嵌URL、去tougao/other噪声
防护: JS cookie 挑战 (GET 拿 cookie 后带 Cookie 访问)
"""
import re
import sys
import os
import time
import sqlite3
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin
SITE_NAME = "兴义之窗-通知公告"
GROUP_NAME = "贵州"
DOMAIN = "www.xyzc.cn"
BASE_URL = "http://www.xyzc.cn"
LIST_URL = "http://www.xyzc.cn/news/gonggao"
TOTAL_PAGES = 307 # 分页条显示共307页
SCRIPT_NAME = os.path.basename(__file__)
SEARCH_DB = os.getenv("SEARCH_DB", "/root/search.db")
DB_PATH = SEARCH_DB
HEADERS = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
TIMEOUT = 30
_MAX_PAGES = 1 # 默认1页, 可 --pages=N 覆盖
# ---- 参数解析: 兼容 --pages=N 和 --pages N ----
i = 0
while i < len(sys.argv):
a = sys.argv[i]
if a.startswith("--pages="):
_MAX_PAGES = int(a.split("=", 1)[1])
elif a == "--pages" and i + 1 < len(sys.argv):
_MAX_PAGES = int(sys.argv[i + 1]); i += 1
i += 1
if _MAX_PAGES > TOTAL_PAGES:
_MAX_PAGES = TOTAL_PAGES
# ─── 正文取文本（2026-09-11）：行内节点直接拼接，只在块级边界 /
处换行 ───
# ⚠️ 不要用 el.get_text("\n") 取正文 —— 它是「每个**文本节点**之间插 \n」，Word 粘贴的
# 公文把一行拆成 提取码：pwaj。查阅…，这些行内节点于是各自
# 成行（福泉 id=2095080103703914437 实例：`提取码：`/`pwaj`/`。查阅…` 各占一行）。
_BLOCK_TAGS = {'address', 'article', 'aside', 'blockquote', 'details', 'dialog', 'dd', 'div',
'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2',
'h3', 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'li', 'main', 'nav', 'ol',
'p', 'pre', 'section', 'table', 'tbody', 'thead', 'tfoot', 'tr', 'td', 'th',
'ul', 'center', 'caption'}
def body_text(el):
"""块级边界出换行、行内节点直接拼接、
出换行（≈ 浏览器看到的换行结构）。"""
if el is None:
return ''
import re as _re
from bs4 import NavigableString
out = []
def walk(node):
for ch in node.children:
if isinstance(ch, NavigableString):
out.append(str(ch))
elif getattr(ch, 'name', None) == 'br':
out.append('\n')
elif getattr(ch, 'name', None) in _BLOCK_TAGS:
out.append('\n')
walk(ch)
out.append('\n')
else:
walk(ch)
walk(el)
t = ''.join(out)
t = _re.sub(r'[ \t\r\f\v]*\n[ \t\r\f\v]*', '\n', t)
t = _re.sub(r'\n{3,}', '\n\n', t)
return t.strip()
def make_session():
"""Discuz 门户有 JS cookie 挑战: 先访问首页拿 cookie, 之后请求带 cookie"""
s = requests.Session()
s.headers.update(HEADERS)
try:
s.get(BASE_URL, timeout=TIMEOUT) # 触发 Set-Cookie
except Exception:
pass
return s
session = make_session()
def fetch(url):
try:
resp = session.get(url, timeout=TIMEOUT,
headers={"Referer": "http://www.xyzc.cn/news/gonggao"})
if resp.status_code != 200:
return None
# GBK 编码
raw = resp.content
for enc in ("utf-8", "gbk", "gb18030"):
try:
return raw.decode(enc)
except Exception:
continue
return raw.decode("gbk", errors="replace")
except Exception:
return None
def clean_title(title):
"""标题清洗: strip 实体前缀 + ' - 通知公告' 后缀 + 省略号截断"""
title = re.sub(r'^[\s\xa0·\u00b7]+', '', title or '')
title = re.sub(r'\s*[-–—]\s*通知公告\s*$', '', title)
title = re.sub(r'\s*\.{3,}\s*$', '', title)
return title.strip()
def parse_date(s):
"""'更新：2026-7-31 16:34' -> '2026-07-31'"""
if not s:
return ""
m = re.search(r'(\d{4})-(\d{1,2})-(\d{1,2})', s)
if m:
return "%04d-%02d-%02d" % (int(m.group(1)), int(m.group(2)), int(m.group(3)))
m = re.search(r'(\d{4})年(\d{1,2})月(\d{1,2})日', s)
if m:
return "%04d-%02d-%02d" % (int(m.group(1)), int(m.group(2)), int(m.group(3)))
return ""
def parse_list(html):
"""解析列表: div.news-list-content > dl > dt > a + dd > p.news-txt-time"""
items = []
soup = BeautifulSoup(html, "html.parser")
box = soup.find("div", class_="news-list-content")
if not box:
return items
for dl in box.find_all("dl"):
dt_a = dl.find("dt")
if not dt_a:
continue
a = dt_a.find("a")
if not a or not a.get("href"):
continue
href = a["href"].strip()
if "article-" not in href:
continue # 只抓 article-{id}-1.html 文章链接
if not href.startswith("http"):
# 列表 href 相对根路径, 不能用 urljoin(列表页URL) 会带 /news/ 前缀 404
href = urljoin(BASE_URL + "/", href)
title = clean_title(a.get_text())
if not title:
continue
date = ""
dd = dl.find("dd")
if dd:
p = dd.find("p", class_="news-txt-time")
if p:
date = parse_date(p.get_text())
items.append({"title": title, "url": href, "date": date})
return items
def extract_content(html, page_url):
"""提取正文: div.news-content-txt (段落div + br); 返回 (content, has_table, attachments_html)"""
soup = BeautifulSoup(html, "html.parser")
# 标题: news-content-tit 内 h2 (页面顶部导航有空的 h2, 不能全局 find)
date = ""
dl = soup.find("div", class_="news-content-tit")
if dl:
h2 = dl.find("h2")
title = clean_title(h2.get_text()) if h2 else ""
dt = dl.find("dt")
if dt:
date = parse_date(dt.get_text())
else:
h2 = soup.find("h2")
title = clean_title(h2.get_text()) if h2 else ""
# 正文容器
content_div = soup.find("div", class_=re.compile(r"news-content-txt"))
if not content_div:
return title, date, "", 0, ""
# 噪声: tougao 投稿 / news-content-other 收藏分享 在容器外, 不用管
parts = []
has_table = 0
atts = [] # (href, name)
for el in content_div.find_all(recursive=False):
if el.name == "table":
parts.append(str(el))
has_table = 1
elif el.name in ("p", "div"):
# 收集附件链接
for a in el.find_all("a"):
href = a.get("href") or ""
name = a.get_text(strip=True)
if re.search(r'\.(pdf|docx?|xlsx?|zip|rar|wps|et|dps)(\?|$)', href.lower()) and name:
abs_href = urljoin(page_url, href) if not href.startswith("http") else href
atts.append((abs_href, name))
txt = body_text(el)
txt = re.sub(r'\n+', '\n', txt)
txt = re.sub(r'[ \t\u3000]+', ' ', txt)
txt = re.sub(r'\n\s*\n', '\n', txt)
txt = txt.strip()
if txt:
# br 分隔的短行合并为段落
lines = [l.strip() for l in txt.split("\n") if l.strip()]
parts.append("".join(lines))
# 段落用 \n\n 连接
content = "\n\n".join(parts)
# 附件转