]*href="([^"]*)"[^>]*>(.*?)', _abs_link, raw, flags=re.S | re.I)
# 3. 剥 script/style 和多余标签，保留 /
raw = re.sub(r'<(script|style)[^>]*>.*?', '', raw, flags=re.S | re.I)
# 规范化
标签（去内联样式）
raw = re.sub(r']*>', '', raw)
raw = re.sub(r'
', '
', raw)
# 去掉其他标签属性，保留标签名
raw = re.sub(r'<(/?)(div|span|font|strong|b|em|i|u|h[1-6]|ul|ol|li|tbody|thead|tr|td|th|img|br|hr|section|article)([^>]*)>',
lambda m: f'<{m.group(1)}{m.group(2)}>', raw)
# 4. 还原表格
for i, t in enumerate(tables):
# 表格内也规范化 /
标签
t2 = re.sub(r']*>', ' ', t)
t2 = re.sub(r'<(/?)(div|span|font|strong|b|em|i|u)([^>]*)>', lambda m: f'<{m.group(1)}{m.group(2)}>', t2)
raw = raw.replace(f'@@TABLE{i}@@', t2)
# 4.5 纯文本段落 div → p（search_app 只认 / 走 HTML 渲染）
# 无子块级元素的 div（文本段落）转
def _div_to_p(m):
inner = m.group(1).strip()
# 只转不含块级标签的 div
if re.search(r'<(table|div|p|ul|ol|h[1-6])', inner, re.I):
return f' {inner} '
return f'{inner} '
# 多次迭代处理嵌套
for _ in range(5):
new_raw = re.sub(r'([^<]*(?:<[^/][^>]*>[^<]*)*[^<]*) ', _div_to_p, raw, flags=re.S)
if new_raw == raw:
break
raw = new_raw
# 5. 清理纯空白段落
def _clean_p(m):
inner = m.group(1)
text = re.sub(r'<[^>]+>', '', inner)
text = re.sub(r' |\u3000| |\s', '', text)
if not text and '{inner.strip()}'
raw = re.sub(r'(.*?) ', _clean_p, raw, flags=re.S)
# 6. 多空行压缩
raw = re.sub(r'\n{3,}', '\n\n', raw)
raw = re.sub(r'\s* ', '', raw)
body = raw.strip()
if not body:
return ''
return body
def _extract_nodes(node, parts):
for child in node.children:
if child.name == 'p':
text = child.get_text(' ', strip=True)
if text:
parts.append(text)
elif child.name == 'table':
md = html_table_to_html(str(child))
if md:
parts.append(md)
elif child.name in ('div', 'section', 'article'):
# 纯文本div直接提取
if child.name == 'div' and not child.find(['div', 'p', 'table']):
text = child.get_text(' ', strip=True)
if text:
parts.append(text)
else:
_extract_nodes(child, parts)
def html_table_to_html(table, base_url=""):
"""保留 HTML 表格结构，仅将相对链接/图片转绝对 URL"""
from bs4 import BeautifulSoup
tbl = BeautifulSoup(str(table), 'html.parser')
for a in tbl.find_all('a'):
href = a.get('href', '')
if href and not href.startswith(('http', 'javascript', '#')):
a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href
for img in tbl.find_all('img'):
src = img.get('src', '')
if src and not src.startswith(('http', '//', 'data:')):
img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src
return str(tbl)
def main():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--pages', type=int, default=5, help='尝试抓取前N页（注意：page2+需要浏览器环境）')
args = parser.parse_args()
max_pages = args.pages
print(f"[{SITE_NAME}] 开始 (max_pages={max_pages})")
all_items = []
seen_urls = set()
for page in range(1, max_pages + 1):
items = fetch_list_page(page)
if not items:
if page == 1:
print(" ⚠️ 第1页为空，退出")
return
print(f" 第{page}页: 空（page2+需浏览器环境）")
break
# 去重：page2+可能返回和page1相同的内容（AJAX不工作）
new_items = [it for it in items if it['url'] not in seen_urls]
for it in new_items:
seen_urls.add(it['url'])
if new_items:
print(f" 第{page}页: {len(items)} 条（新{len(new_items)}条）")
all_items.extend(new_items)
else:
print(f" 第{page}页: {len(items)} 条（全部重复，结束）")
print(f"\n列表合计: {len(all_items)} 条")
if not all_items:
print(" 列表为空，退出")
return
db_items = []
success = 0
failed = 0
for i, item in enumerate(all_items):
url = item['url']
title = item['title']
date_str = item['date']
print(f" [{i+1}/{len(all_items)}] {title[:40]}...", end=' ')
body = fetch_detail(url)
if not body:
print("⚠️ 空正文")
failed += 1
else:
print(f"✅ {len(body)}字")
success += 1
db_items.append({
'site_name': SITE_NAME,
'source_url': url,
'url': url,
'title': title,
'pub_date': date_str,
'summary': body[:500] if body else '',
'content': body,
})
push_to_searchdb(db_items, batch_label=SITE_NAME)
print(f"\n[{SITE_NAME}] 完成: 共{len(all_items)}条, 正文成功{success}, 空{failed}")
if __name__ == '__main__':
main()
|