转 段落
rc = extract_balanced_div(r.text, r']*class="right-content"[^>]*>')
if rc.strip():
rc = re.sub(r'', '', rc, flags=re.DOTALL | re.I)
parts = []
for href, name in re.findall(r']+href="([^"]+)"[^>]*>(.*?)', rc, re.DOTALL):
name = re.sub(r'<[^>]+>', '', name).strip()
if not name or len(name) < 3:
continue
if any(k in name for k in ('返回', '更多', '上一页', '下一页')):
continue
# 只保留真实附件链接(文件后缀/file/upload/attach), 排除 relate.jsp 等 JS 链接
if not re.search(r'\.(pdf|docx?|xlsx?|zip|rar|wps|jpe?g|png)$|/file/|upload|attach', href, re.I):
continue
url = href if href.startswith('http') else BASE + href
parts.append(f'
{name}
')
content = '\n'.join(parts)
return title, content
def run(max_pages=5):
records = []
seen = set()
for pg in range(max_pages):
items = fetch_list(pg + 1)
if not items:
print(f" Page {pg+1}: 0 items (stop)")
break
print(f" Page {pg+1}: {len(items)} items")
for it in items:
if it['pub_date'] and it['pub_date'] < CUTOFF:
continue
if it['url'] in seen:
continue
seen.add(it['url'])
title, content = fetch_detail(it['url'])
if not title:
title = it['title']
if not content.strip():
continue
records.append({
'title': title, 'url': it['url'], 'source_url': it['url'],
'pub_date': it['pub_date'],
'site_name': SITE_NAME, 'content': content, 'summary': '',
})
print(f" Total: {len(records)} items")
valid = [r for r in records if r['content'].strip()]
if valid:
push_to_searchdb(valid, "xingtai_sthjj")
return len(valid)
if __name__ == '__main__':
ap = argparse.ArgumentParser()
ap.add_argument('--pages', type=int, default=5)
args = ap.parse_args()
cnt = run(max_pages=args.pages)
print(f"Done: {cnt} records")