, table 保留 HTML, 附件独立成段""" parts = [] # 优先定位真实正文容器 body = main_div zoom = main_div.find("div", id="fontzoom") or main_div.find(class_=lambda c: c and isinstance(c, str) and "conTxt" in c) if zoom: body = zoom # 递归遍历块级元素 def _walk(node): for child in node.children: name = getattr(child, 'name', None) if not name: continue if name in ('script', 'style'): continue if name == 'table': if child.find_parent('table'): continue parts.append(table_abs(child, page_url)) continue if name in ('p', 'div', 'section', 'ul', 'ol', 'h2', 'h3', 'h4'): if child.find_parent('table'): continue txt_all = child.get_text('', strip=True) # 跳过导航/页眉元信息块 if child.name == 'div' and len(txt_all) < 300 and not child.find('table'): if re.search(r'(时间[:：]|来源[:：]|访问量|字体|分享到|无障碍|网站地图)', txt_all): continue if child.name == 'div' and child.find(['p', 'table', 'ul', 'ol'], recursive=False): _walk(child) continue if len(txt_all) < 3: continue txt = clean_span(txt_all) if txt and len(txt) > 2 and not re.match(r'^(\s*【|\[\s*关闭)', txt): parts.append(f'
{txt}
') _walk(body) # 去重保序 uniq = [] for p in parts: if p not in uniq: uniq.append(p) return '\n'.join(uniq) def parse_detail(url): html = fetch(url) if not html: return None, None, None soup = BeautifulSoup(html, "html.parser") title = "" mt = soup.find("meta", attrs={"name": re.compile("ArticleTitle", re.I)}) if mt and mt.get("content"): title = mt["content"].strip() if not title: t = soup.find("title") if t: title = re.sub(r'[-_].*$', '', t.get_text(strip=True)).strip() date = "" pd = soup.find("meta", attrs={"name": re.compile("PubDate", re.I)}) if pd and pd.get("content"): m = re.search(r'(\d{4}-\d{2}-\d{2})', pd["content"]) if m: date = m.group(1) content = "" main = soup.find("div", id="mainContent") or soup.find("div", id="wrap") if main: content = extract_main(main, url) return title, content, date def main(): print(f"=== {SITE_NAME} ===") all_items = [] for page in range(_MAX_PG): if page == 0: url = BASE + LIST_PATH + "/" else: url = BASE + LIST_PATH + f"/index_{page + 1}.html" html = fetch(url) if not html: print(f" p{page+1} FAIL"); continue items = parse_list(html) if not items: print(f" p{page+1} empty, stop"); break all_items.extend(items) print(f" p{page+1}: {len(items)}") time.sleep(0.5) seen, filtered = set(), [] for title, url, date in all_items: if url in seen: continue seen.add(url) if date and date < CUTOFF: continue filtered.append((title, url, date)) print(f" 去重/3yr后: {len(filtered)}") results = [] for idx, (title, url, date) in enumerate(filtered): dt, content, dd = parse_detail(url) ft = dt or title fd = date or dd if not content or len(content.strip()) < 10: print(f" [{idx+1}] ⚠空 {title[:30]}") continue summary = re.sub(r'<[^>]+>', '', content)[:200].strip() or ft[:200] results.append({"site_name": SITE_NAME, "title": ft, "url": url, "content": content, "summary": summary, "pub_date": fd}) print(f" [{idx+1}] ✅ {ft[:35]} ({len(content)}B)") time.sleep(0.3) if results: push_to_searchdb(results, "liannan_hjbh_qtxx") print(f"入库 {len(results)}") print("完成") if __name__ == "__main__": main()