, table 保留 HTML, 附件独立成段""" for child in node.children: name = getattr(child, 'name', None) if not name: continue if name == 'table': if child.find_parent('table'): continue parts.append(table_abs(child, page_url)) continue if name in ('script', 'style'): continue if name in ('p', 'div', 'section', 'ul', 'ol', 'h2', 'h3', 'h4'): if child.find_parent('table'): continue # 附件链接独立成段 att_anchors = [] for a_tag in child.find_all('a', href=True): h = a_tag['href'].lower() if any(h.endswith(e) for e in ('.pdf', '.doc', '.docx', '.xls', '.xlsx', '.zip', '.rar', '.wps')): atext = a_tag.get_text(' ', strip=True) or '附件' att_anchors.append(f'
') # 容器内含 table -> 递归（不 get_text 拍平） inner_tables = child.find_all('table', recursive=True) if inner_tables: # 容器自身若有直接文本先收，然后递归处理内部 direct_text = '' for c2 in child.children: if getattr(c2, 'name', None) is None and getattr(c2, 'string', None): direct_text += str(c2) or '' if direct_text.strip(): t2 = clean_span(re.sub(r'\s+', ' ', direct_text)).strip() if t2 and len(t2) > 2: parts.append(f'{t2}
') _extract_node(child, page_url, parts) for aa in att_anchors: parts.append(aa) continue txt = clean_span(child.get_text('', strip=True)) txt = re.sub(r'&#\d+;', '', txt) if txt and len(txt) > 2: # 若整段就是附件链接文本(或由附件+少量文本构成)且已生成链接, 不重复输出纯文本 if att_anchors and re.sub(r'\s+', '', txt) == re.sub(r'\s+', '', re.sub(r'<[^>]+>', '', att_anchors[0])): pass else: parts.append(f'{txt}
') for aa in att_anchors: parts.append(aa) def extract_content(detail_div, page_url): parts = [] _extract_node(detail_div, page_url, parts) # 去重保序(完全相同的段) uniq = [] for p in parts: if p not in uniq: uniq.append(p) return '\n'.join(uniq) def extract_content(detail_div, page_url): parts = [] _extract_node(detail_div, page_url, parts) # 去重保序(完全相同的段) uniq = [] for p in parts: if p not in uniq: uniq.append(p) return '\n'.join(uniq) def parse_detail(url): html = fetch(url) if not html: return None, None, None soup = BeautifulSoup(html, "html.parser") title = "" mt = soup.find("meta", attrs={"name": re.compile("ArticleTitle", re.I)}) if mt and mt.get("content"): title = mt["content"].strip() if not title: h1 = soup.find("h1") if h1: title = h1.get_text(strip=True) date = "" pd = soup.find("meta", attrs={"name": re.compile("PubDate", re.I)}) if pd and pd.get("content"): m = re.search(r'(\d{4}-\d{2}-\d{2})', pd["content"]) if m: date = m.group(1) content = "" for sel in ["div.TRS_UEDITOR", "div.TRS_Editor", "div#zoom"]: d = soup.select_one(sel) if d: content = extract_content(d, url) break return title, content, date def main(): print(f"=== {SITE_NAME} ===") all_items = [] for page in range(_MAX_PG): if page == 0: url = BASE + LIST_PATH + "/" else: url = BASE + LIST_PATH + f"/index_{page + 1}.html" html = fetch(url) if not html: print(f" p{page+1} FAIL"); continue items = parse_list(html) if not items: print(f" p{page+1} empty, stop"); break all_items.extend(items) print(f" p{page+1}: {len(items)}") time.sleep(0.5) # 去重 + CUTOFF seen, filtered = set(), [] for title, url, date in all_items: if url in seen: continue seen.add(url) if date and date < CUTOFF: continue filtered.append((title, url, date)) print(f" 列表去重/3yr过滤后: {len(filtered)}") results = [] for idx, (title, url, date) in enumerate(filtered): dt, content, dd = parse_detail(url) ft = dt or title fd = date or dd if not content or len(content.strip()) < 10: print(f" [{idx+1}] ⚠空 {title[:30]}") continue summary = re.sub(r'<[^>]+>', '', content)[:200].strip() or ft[:200] results.append({"site_name": SITE_NAME, "title": ft, "url": url, "content": content, "summary": summary, "pub_date": fd}) print(f" [{idx+1}] ✅ {ft[:35]} ({len(content)}B)") time.sleep(0.3) if results: push_to_searchdb(results, "qiannan_hjyxpjypf") print(f"入库 {len(results)}") print("完成") if __name__ == "__main__": main()