(.*?)

title = "" m = re.search(r'

\s*(.*?)\s*

', html, re.DOTALL) if m: raw = m.group(1) # Remove HTML comments first (before any tag stripping) raw = re.sub(r'', '', raw, flags=re.DOTALL) # Remove all remaining HTML tags title = re.sub(r'<[^>]+>', '', raw).strip() # Clean any leftover $[...] template markers title = re.sub(r'<\$[^>]+>', '', title).strip() if not title: m = re.search(r'(.*?)', html) if m: t = m.group(1).replace('兖州区人民政府建设项目环境影响评价 ', '').strip() title = t if t and len(t) > 5 else "" # Date:

发布日期： publish_date = "" m = re.search(r'发布日期：\s*(\d{4}-\d{2}-\d{2})', html) if m: publish_date = m.group(1) if not publish_date: m = re.search(r"]+Maketime[^>]+content=['\"](\d{4}-\d{2}-\d{2})", html) if m: publish_date = m.group(1) if not publish_date: m = re.search(r'成文日期[^>]*>([^<]*\d{4}-\d{2}-\d{2})', html) if m: publish_date = m.group(1).strip()[-10:] # Content:

...

content = "" m = re.search(r'

(.*?)

\s*

]*>', '', raw) raw = re.sub(r'', '', raw, flags=re.DOTALL) # Preserve tables: convert to text format before stripping def table_to_text(tbl): rows = re.findall(r']*>.*?', tbl, re.DOTALL) lines = [] for row in rows: cells = re.findall(r']*>(.*?)', row, re.DOTALL) # Clean each cell text cell_texts = [] for c in cells: ct = re.sub(r'<[^>]+>', '', c) ct = ct.replace(' ', ' ').strip() cell_texts.append(ct) lines.append(' | '.join(cell_texts)) # Add Markdown table separator after header if there are at least 2 rows if len(lines) >= 2: header_cells = lines[0].split(' | ') sep = ' | '.join(['---'] * len(header_cells)) lines.insert(1, sep) return '\n'.join(lines) raw = re.sub(r']*>.*?

', lambda m: '\n' + table_to_text(m.group()) + '\n', raw, flags=re.DOTALL) # Replace

with double newline for paragraph breaks raw = re.sub(r'

\s*]*>', '\n\n', raw) # Replace
with newline raw = re.sub(r'', '\n', raw) # Strip remaining HTML tags content = re.sub(r'<[^>]+>', '', raw) # Decode entities content = content.replace(' ', ' ').replace('&', '&').replace('<', '<').replace('>', '>') # Normalize whitespace content = re.sub(r'[ \t]+', ' ', content) content = re.sub(r'\n{3,}', '\n\n', content) content = content.strip() # Source source = "" m = re.search(r'信息来源[：:]<[^>]*>[^<]*begin-->(.*?)]*>([^<]+)', html) if m and not source: source = m.group(1).strip() return { "title": title, "date": publish_date, "content": content, "source": source, } except Exception as e: print(f"[ERROR] fetch_detail: {e}") return None def main(): init_session() all_items = [] total_pages = 1 # Fetch page 1 to get total pages items, total_pages = fetch_list(1) if not items: print("[ERROR] No items fetched") sys.exit(1) all_items.extend(items) print(f"[INFO] Page 1: {len(items)} items, total pages: {total_pages}") # Fetch remaining pages up to MAX_PAGES pages_to_fetch = min(total_pages, MAX_PAGES) for page in range(2, pages_to_fetch + 1): items, _ = fetch_list(page) if items: all_items.extend(items) print(f"[INFO] Page {page}: {len(items)} items") time.sleep(0.5) print(f"[INFO] Total items: {len(all_items)}") batch = [] for idx, (title_from_list, url) in enumerate(all_items): detail = fetch_detail(url) if detail is None: continue title = detail["title"] or title_from_list publish_date = detail["date"] content = detail["content"] source = detail.get("source", "兖州区生态环境局") if not content: # Attach embed for short content content = f"[无正文内容] {title}" batch.append({ "url": url, "title": title, "content": content, "pub_date": publish_date, "site_name": "兖州区人民政府-建设项目环境影响评价", "source_url": url, }) print(f" [{idx+1}] {title[:45]}... {publish_date}") time.sleep(0.3) if batch: result = push_to_searchdb(batch) print(f"\n[DONE] 完成: {len(batch)} 条") else: print("[DONE] 无数据") if __name__ == "__main__": main()