body = "" zoom_idx = html.find('id="zoom"') if zoom_idx >= 0: start = html.find('>', zoom_idx) + 1 depth = 1 i = start while i < len(html) and depth > 0: if html[i:i+4] == '': depth -= 1 i += 6 else: i += 1 raw = html[start:i-6] if depth == 0 else html[start:start+5000] if raw.strip(): body = clean_content_html(raw.strip()) if not body or len(body) < 20: body = "" return title, publish_date, body except Exception as e: print(f" [错误] 详情页 {detail_url}: {e}", flush=True) return "", "", "" def clean_content_html(html_content): """清洗正文 HTML""" if not html_content: return "" soup = BeautifulSoup(html_content, "html.parser") # 处理图片路径 for img in soup.find_all("img"): src = img.get("src", "") if src and not src.startswith("http"): img["src"] = urljoin(BASE_URL, src) # 处理附件链接 for a in soup.find_all("a"): href = a.get("href", "") if href and not href.startswith("http") and not href.startswith("#") and not href.startswith("javascript"): a["href"] = urljoin(BASE_URL, href) result = str(soup) # 清理多余样式标签 result = re.sub(r'\sstyle="[^"]*"', '', result) result = re.sub(r']*>|', '', result) result = re.sub(r']*>|', '', result) result = re.sub(r']*>|', '', result) result = re.sub(r']*>|', '', result) result = re.sub(r'\n{3,}', '\n\n', result) return result.strip() def crawl(max_pages=None): db = sqlite3.connect(DB_PATH, timeout=60) db.execute("PRAGMA journal_mode=WAL") db.execute("PRAGMA synchronous=NORMAL") existing = set() cur = db.execute("SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)) for row in cur: existing.add(row[0]) print(f"已有 {len(existing)} 条 {SITE_NAME} 记录", flush=True) first_items = fetch_list(1) if not first_items: print("无法获取列表数据", flush=True) db.close() return 0 # 总量 ~60 条, 6 页 total_pages = 6 pages_to_fetch = min(max_pages, total_pages) if max_pages else 5 print(f"共约 {total_pages} 页, 爬取前 {pages_to_fetch} 页", flush=True) total_new = 0 total_skip = 0 total_error = 0 for page in range(1, pages_to_fetch + 1): print(f"[第 {page}/{pages_to_fetch} 页] ", end="", flush=True) items = first_items if page == 1 else fetch_list(page) if not items: print("无数据", flush=True) continue page_new = 0 for item in items: full_url = item["url"] if full_url in existing: total_skip += 1 continue title = item["title"] publish_date = item["date"] if not title: total_skip += 1 continue # 获取详情（补充标题/日期/正文） det_title, det_date, body = fetch_detail(full_url) final_title = det_title if det_title else title if det_date and not publish_date: publish_date = det_date # 摘要 summary = re.sub(r'<[^>]+>', ' ', body) if body else final_title summary = re.sub(r'\s+', ' ', summary).strip()[:300] has_table = 1 if body and ' 0: row = db.execute("SELECT id FROM gov_raw WHERE page_url=?", (full_url,)).fetchone() if row: rid = row[0] try: db.execute("INSERT INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)", (rid, final_title[:500], SITE_NAME, summary[:200])) except sqlite3.IntegrityError: pass existing.add(full_url) db.commit() total_new += 1 page_new += 1 else: total_skip += 1 except Exception as e: print(f"x", end="", flush=True) total_error += 1 print(f"+{page_new} 条 (累计 {total_new})", flush=True) if page < pages_to_fetch: time.sleep(0.5) db.close() print(f"\n完成! 新增 {total_new}, 跳过 {total_skip}, 错误 {total_error}", flush=True) return total_new if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--max-pages", type=int, default=None) args = parser.parse_args() crawl(max_pages=args.max_pages)

inside .article_intro title = "" m = re.search(r']*>\s*(.*?)\s*

inside .article_intro title = "" m = re.search(r']>\s(.?)\s