与它内部的

，导致同一内容重复（容器那份常还被 get_text(strip=True) 拍平）。判据：某段被列表内另一段完全包含 → 它是容器段 → 剔除。保护：剔除后为空则原样返回。 """ if not parts: return parts ps = [p for p in parts if isinstance(p, str)] if len(ps) != len(parts): return parts # ① 先按值去重 —— 否则两段完全相同时 a in b 与 b in a 双向成立， # 会把两段都判成「容器段」剔光，再触发下面的保护而原样返回（等于没修）。 seen, uniq = set(), [] for p in parts: if p not in seen: seen.add(p); uniq.append(p) # ② 再剔容器段（被其它段完全包含的那个） keep = [a for a in uniq if not (len(a) >= 40 and any(b is not a and b and b in a for b in uniq))] return keep if keep else uniq def log(msg): print(f"[{SITE_NAME}] {msg}", flush=True) def fetch(url): result = subprocess.run(["curl", "-s", "-L", url, "-H", HEADERS], capture_output=True, text=True, timeout=25) if result.returncode != 0: log(f"fetch fail: {url[-60:]}") return None return result.stdout def extract_list(html): items = [] for m in re.finditer(r'

\s*]*>(.*?)\s*

', html, re.DOTALL): rel_url = m.group(1).strip() title = m.group(2).strip() full_url = urljoin(BASE_URL, rel_url) if full_url in seen_urls: continue seen_urls.add(full_url) pos = m.end() dm = re.search(r'发布时间：\s*\n\s*(\d{4}-\d{2}-\d{2})', html[pos:pos+300]) date_str = dm.group(1).strip() if dm else "" items.append((full_url, title, date_str)) return items def extract_detail(html, url): title = "" content = "" pub_date = "" m = re.search(r'ArticleTitle"\s*content="([^"]+)"', html) if m: title = m.group(1).strip() m = re.search(r'PubDate"\s*content="([^"]+)"', html) if m: pub_date = m.group(1).strip() soup = BeautifulSoup(html, "html.parser") zoomcon = soup.find(id="zoomcon") if zoomcon: parts = [] for p in zoomcon.find_all("p"): text = p.get_text(strip=True) if text: parts.append(text) if len(parts) < 2: parts = [] for child in zoomcon.find_all(["p", "div", "span"], recursive=True): text = child.get_text(strip=True) if text: parts.append(text) deduped = [] for t in parts: if not deduped or t != deduped[-1]: deduped.append(t) parts = deduped parts = _drop_container_parts(parts) content = "\n\n".join(parts) return title, content, pub_date def save_to_db(articles): conn = sqlite3.connect(DB_PATH, timeout=60) cursor = conn.cursor() inserted = 0 skipped = 0 for url, title, date, content in articles: try: cursor.execute( "INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, source_url, content) VALUES (?, ?, ?, ?, ?, ?)", (SITE_NAME, title, url, date, SOURCE, content or "")) if cursor.rowcount > 0: inserted += 1 else: skipped += 1 except Exception as e: log(f"db fail: {url[-50:]}: {e}") conn.commit() conn.close() return inserted, skipped def main(): pages = [BASE_URL] + [URL_TPL.replace("{i}", str(i)) for i in range(2, MAX_PAGES + 1)] total = [] for page_url in pages: log(f"list: {page_url}") html = fetch(page_url) if not html: continue items = extract_list(html) if not items: log(" no items, stop") break log(f" {len(items)} items") for url, title, date_str in items: if date_str < CUTOFF: log(f" skip(old): {date_str} {title[:40]}") continue time.sleep(random.uniform(0.3, 0.8)) dh = fetch(url) if not dh: continue dt, content, dd = extract_detail(dh, url) if not dt: dt = title if not dd: dd = date_str if not content: log(f" no content: {dt[:40]}") continue total.append((url, dt, dd, content)) time.sleep(0.3) ins, skip = save_to_db(total) log(f"done! new={ins} skip={skip} total={len(total)}") if __name__ == "__main__": main()