/![]()
, 剥内联样式规范化
body_parts = []
for el in zoom.find_all(["p", "table", "img"]):
if el.name == "p":
if el.find_parent("table") or el.find("table") or el.find("p"):
continue
txt = el.get_text(" ", strip=True)
txt = re.sub(r"\s+", " ", txt)
if txt:
body_parts.append(f"
{txt}
")
elif el.name == "table":
body_parts.append(str(el))
elif el.name == "img":
src = el.get("src", "")
if src:
full = urljoin(page_url, src)
alt = el.get("alt", "")
body_parts.append(f'
')
# 附件段落
for att in attachments:
body_parts.append(f'{att["name"]}
')
content = "\n".join(body_parts)
# 兜底: 无 #zoom 时用 sx_Textcontent 或纯文本
if not content:
other = soup.find(class_="sx_Textcontent")
if other:
ps = []
for el in other.find_all(["p", "table", "img"]):
if el.name == "p" and not el.find_parent("table") and not el.find("table") and not el.find("p"):
txt = el.get_text(" ", strip=True)
if txt:
ps.append(f"{txt}
")
elif el.name == "table":
ps.append(str(el))
elif el.name == "img" and el.get("src"):
ps.append(f'
')
content = "\n".join(ps)
if not content:
txt = soup.get_text(" ", strip=True)
content = f"{txt}
" if txt else ""
return title, date, content, attachments
def main():
max_pages = parse_pages_arg()
if max_pages is None:
max_pages = 60 # 默认全量(219条/20=11页)
print(f"[AutoPg] max_pages={max_pages} (SITE={SITE_NAME})")
items = []
seen_urls = set()
for pg in range(1, max_pages + 1):
html = fetch_list_page(pg)
if not html or len(html) < 50:
print(f" Page {pg}: 空页, 停止")
break
page_items = parse_list_items(html)
new_items = [x for x in page_items if x["url"] not in seen_urls]
for x in new_items:
seen_urls.add(x["url"])
items.extend(new_items)
print(f" Page {pg}: {len(page_items)}条(新{len(new_items)}) 累计{len(items)}")
# CUTOFF 检查: 列表日期 < CUTOFF 即停
if new_items:
dates = [x["date"] for x in new_items if x["date"]]
if dates and min(dates) < CUTOFF:
print(f" 已达 CUTOFF {CUTOFF}, 停止翻页")
break
time.sleep(0.5)
print(f"\n共 {len(items)} 条列表项, 开始抓详情...")
results = []
for i, item in enumerate(items):
title, date, content, attachments = extract_detail(item["url"])
if not title:
title = item["title"]
if not date:
date = item["date"]
if not content:
print(f" [WARN] 空正文: {title[:50]}")
results.append({
"site_name": SITE_NAME,
"title": title,
"pub_date": date,
"content": content,
"source_url": item["url"],
"url": item["url"],
"category": "行政许可",
"attachments": json.dumps(attachments, ensure_ascii=False) if attachments else "",
})
if (i + 1) % 20 == 0:
print(f" [{i+1}/{len(items)}] 最新: {title[:40]}")
time.sleep(0.3)
push_to_searchdb(results, batch_label=SITE_NAME)
print(f"\n{'='*50}")
print(f"站点: {SITE_NAME}")
print(f"采集条数: {len(results)}")
print(f"日期范围: {min((r['pub_date'] for r in results if r['pub_date']), default='-')} ~ {max((r['pub_date'] for r in results if r['pub_date']), default='-')}")
print(f"{'='*50}")
if __name__ == "__main__":
main()