, headings, and
for tag in content_div.find_all(["p", "h1", "h2", "h3", "h4", "h5", "h6"]):
tag.insert(0, MARKER)
tag.append(MARKER)
for br in content_div.find_all("br"):
br.replace_with(MARKER)
# 3) Unwrap inline tags with empty separator (preserves already-converted markdown)
for tag in content_div.find_all(["span", "b", "strong", "font", "em", "i", "u", "s"]):
tag.unwrap()
# 4) Extract text - markdown links survive get_text() as text
text = content_div.get_text(separator="", strip=True)
# 5) Clean up markers
text = re.sub(r"\x00P\x00(\s*\x00P\x00)+", "\x00P\x00", text)
text = text.replace("\x00P\x00", "\n\n")
text = re.sub(r"\n{3,}", "\n\n", text)
text = text.strip()
# 6) Append tables at end
for tbl_html in tables_html:
text += f"\n\n{tbl_html}"
# 7) Empty content fallback
if not text.strip() or len(text.strip()) < 20:
if title:
text = f"[{title}]({detail_url})"
return text, attachments
def crawl_all(months_back=36):
"""Full crawl all pages."""
cutoff = datetime.now() - timedelta(days=months_back * 30) if months_back else None
html = fetch_list(1)
if not html:
log("[ERROR] 无法获取第一页")
return
total_pages = get_total_pages(html)
log(f"共 {total_pages} 页")
all_items = []
seen_urls = set()
for page in range(1, total_pages + 1):
if page > 1:
html = fetch_list(page)
if not html:
continue
items = parse_list(html)
new_count = 0
for item in items:
if item["url"] not in seen_urls:
if cutoff and item["date"]:
try:
d = datetime.strptime(item["date"], "%Y-%m-%d")
if d < cutoff:
continue
except ValueError:
pass
seen_urls.add(item["url"])
all_items.append(item)
new_count += 1
log(f" [PAGE {page}/{total_pages}] +{new_count} 条")
if new_count == 0 and page > 1:
log(" [STOP] no new items")
break
log(f"\n共 {len(all_items)} 条待爬详情")
for idx, item in enumerate(all_items, 1):
title, date_str, content, attachments = extract_detail(item["url"])
if not item["date"] and date_str:
item["date"] = date_str
if title:
item["title"] = title
item["content"] = content
item["attachments"] = attachments
output_item(item, idx)
if idx % 10 == 0:
log(f" [PROGRESS] {idx}/{len(all_items)}")
added, skipped = push_to_db(all_items)
log(f"[DB] 入库: 新增{added} 跳过{skipped}")
log(f"\n[DONE] 共爬取 {len(all_items)} 条")
def crawl_incremental():
"""Incremental: only first page."""
html = fetch_list(1)
if not html:
return
items = parse_list(html)
log(f"[LIST] page 1 -> {len(items)} 条")
for idx, item in enumerate(items, 1):
title, date_str, content, attachments = extract_detail(item["url"])
if not item["date"] and date_str:
item["date"] = date_str
if title:
item["title"] = title
item["content"] = content
item["attachments"] = attachments
output_item(item, idx)
added, skipped = push_to_db(items)
log(f"[DB] 入库: 新增{added} 跳过{skipped}")
log(f"[DONE] 增量爬取 {len(items)} 条")
def output_item(item, idx):
record = {
"title": item.get("title", ""),
"page_url": item.get("url", ""),
"publish_date": item.get("date", ""),
"content": item.get("content", ""),
"attachments": item.get("attachments", []),
"site_name": f"{SITE}-{COLUMN}",
"column": COLUMN,
"province": PROVINCE,
}
print(json.dumps(record, ensure_ascii=False))
def push_to_db(items):
"""INSERT OR IGNORE 直插 gov_raw; FTS 触发器 trg_gov_raw_fts_ins 自动同步 gov_search"""
if not items:
return 0, 0
if not os.path.exists(DB_PATH):
log(f" 未发现 {DB_PATH}, 跳过入库")
return 0, 0
conn = sqlite3.connect(DB_PATH, timeout=290)
conn.execute("PRAGMA busy_timeout=290000")
added = skipped = 0
cur = conn.cursor()
for it in items:
content = it.get("content", "") or ""
attachments = it.get("attachments", [])
att_json = json.dumps(attachments, ensure_ascii=False) if attachments else ""
url = it.get("url", "")
try:
cur.execute(
"""INSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments, group_name, industry)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""",
(f"{SITE}-{COLUMN}", url, url, it.get("title", ""), it.get("date", ""),
content, content[:500], COLUMN, "crawl_guixi_sthj_xxgk.py", att_json,
PROVINCE, "other")
)
if cur.rowcount > 0:
added += 1
else:
skipped += 1
except sqlite3.Error as e:
log(f" DB err: {e}")
skipped += 1
conn.commit()
conn.close()
return added, skipped
if __name__ == "__main__":
mode = sys.argv[1] if len(sys.argv) > 1 else "incremental"
if mode == "incremental":
crawl_incremental()
elif mode == "full":
months = int(sys.argv[2]) if len(sys.argv) > 2 else 36
crawl_all(months)
elif mode == "list":
html = fetch_list(1)
items = parse_list(html)
log(f"共 {len(items)} 条")
for it in items[:5]:
log(f" {it['date']} | {it['title'][:40]} | {it['url']}")
else:
log(f"Usage: {sys.argv[0]} [incremental|full [months]]")