([^<]+)

", zoom_start) + 1 depth = 1 pos = content_start while pos < len(html) and depth > 0: next_open = html.find("", pos) if next_close < 0: break if next_open >= 0 and next_open < next_close: depth += 1 pos = next_open + 5 else: depth -= 1 pos = next_close + 6 content = html[content_start:pos - 6].strip() if not content or len(content) < 20: return {"content": "", "error": "empty content"} # Convert relative URLs to absolute content = re.sub( r'(href|src)=([\'"])(/)(?![/])', lambda m: m.group(1) + "=" + m.group(2) + BASE_URL + m.group(3), content, ) return {"content": content, "full_title": full_title} def insert_db(items, cat_name): if not items: print(" 无数据插入") return 0 conn = sqlite3.connect(DB_PATH, timeout=60) c = conn.cursor() inserted = 0 for item in items: try: clean_text = re.sub(r'<[^>]+>', '', item["content"]).strip() c.execute( """INSERT OR IGNORE INTO gov_raw (title, page_url, site_name, summary, content, publish_date) VALUES (?, ?, ?, ?, ?, ?)""", (item["title"], item["url"], f"{SITE_NAME}-{cat_name}", clean_text[:500], item["content"], item["date"]), ) if c.rowcount > 0: inserted += 1 except Exception as e: print(f" DB insert error: {e}") conn.commit() conn.close() return inserted def rebuild_fts(): try: conn = sqlite3.connect(DB_PATH, timeout=60) c = conn.cursor() c.execute("SELECT 1 /* noop: gov_search 由触发器维护, 无需 rebuild */") conn.commit() conn.close() print(" FTS 重建完成") except Exception as e: print(f" FTS 重建失败: {e}") def main(): if len(sys.argv) < 2: print("Usage: python3 crawl_xingan.py [incremental_days]") sys.exit(1) catid = sys.argv[1] cat_name = get_cat_name(catid) is_incremental = len(sys.argv) >= 3 cutoff_days = int(sys.argv[2]) if is_incremental else 9999 print(f"=== 开始爬取 {SITE_NAME} - {cat_name} (catid={catid}) ===") if is_incremental: print(f"增量模式: 最近 {cutoff_days} 天") else: print("全量模式") post_data = build_post_data(catid) # Step 1: Get first page time.sleep(PAGE_DELAY) data = api_get(1, post_data) if not data or not data.get("total"): print(" ❌ 无法获取数据") return total = data["total"] items = data.get("data", []) print(f" 总记录: {total}, 第1页: {len(items)} 条") # Calculate total pages (20 per page) total_pages = (total + 19) // 20 print(f" 总页数: {total_pages}") # Step 2: Get remaining pages for page_no in range(2, total_pages + 1): time.sleep(PAGE_DELAY) data = api_get(page_no, post_data) if data and data.get("data"): items.extend(data["data"]) else: print(f" ⚠ 第{page_no}页空, 等待后重试...") time.sleep(5) data = api_get(page_no, post_data) if data and data.get("data"): items.extend(data["data"]) if page_no % 5 == 0: print(f" 已扫描 {page_no}/{total_pages} 页, {len(items)} 条") print(f" 列表扫描完成: {len(items)} 条") # Step 3: Filter by date (incremental mode) now = datetime.now() filtered = [] for item in items: date_str = item.get("inputtime", "") if not date_str: continue try: item_date = datetime.strptime(str(date_str)[:10], "%Y-%m-%d") except ValueError: continue days_diff = (now - item_date).days if days_diff <= cutoff_days: filtered.append({ "title": item.get("title", ""), "url": item.get("url", ""), "date": date_str[:10], "id": item.get("id"), }) print(f" 日期过滤后: {len(filtered)} 条") # Step 4: Check which already exist in DB if filtered: conn = sqlite3.connect(DB_PATH, timeout=60) c = conn.cursor() existing_urls = set() for item in filtered: c.execute("SELECT page_url FROM gov_raw WHERE page_url=?", (item["url"],)) row = c.fetchone() if row: existing_urls.add(item["url"]) conn.close() to_fetch = [it for it in filtered if it["url"] not in existing_urls] print(f" 需抓详情: {len(to_fetch)} 条 (已有 {len(filtered) - len(to_fetch)} 条)") if not to_fetch: print(" 无新增数据") return # Step 5: Fetch detail pages success = [] for i, item in enumerate(to_fetch): detail = fetch_detail(item["url"]) if detail.get("error"): if (i + 1) % 20 == 0: print(f" 详情进度: {i+1}/{len(to_fetch)} (成功: {len(success)})") continue success.append({ "title": detail.get("full_title") or item["title"], "url": item["url"], "date": item["date"], "content": detail["content"], }) if (i + 1) % 20 == 0: print(f" 详情进度: {i+1}/{len(to_fetch)} (成功: {len(success)})") print(f" 详情完成: 成功 {len(success)}/{len(to_fetch)}") # Step 6: Insert into DB inserted = insert_db(success, cat_name) print(f" 入库: {inserted} 新") # Step 7: Rebuild FTS rebuild_fts() print(f"=== 完成: {cat_name}, 新增 {inserted} 条 ===") if __name__ == "__main__": main()