#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ crawl_njls_jrsgggs.py - 溧水区-招投标信息 CMS: TRS createPageHTML静态分页 列表: /jrs/gggs/index.html → /jrs/gggs/index_{N-1}.html (100页) ul.zlm_2020_dqlm_list > li > a[title] + p (date YYYY-MM-DD) 详情: /jrs/gggs/YYYYMM/t{id}.html 标题: div.lsxl_2020 (取"发布时间"前文本) 日期: div.release_time (发布时间：YYYY-MM-DD) 内容: div.view.TRS_UEDITOR 或 div.doc_con """ import requests import re import sys import os import time from bs4 import BeautifulSoup DB_PATH = "/root/search.db" BASE_URL = "http://www.njls.gov.cn" SITE_NAME = "溧水区-招投标信息" GROUP = "江苏" INDUSTRY = "其他" HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", } PAGE_TPL = BASE_URL + "/jrs/gggs/index_{page}.html" def get_session(): session = requests.Session() session.headers.update(HEADERS) return session def fetch_list_page(session, page_no): """Fetch list page. page_no 0-based: 0=index.html, 1=index_1.html, ...""" if page_no == 0: url = BASE_URL + "/jrs/gggs/" else: url = PAGE_TPL.format(page=page_no) try: r = session.get(url, timeout=30) r.encoding = "utf-8" return r.text except Exception as e: print(f" [ERROR] page {page_no}: {e}") return None def parse_list(html): """Extract (url, title, date) from list page""" items = [] soup = BeautifulSoup(html, "html.parser") ul = soup.find("ul", class_="zlm_2020_dqlm_list") if not ul: return items for li in ul.find_all("li", recursive=False): a = li.find("a") p_date = li.find_all("p") if not a or not a.get("href"): continue href = a.get("href", "").strip() title = a.get("title", "").strip() or a.get_text(strip=True) date_str = "" for p in p_date: m = re.match(r"(\d{4}-\d{2}-\d{2})", p.get_text(strip=True)) if m: date_str = m.group(1) break # Resolve relative URL if href.startswith("./"): href = BASE_URL + "/jrs/gggs/" + href[2:] elif href.startswith("/"): href = BASE_URL + href items.append((href, title, date_str)) return items def fetch_detail(session, url): try: r = session.get(url, timeout=30) r.encoding = "utf-8" return r.text except Exception as e: print(f"ERR:{e}") return None def extract_detail(html, url): soup = BeautifulSoup(html, "html.parser") # Title from div.lsxl_2020 title = "" title_div = soup.find("div", class_="lsxl_2020") if title_div: txt = title_div.get_text(strip=True) # Title is before "发布时间" idx = txt.find("发布时间") if idx > 0: title = txt[:idx].strip() else: title = txt if not title: h1 = soup.find("h1") if h1: title = h1.get_text(strip=True) # Date from div.release_time date_str = "" date_div = soup.find("div", class_="release_time") if date_div: m = re.search(r"(\d{4}-\d{2}-\d{2})", date_div.get_text()) if m: date_str = m.group(1) # Content from div.view.TRS_UEDITOR or div.doc_con content = "" content_div = (soup.find("div", class_="view") or soup.find("div", class_="doc_con")) if content_div: parts = [] for elem in content_div.find_all(["p", "table"], recursive=True): if elem.name == "p": if elem.find_parent("table"): continue text = elem.get_text(strip=True) if text and text not in ("\xa0", ""): parts.append(text) elif elem.name == "table": parts.append(str(elem)) content = "\n\n".join(parts) summary = re.sub(r"<[^>]+>", "", content)[:200] if content else "" return title, date_str, content, summary def save_to_db(items_data): import sqlite3 conn = sqlite3.connect(DB_PATH, timeout=10) c = conn.cursor() inserted = 0 fts_batch = [] for title, date_str, content, summary, page_url in items_data: try: c.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,)) if c.fetchone(): continue c.execute( """INSERT INTO gov_raw (page_url, title, site_name, publish_date, content, summary, source_url, category, industry, group_name, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (page_url, title, SITE_NAME, date_str, content, summary, page_url, "政府公告", INDUSTRY, GROUP, "crawl_njls_jrsgggs.py"), ) new_id = c.lastrowid if new_id: fts_batch.append((new_id, title, SITE_NAME, summary or "")) inserted += 1 except Exception as e: print(f" [DB ERROR] {title[:30]}: {e}") conn.commit() if fts_batch: for rid, title, site, summary in fts_batch: try: c.execute( "INSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)", (rid, title, site, summary), ) except Exception as e: print(f" [FTS ERROR] id={rid}: {e}") conn.commit() print(f" FTS同步: {len(fts_batch)}条") conn.close() return inserted def main(): import argparse parser = argparse.ArgumentParser(description=f"{SITE_NAME}爬虫") parser.add_argument("--pages", type=int, default=5, help="爬取页数") parser.add_argument("--full", action="store_true", help="全量爬取") args = parser.parse_args() max_pages = 999 if args.full else args.pages session = get_session() total_new = 0 for page_no in range(0, max_pages): print(f"[{time.strftime('%H:%M:%S')}] 第{page_no+1}页...", end=" ", flush=True) html = fetch_list_page(session, page_no) if not html: print("失败，停止") break items = parse_list(html) if not items: print("无数据，停止") break print(f"{len(items)}条", flush=True) batch = [] for href, title, date_str in items: detail_html = fetch_detail(session, href) if not detail_html: continue dt, dd, content, summary = extract_detail(detail_html, href) batch.append((dt or title, dd or date_str, content, summary, href)) time.sleep(0.3) saved = save_to_db(batch) total_new += saved print(f" 入库{saved}/{len(batch)}条 (累计{total_new})", flush=True) if not args.full and saved == 0 and len(batch) > 0: print(" 全部已存在，增量停止") break print(f"\n===== 完成 =====") print(f"新增入库: {total_new} 条") print(f"站点: {SITE_NAME}") if __name__ == "__main__": main()