标签 tag = soup.find('title') if tag: t = tag.get_text(strip=True) # 去掉站点后缀 " - 伊吾县人民政府" / "_伊吾县人民政府" t = re.sub(r'[-—_].*?伊吾县人民政府.*$', '', t).strip() # 去掉尾部多余的站点名 t = re.sub(r'\s*[|｜_\-]\s*伊吾县人民政府$', '', t).strip() result['title'] = t # 降级：ucaptitle 标签 if not result.get('title'): uc = soup.find('ucaptitle') if uc: result['title'] = uc.get_text(strip=True) # 日期：优先 PUBLISHTIME 标签（实际发布时间），其次 meta PubDate pt = soup.find('publishtime') if pt: m = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', pt.get_text()) if m: result['publish_date'] = m.group(1) if not result.get('publish_date'): meta = soup.find('meta', attrs={'name': 'PubDate'}) if meta and meta.get('content'): m = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', meta['content']) if m: result['publish_date'] = m.group(1) # 正文：div.v_news_content content_div = soup.find('div', class_='v_news_content') if not content_div: # 降级：div#vsb_content_4 content_div = soup.find('div', id='vsb_content_4') if not content_div: # 降级：div.c51920_content content_div = soup.find('div', class_='c51920_content') if content_div: content_html = str(content_div) # 清理 script/style content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL | re.I) content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL | re.I) # 清理 UCAPCONTENT 标签（保留内部内容） content_html = re.sub(r'</?UCAPCONTENT[^>]*>', '', content_html, flags=re.I) result['content'] = content_html.strip() return result.get('title'), result.get('content'), result.get('publish_date') def main(): print(f"\n{'='*50}") print(f"🏠 {SITE_NAME}") print(f"{'='*50}") # 爬列表页 all_list = [] for page in range(1, MAX_PAGES + 1): if page == 1: url = LIST_URL else: url = PAGE_URL.replace('{n}', str(page)) print(f"\n📄 第 {page} 页 ({url})...", end=" ", flush=True) html = fetch(url) if not html: print("❌ 请求失败") break items = parse_list(html) if not items: print("0 条（可能已到底）") break print(f"✅ {len(items)} 条") all_list.extend(items) print(f"\n📊 列表总计: {len(all_list)} 条") # 爬详情 all_items, seen = [], set() for i, (title, url, list_date) in enumerate(all_list): if url in seen: continue seen.add(url) print(f" [{i+1}/{len(all_list)}] {title[:50]}...", end=" ", flush=True) dt, content, date = fetch_detail(url) final_title = dt or title final_date = date or list_date # 统一日期格式 YYYY-MM-DD if final_date: m = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', str(final_date)) if m: y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3)) final_date = f"{y:04d}-{mo:02d}-{d:02d}" else: final_date = "" # 纯文本摘要（剥离HTML标签） content_text = content or "" summary = re.sub(r'<[^>]+>', ' ', content_text).strip() summary = re.sub(r'\s+', ' ', summary)[:300] all_items.append({ "site_name": SITE_NAME, "title": final_title, "url": url, "content": content_text, "pub_date": final_date, "summary": summary, "tags": SITE_NAME, }) print("✅") time.sleep(0.3) # 推送 if all_items: push_to_searchdb(all_items, "xjyiwu_ghxx") else: print(" ⏭ 无数据，跳过推送") print(f"\n✅ 完成! 共 {len(all_items)} 条") if __name__ == "__main__": t0 = time.time() main() print(f"⏱ 耗时: {time.time()-t0:.1f}s")

#!/usr/bin/env python3 """ 伊吾县 - 规划信息 (xjyiwu.gov.cn) 列表: /xjyiwu/c123447/zfxxgk_gknrz.shtml 分页: zfxxgk_gknrz_N.shtml 详情: /xjyiwu/c123447/YYYYMM/uuid.shtml 正文: div.v_news_content """ import sys, os, re, time from datetime import datetime, timezone, timedelta import requests, urllib3 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from crawler_lib import push_to_searchdb urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) SITE_NAME = "伊吾县-规划信息" BASE_URL = "https://www.xjyiwu.gov.cn" LIST_URL = "https://www.xjyiwu.gov.cn/xjyiwu/c123447/zfxxgk_gknrz.shtml" PAGE_URL = "https://www.xjyiwu.gov.cn/xjyiwu/c123447/zfxxgk_gknrz_{n}.shtml" MAX_PAGES = 5 HEADERS = {"User-Agent": "Mozilla/5.0"} def fetch(url): try: r = requests.get(url, headers=HEADERS, timeout=20, verify=False) r.encoding = 'utf-8' return r.text except: return None def parse_list(html): """ 从列表页 HTML 提取 (title, url, date) 元组结构:

TITLEYYYY-MM-DD

""" from bs4 import BeautifulSoup items = [] soup = BeautifulSoup(html, 'html.parser') # 找到列表容器 div.gknr_list 内的 dl for dl in soup.find_all('dl'): for dd in dl.find_all('dd'): a = dd.find('a', href=True) span = dd.find('span') if a and span: href = a.get('href', '').strip() title = a.get_text(strip=True) date_text = span.get_text(strip=True) if not href or not title or len(title) < 3: continue # 相对路径转绝对 if not href.startswith('http'): href = BASE_URL + href # 只取属于当前栏目 c123447 的条目 # 日期验证 m = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', date_text) if m: y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3)) date_text = f"{y:04d}-{mo:02d}-{d:02d}" else: date_text = "" items.append((title, href, date_text)) if items: break # 找到即退出 return items def fetch_detail(url): """访问详情页，返回 (title, content, pub_date)""" html = fetch(url) if not html: return None, None, None from bs4 import BeautifulSoup soup = BeautifulSoup(html, 'html.parser') result = {} # 标题：优先标签 tag = soup.find('title') if tag: t = tag.get_text(strip=True) # 去掉站点后缀 " - 伊吾县人民政府" / "_伊吾县人民政府" t = re.sub(r'[-—_].*?伊吾县人民政府.*$', '', t).strip() # 去掉尾部多余的站点名 t = re.sub(r'\s*[|｜_\-]\s*伊吾县人民政府$', '', t).strip() result['title'] = t # 降级：ucaptitle 标签 if not result.get('title'): uc = soup.find('ucaptitle') if uc: result['title'] = uc.get_text(strip=True) # 日期：优先 PUBLISHTIME 标签（实际发布时间），其次 meta PubDate pt = soup.find('publishtime') if pt: m = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', pt.get_text()) if m: result['publish_date'] = m.group(1) if not result.get('publish_date'): meta = soup.find('meta', attrs={'name': 'PubDate'}) if meta and meta.get('content'): m = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', meta['content']) if m: result['publish_date'] = m.group(1) # 正文：div.v_news_content content_div = soup.find('div', class_='v_news_content') if not content_div: # 降级：div#vsb_content_4 content_div = soup.find('div', id='vsb_content_4') if not content_div: # 降级：div.c51920_content content_div = soup.find('div', class_='c51920_content') if content_div: content_html = str(content_div) # 清理 script/style content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL | re.I) content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL | re.I) # 清理 UCAPCONTENT 标签（保留内部内容） content_html = re.sub(r'</?UCAPCONTENT[^>]*>', '', content_html, flags=re.I) result['content'] = content_html.strip() return result.get('title'), result.get('content'), result.get('publish_date') def main(): print(f"\n{'='*50}") print(f"🏠 {SITE_NAME}") print(f"{'='*50}") # 爬列表页 all_list = [] for page in range(1, MAX_PAGES + 1): if page == 1: url = LIST_URL else: url = PAGE_URL.replace('{n}', str(page)) print(f"\n📄 第 {page} 页 ({url})...", end=" ", flush=True) html = fetch(url) if not html: print("❌ 请求失败") break items = parse_list(html) if not items: print("0 条（可能已到底）") break print(f"✅ {len(items)} 条") all_list.extend(items) print(f"\n📊 列表总计: {len(all_list)} 条") # 爬详情 all_items, seen = [], set() for i, (title, url, list_date) in enumerate(all_list): if url in seen: continue seen.add(url) print(f" [{i+1}/{len(all_list)}] {title[:50]}...", end=" ", flush=True) dt, content, date = fetch_detail(url) final_title = dt or title final_date = date or list_date # 统一日期格式 YYYY-MM-DD if final_date: m = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', str(final_date)) if m: y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3)) final_date = f"{y:04d}-{mo:02d}-{d:02d}" else: final_date = "" # 纯文本摘要（剥离HTML标签） content_text = content or "" summary = re.sub(r'<[^>]+>', ' ', content_text).strip() summary = re.sub(r'\s+', ' ', summary)[:300] all_items.append({ "site_name": SITE_NAME, "title": final_title, "url": url, "content": content_text, "pub_date": final_date, "summary": summary, "tags": SITE_NAME, }) print("✅") time.sleep(0.3) # 推送 if all_items: push_to_searchdb(all_items, "xjyiwu_ghxx") else: print(" ⏭ 无数据，跳过推送") print(f"\n✅ 完成! 共 {len(all_items)} 条") if __name__ == "__main__": t0 = time.time() main() print(f"⏱ 耗时: {time.time()-t0:.1f}s")