#!/usr/bin/env python3 """Crawler for 淮南市大通区人民政府 - 通知公告 v2 (FIXED encoding)""" import sys, os, re, json, time, subprocess sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import requests from bs4 import BeautifulSoup # 支持 --pages 参数 import argparse _AP = argparse.ArgumentParser() _AP.add_argument("--pages", type=int, default=0, help="限制页数") _ARGS, _ = _AP.parse_known_args() SITE_NAME = "淮南市大通区人民政府-通知公告" BASE = 'https://www.hndt.gov.cn' COLUMN_ID = '6787542' PAGE_COUNT = _ARGS.pages if _ARGS.pages > 0 else 2 # 默认2页 HEADERS = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', } def get_page(url): """GET with forced UTF-8 — server omits charset in Content-Type""" r = requests.get(url, headers=HEADERS, timeout=30) r.encoding = 'utf-8' # CRITICAL: server sends text/html without charset return r.text def extract_list(html): soup = BeautifulSoup(html, 'html.parser') items = [] ul = soup.find('ul', class_=re.compile(r'doc_list')) if not ul: return items for li in ul.find_all('li', recursive=False): a = li.find('a') if not a: continue href = a.get('href', '') if not href: continue title = a.get('title', '') or a.get_text(strip=True) date_span = li.find('span', class_='date') date = date_span.get_text(strip=True) if date_span else '' if href.startswith('/'): href = BASE + href elif not href.startswith('http'): href = BASE + '/' + href.lstrip('/') items.append({'url': href, 'title': title, 'date': date}) return items def parse_detail(html): soup = BeautifulSoup(html, 'html.parser') title = '' date = '' content = '' attachments = [] h1 = soup.find('h1', class_='newstitle') if h1: title = h1.get_text(strip=True) if not title: meta = soup.find('meta', attrs={'name': 'ArticleTitle'}) if meta: title = meta.get('content', '') date_span = soup.find('span', class_='sp') if date_span: m = re.search(r'发布时间[：:]\s*(\d{4}-\d{2}-\d{2})', date_span.get_text()) if m: date = m.group(1) if not date: meta = soup.find('meta', attrs={'name': 'PubDate'}) if meta: pub = meta.get('content', '') m = re.search(r'(\d{4}-\d{2}-\d{2})', pub) if m: date = m.group(1) content_div = soup.find('div', class_=re.compile(r'wzcon')) if not content_div: content_div = soup.find('div', class_=re.compile(r'j-fontContent')) if content_div: for a_tag in content_div.find_all('a'): ahref = a_tag.get('href', '') if re.search(r'\.(doc|docx|pdf|xls|xlsx|zip|rar|txt)(\?|$)', ahref, re.I): atext = a_tag.get_text(strip=True) or ahref.split('/')[-1] if not ahref.startswith('http'): ahref = BASE + '/' + ahref.lstrip('/') a_tag.replace_with(f'[{atext}]({ahref})') attachments.append({'title': atext, 'url': ahref}) texts = [] for child in content_div.children: if hasattr(child, 'name') and child.name == 'table': texts.append('\n' + str(child) + '\n') elif hasattr(child, 'get_text'): txt = child.get_text('\n', strip=False) txt = re.sub(r'\s+', ' ', txt).strip() if txt: texts.append(txt) elif isinstance(child, str): txt = child.strip() if txt: texts.append(txt) content = '\n\n'.join(texts) content = re.sub(r'\n{3,}', '\n\n', content).strip() return title, date, content, json.dumps(attachments, ensure_ascii=False) if attachments else '' def main(): all_items = [] for page in range(1, PAGE_COUNT + 1): url = f'{BASE}/zwgk/tzgg/index.html' if page == 1 else f'{BASE}/content/column/{COLUMN_ID}?pageIndex={page}' print(f'[页码 {page}/{PAGE_COUNT}] {url}') try: items = extract_list(get_page(url)) if not items: print(' → 无数据，停止') break print(f' → 提取 {len(items)} 条') all_items.extend(items) except Exception as e: print(f' → 失败: {e}') continue time.sleep(0.5) print(f'\n共 {len(all_items)} 条') output_path = '/tmp/hndt_v2_import.jsonl' count = 0 with open(output_path, 'w', encoding='utf-8') as f: for i, item in enumerate(all_items, 1): print(f'[{i}/{len(all_items)}] {item["title"][:40] if item["title"] else item["url"][:30]}...', end=' ', flush=True) try: html = get_page(item['url']) title, date, content, attachments = parse_detail(html) record = { 'title': title or item['title'], 'page_url': item['url'], 'content': content, 'publish_date': date or item['date'], 'summary': title or item['title'], 'site_name': SITE_NAME, 'tags': '通知公告', 'attachments': attachments } f.write(json.dumps(record, ensure_ascii=False) + '\n') count += 1 print('✓') except Exception as e: print(f'✗ {e}') f.write(json.dumps({ 'title': item['title'], 'page_url': item['url'], 'content': '', 'publish_date': item['date'], 'summary': item['title'], 'site_name': SITE_NAME, 'tags': '通知公告', 'attachments': '' }, ensure_ascii=False) + '\n') count += 1 time.sleep(0.3) print(f'\n✅ JSONL: {output_path} ({count} 条) — 确认前3条编码:') with open(output_path, 'r') as f: for i, line in enumerate(f): if i >= 3: break rec = json.loads(line) print(f' {i+1}. {rec["title"][:50]}') # Delete old garbled data print('\n🗑️ 删除旧乱码数据...') subprocess.run(['systemctl', 'stop', 'search_app'], capture_output=True) time.sleep(1) import sqlite3 db = sqlite3.connect('/mnt/data/search.db') deleted = db.execute("DELETE FROM gov_raw WHERE page_url LIKE '%hndt.gov.cn%'").rowcount db.commit() db.close() print(f' 删了 {deleted} 条') # Import print('\n📥 导入新数据...') r = subprocess.run(['python3', '/root/gov_crawler/import_jsonl.py', output_path], capture_output=True, text=True, timeout=300, cwd='/mnt/data') print(r.stdout[-400:] if len(r.stdout) > 400 else r.stdout) # Rebuild FTS print('\n🔨 重建 FTS...') db = sqlite3.connect('/mnt/data/search.db') db.execute('DELETE FROM gov_search') db.execute('DELETE FROM gov_search_v3') db.execute("INSERT INTO gov_search(gov_search) VALUES('rebuild')") db.execute("INSERT INTO gov_search_v3(title, site_name, source_url, publish_date) " "SELECT title, site_name, page_url, publish_date FROM gov_raw") db.commit() db.close() subprocess.run(['systemctl', 'start', 'search_app'], capture_output=True, timeout=30) print('✅ 搜索服务已重启') print(f'\n✅ 完成！{count} 条，编码已修复') if __name__ == '__main__': main()