#!/usr/bin/env python3 """crawl_ynfn_sthj.py - 富宁县-生态环境（数融UCAP CMS /queryList API）""" import sys, os, json, re, time, requests, warnings, argparse from datetime import datetime sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from crawler_lib import push_to_searchdb warnings.filterwarnings('ignore', category=requests.packages.urllib3.exceptions.InsecureRequestWarning) SITE_NAME = '富宁县-生态环境' API_URL = 'https://www.ynfn.gov.cn/queryList' API_HEADERS = {'Content-Type': 'application/json'} PAGE_SIZE = 15 def fetch_page(page_num): """Fetch a single page from the API""" payload = { "channelCode": ["ynfnxsthj"], "webSiteCode": ["fnxrmzf"], "current": page_num, "pageSize": PAGE_SIZE } r = requests.post(API_URL, json=payload, headers=API_HEADERS, timeout=30, verify=False) data = r.json() return data.get('data', {}) def clean_html_content(html_text): """Convert HTML to clean plain text with proper paragraph separation""" if not html_text: return '' from bs4 import BeautifulSoup soup = BeautifulSoup(html_text, 'html.parser') parts = [] seen_tables = set() for elem in soup.find_all(['p', 'table', 'img']): style = elem.get('style', '') or '' if 'display:none' in style.replace(' ', ''): continue if elem.name == 'p': if elem.find_parent('table') or elem.find('table'): continue texts = [] for br in elem.find_all('br'): br.replace_with('\n') for s in elem.stripped_strings: s = s.strip() if s and s != '\xa0': texts.append(s) if texts: text = '\n'.join(texts) text = re.sub(r'[ \t]+(?=[\u4e00-\u9fff])', '', text) text = re.sub(r'(?<=[\u4e00-\u9fff])[ \t]+', '', text) text = re.sub(r'\u00a0', ' ', text) parts.append(text) elif elem.name == 'table': table_key = str(elem)[:200] if table_key in seen_tables: continue seen_tables.add(table_key) md_table = html_table_to_html(elem) if md_table: parts.append(md_table) elif elem.name == 'img': src = elem.get('src', '') alt = elem.get('alt', '') if src and not src.startswith('data:'): if not src.startswith('http'): src = 'https://www.ynfn.gov.cn' + src parts.append(f'![{alt}]({src})') return '\n\n'.join(parts) def html_table_to_html(table, base_url=""): """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL""" from bs4 import BeautifulSoup tbl = BeautifulSoup(str(table), 'html.parser') for a in tbl.find_all('a'): href = a.get('href', '') if href and not href.startswith(('http', 'javascript', '#')): a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href for img in tbl.find_all('img'): src = img.get('src', '') if src and not src.startswith(('http', '//', 'data:')): img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src return str(tbl) def get_db_count(site_name): import sqlite3 try: db = sqlite3.connect('/root/search.db', timeout=10) cur = db.execute("SELECT COUNT(*) FROM gov_raw WHERE site_name=?", (site_name,)) cnt = cur.fetchone()[0] db.close() return cnt except: return -1 def run(args): max_pages = args.pages all_records = [] for pn in range(1, max_pages + 1): print(f" Fetching page {pn}...") try: data = fetch_page(pn) except Exception as e: print(f" API error page {pn}: {e}") break total = data.get('total', 0) results = data.get('results', []) if not results: print(f" No more results (total={total})") break print(f" Page {pn}: {len(results)} items (total={total})") for item in results: src = item.get('source', {}) title = src.get('title', '').strip() if not title: continue pub_date = src.get('pubDate', '')[:10] if not pub_date: pub_date = datetime.now().strftime('%Y-%m-%d') urls_str = src.get('urls', '{}') detail_url = '' try: urls_obj = json.loads(urls_str) if isinstance(urls_str, str) else urls_str detail_url = 'https://www.ynfn.gov.cn' + urls_obj.get('pc', '') except: pass content_html = src.get('content', {}).get('content', '') content = clean_html_content(content_html) attachments = [] article_files = src.get('articleFiles', '[]') try: files = json.loads(article_files) if isinstance(article_files, str) else article_files for f in files: fname = f.get('fileName', '') fpath = f.get('filePath', '') domain = f.get('domainName', 'https://www.ynfn.gov.cn') if fname and fpath: furl = domain + fpath attachments.append({'name': fname, 'url': furl}) content += f'\n\n[附件：{fname}]({furl})' except: pass all_records.append({ 'title': title, 'url': detail_url, 'source_url': detail_url, 'pub_date': pub_date, 'site_name': SITE_NAME, 'content': content, 'summary': content[:500] if content else title[:200], 'attachments': json.dumps(attachments, ensure_ascii=False) if attachments else '', }) if pn < max_pages: time.sleep(0.3) print(f"\n Total: {len(all_records)} items") if all_records: count_before = get_db_count(SITE_NAME) push_to_searchdb(all_records, "ynfn_sthj") count_after = get_db_count(SITE_NAME) print(f" DB: {count_before} → {count_after} (+{count_after - count_before})") return len(all_records) if __name__ == '__main__': parser = argparse.ArgumentParser(description='富宁县-生态环境爬虫') parser.add_argument('--pages', type=int, default=5, help='pages to crawl') args = parser.parse_args() print(f"富宁县-生态环境: crawling {args.pages} pages") cnt = run(args) print(f"Done: {cnt} records")