#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ 江苏索普集团-环境信息 爬虫 Custom PHP CMS, static HTML List: /list/8-40-N.html (N=2..9), page 1 is wrong URL Detail: /info/8-40-XXXXX.html """ import re, time, os from urllib.parse import urljoin import requests from bs4 import BeautifulSoup, NavigableString BASE_URL = "http://www.sopo.com.cn" SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"} def safe_get(url, timeout=15): for attempt in range(3): try: r = requests.get(url, headers=HEADERS, timeout=timeout) r.encoding = 'utf-8' return r except Exception as e: if attempt < 2: time.sleep(2) else: print(f" [WARN] Failed {url}: {e}") return None def extract_content_from_info(info_div, page_url): """Extract content from #my-info div, preserving tables and attachments.""" parts = [] for child in info_div.children: name = getattr(child, 'name', None) if name == 'table': parts.append(str(child)) parts.append("\n\n") elif name == 'center': inner_table = child.find('table') if inner_table: parts.append(str(inner_table)) else: t = child.get_text(' ', strip=True) if t: parts.append(t) parts.append("\n\n") elif name in ('p', 'div'): inner_table = child.find('table') if inner_table: # Parse paragraph text and table separately txt_parts = [] for sub in child.children: if getattr(sub, 'name', None) == 'table': parts.append(str(sub)) parts.append("\n\n") else: t = sub.get_text(' ', strip=True) if hasattr(sub, 'get_text') else str(sub).strip() if t and len(t) > 3: txt_parts.append(t) if txt_parts: parts.append(' '.join(txt_parts)) parts.append("\n\n") else: # Strip inline tags from HTML to avoid number splits child_html = str(child) child_html = re.sub( r']*>', '', child_html, flags=re.I ) clean = BeautifulSoup(child_html, 'html.parser') txt = clean.get_text(' ', strip=True) txt = re.sub(r'\s+', ' ', txt).strip() if txt else "" # Check images (even when text is empty, e.g. pure image pages) for img in child.find_all('img'): src = img.get('src', '') if src: alt = img.get('alt', '') or "image" parts.append(f"![{alt}]({urljoin(page_url, src)})\n\n") if txt: parts.append(txt) parts.append("\n\n") elif name == 'h1': txt = child.get_text(' ', strip=True) if txt: parts.append(f"**{txt}**\n\n") elif name == 'h2': txt = child.get_text(' ', strip=True) if txt: parts.append(f"## {txt}\n\n") elif name == 'a': href = child.get('href', '') text = child.get_text(' ', strip=True) if "upload" in href.lower() or any(href.lower().endswith(e) for e in ('.doc', '.docx', '.pdf', '.xls', '.xlsx')): parts.append(f"[{text}]({urljoin(page_url, href)})\n\n") elif name == 'ul': for li in child.find_all('li', recursive=False): li_text = li.get_text(' ', strip=True) if li_text: parts.append(f"- {li_text}\n") parts.append("\n") elif name == 'ol': for li in child.find_all('li', recursive=False): li_text = li.get_text(' ', strip=True) if li_text: parts.append(f" {li_text}\n") parts.append("\n") elif name is None: t = str(child).strip() if t and len(t) > 5: parts.append(t) parts.append("\n\n") elif name == 'style': continue # Skip result = ''.join(parts).strip() result = re.sub(r'\n{4,}', '\n\n', result) result = re.sub(r' {3,}', ' ', result) return result def parse_detail(url): r = safe_get(url) if not r: return None, None, None soup = BeautifulSoup(r.text, "html.parser") # Title from div.my-title title = "" mt = soup.find("div", class_="my-title") if mt: title = mt.get_text(strip=True) # Author/date div date = "" ma = soup.find("div", class_="my-author") if ma: m = re.search(r'(\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2})', ma.get_text()) if m: date = m.group(1).replace("/", "-") # Content from div#my-info info = soup.find("div", id="my-info") content = "" if info: content = extract_content_from_info(info, url) else: content = soup.get_text('\n\n', strip=True) return title, content, date def parse_list_page(url): r = safe_get(url) if not r: return [] soup = BeautifulSoup(r.text, "html.parser") items = [] for dl in soup.find_all("dl", class_="clearfix"): a = dl.select_one("dd .title a[href]") if not a: continue title = a.get_text(strip=True) href = a["href"].strip() if len(title) < 3: continue # Normalize URL if href.startswith("/"): href = urljoin(BASE_URL, href) elif not href.startswith("http"): href = urljoin(url, href) # Date date = "" date_p = dl.select_one("dd .title p:last-child") if date_p: date = date_p.get_text(strip=True) if not date: m = re.search(r'(\d{4}-\d{1,2}-\d{1,2})', dl.get_text()) if m: date = m.group(1) items.append((title, href, date)) return items def main(): import sqlite3 db_paths = ["/root/search.db", "/root/gov_crawler/search.db", os.path.join(SCRIPT_DIR, "search.db")] db_path = None for p in db_paths: if os.path.exists(p): db_path = p break if not db_path: db_path = "/root/search.db" print(f"=== 江苏索普集团-环境信息 ===") # Fetch all list pages all_items = [] # Page 2 (page 1 shows wrong content) url2 = f"{BASE_URL}/list/8-40-2.html" print(f" Fetching page 2: {url2}") items = parse_list_page(url2) print(f" Found {len(items)} items") all_items.extend(items) # Pages 3-9 via pagination pattern for p in range(3, 10): url = f"{BASE_URL}/list/index/id/8-40-2/m/Home/p/{p}.html" print(f" Fetching page {p}: {url}") items = parse_list_page(url) if not items: print(f" Empty, stopping") break print(f" Found {len(items)} items") all_items.extend(items) time.sleep(0.3) print(f"\n Total: {len(all_items)} items\n") if not all_items: print("No items found.") return # Save items list for debug with open("/tmp/sopo_items.txt", "w") as f: for t, h, d in all_items: f.write(f"[{d}] {t}\n {h}\n\n") db = sqlite3.connect(db_path, timeout=60) db.execute("PRAGMA journal_mode=WAL") db.execute("PRAGMA synchronous=OFF") site_name = "江苏索普集团-环境信息" inserted = 0 skipped = 0 errors = 0 for idx, (title, href, list_date) in enumerate(all_items): existing = db.execute("SELECT id FROM gov_raw WHERE page_url = ?", (href,)).fetchone() if existing: skipped += 1 continue detail_title, content, detail_date = parse_detail(href) if not content: errors += 1 if errors > 5: print(" Too many errors, stopping") break print(f" [WARN] Empty content: {title[:50]}") continue final_title = detail_title or title final_date = detail_date or list_date content = content[:500000] if len(content) > 500000 else content try: db.execute( "INSERT OR IGNORE INTO gov_raw (title, content, page_url, publish_date, site_name) VALUES (?, ?, ?, ?, ?)", (final_title, content, href, final_date, site_name) ) db.commit() inserted += 1 except Exception as e: print(f" [DB ERROR] {e}") db.rollback() if (idx + 1) % 10 == 0: print(f" Progress: {idx+1}/{len(all_items)} (inserted={inserted}, skipped={skipped})") time.sleep(0.3) # Bulk FTS rebuild print("\nRebuilding FTS...") db.close() print(f"\n========================================") print(f"Inserted: {inserted}/{len(all_items)} new") print(f"Skipped: {skipped}") print(f"Errors: {errors}") if __name__ == "__main__": main()