{inner}
") else: parts.append(f"{txt}
") for table in content_div.find_all("table"): if len(table.get_text(strip=True)) >= 10: parts.append(table_to_html(table)) # 附件独立段 (放末尾) seen = set() for text, url in attach_links: if url and url not in seen: seen.add(url) parts.append(f'') # 过滤噪声 filtered = [] for p in parts: if any(sk in p for sk in SKIP_TEXTS): continue if re.match(r"^(来源|发布日期)[：:]\s*", p): continue filtered.append(p) return "\n\n".join(filtered) def fetch_detail(url, list_title=""): html = fetch(url) if not html: return None, None, None soup = BeautifulSoup(html, "html.parser") title = "" mt = soup.find("meta", attrs={"name": "ArticleTitle"}) if mt and mt.get("content"): title = mt["content"].strip() if not title: nt = soup.find("div", class_="news-title") if nt: h2 = nt.find("h2") if h2: title = h2.get_text(strip=True) if not title: title = list_title if not title: ttag = soup.find("title") if ttag: title = ttag.get_text(strip=True) title = re.sub(r"\s*-\s*应城市生态环境局\s*$", "", title).strip() date_str = "" md = soup.find("meta", attrs={"name": "PubDate"}) if md and md.get("content"): date_str = md["content"].strip()[:10] if not date_str: dm = re.search(r"发布日期[：:](\d{4}-\d{1,2}-\d{1,2})", html) if dm: date_str = dm.group(1) content_div = soup.find("article", class_="htmledit_views") if not content_div: content_div = soup.find("div", class_="htmledit_views") if not content_div: content_div = soup.find("div", id="js_contentBox") if not content_div: content_div = soup.find("div", class_="news-box") content = "" if content_div: content = extract_block(content_div, url) # Fallback if len(content.strip()) < 20: content = f'
' return title, date_str, content def extract_list_page(page_num): if page_num == 1: url = f"{BASE_URL}{LIST_PATH}.jhtml" else: url = f"{BASE_URL}{LIST_PATH}_{page_num}.jhtml" html = fetch(url) if not html: print(f" Page {page_num} fetch fail") return [] soup = BeautifulSoup(html, "html.parser") articles = [] for card in soup.find_all("div", class_="card"): body = card.find("div", class_="card-body") if not body: continue ul = body.find("ul", class_="news-list") if not ul: continue for li in ul.find_all("li"): a = li.find("a", class_="news-title") if not a: continue href = a.get("href", "") if not href: continue title = a.get("title", "") or a.get_text(strip=True) if not title or len(title) < 5: continue full_url = abs_url(href) date_span = li.find("span", class_="news-time") date = date_span.get_text(strip=True) if date_span else "" articles.append((title, full_url, date)) break return articles def main(): max_pages = MAX_PAGES args = sys.argv[1:] if "--pages=" in " ".join(args): max_pages = int(re.search(r"--pages=(\d+)", " ".join(args)).group(1)) elif args and args[0].isdigit(): max_pages = int(args[0]) print(f"Scraping {SITE_NAME}, max pages: {max_pages}", flush=True) conn = sqlite3.connect(DB_PATH, timeout=60) conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA busy_timeout=15000") total_new = total_found = 0 for pn in range(1, max_pages + 1): articles = extract_list_page(pn) if not articles: print(f" Page {pn}: 0 articles (end)", flush=True) break total_found += len(articles) for title, page_url, date in articles: if date and date < CUTOFF: continue cur = conn.execute("SELECT id FROM gov_raw WHERE page_url = ?", (page_url,)) if cur.fetchone(): continue det_title, det_date, det_content = fetch_detail(page_url, title) final_title = det_title or title final_date = det_date or date final_content = det_content or f'' summary = re.sub(r"\s+", " ", final_content[:200]).strip() try: cur = conn.execute( """INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, category, group_name, script_name) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (SITE_NAME, page_url, page_url, final_title.strip(), final_date, summary, final_content, CATEGORY, GROUP, "crawl_yingcheng_ywgz.py"), ) conn.commit() if cur.rowcount > 0: total_new += 1 except sqlite3.OperationalError as e: print(f" DB error: {e}", file=sys.stderr) time.sleep(0.3) print(f" Page {pn}: {len(articles)} found, {total_new} new so far", flush=True) conn.commit() conn.close() print(f"\nDone! Total found: {total_found}, New: {total_new}", flush=True) if __name__ == "__main__": main()