#!/usr/bin/env python3 """漯河市郾城区人民政府 - 项目环评""" import re, sys, os, json import urllib.request, urllib.error from bs4 import BeautifulSoup import sqlite3 BASE_URL = "https://www.lhyc.gov.cn" LIST_PATH = "/zwgk/fdzdgknr/zdly/hjbh/xmhp" SITE_NAME = "漯河市郾城区-项目环评" SITE_DISPLAY = "漯河市郾城区人民政府" DB = os.environ.get("SEARCH_DB", "/root/search.db") HEADERS = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Accept": "text/html,application/xhtml+xml" } _MAX_PG = None for i, a in enumerate(sys.argv): if a == "--pages" and i + 1 < len(sys.argv): _MAX_PG = int(sys.argv[i + 1]) break def html_table_to_html(table, base_url=""): """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL""" import urllib.parse tbl = BeautifulSoup(str(table), 'html.parser') for a in tbl.find_all('a'): href = a.get('href', '') if href and not href.startswith(('http', 'javascript', '#')): a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href for img in tbl.find_all('img'): src = img.get('src', '') if src and not src.startswith(('http', '//', 'data:')): img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src return str(tbl) def fetch(url): req = urllib.request.Request(url, headers=HEADERS) return urllib.request.urlopen(req, timeout=30).read().decode() def fetch_with_referer(url): headers = dict(HEADERS) headers["Referer"] = "https://www.lhyc.gov.cn/" req = urllib.request.Request(url, headers=headers) return urllib.request.urlopen(req, timeout=30).read().decode() def parse_list(html): soup = BeautifulSoup(html, "html.parser") items = [] ul = soup.find("ul", class_="infoList") if not ul: return items for li in ul.find_all("li"): a = li.find("a", href=True) if not a: continue href = a["href"].strip() title = a.get_text(" ", strip=True) date_span = li.find("span", class_="date") date_str = date_span.get_text(" ", strip=True) if date_span else "" if href.startswith("/"): href = BASE_URL + href items.append({"title": title, "url": href, "date": date_str}) return items def get_page_count(html, current_url): """Get total number of pages from pagination links""" soup = BeautifulSoup(html, "html.parser") max_p = 1 for a in soup.find_all("a", href=True): href = a["href"] m = re.search(r"xmhp_(\d+)$", href) if m: p = int(m.group(1)) if p > max_p: max_p = p return max_p if max_p > 1 else None def parse_detail(html): soup = BeautifulSoup(html, "html.parser") # Title title_tag = soup.find("h2", class_="title") title = title_tag.get_text(" ", strip=True) if title_tag else "" if not title and soup.title: title = re.sub(r"_项目环评.*", "", soup.title.string.strip()).strip() # Date date_text = "" mc = soup.find("div", class_="mainContent") if mc: m = re.search(r"发布日期：(\d{4}-\d{2}-\d{2})", mc.get_text()) if m: date_text = m.group(1) # Content content_div = soup.find("div", class_="conTxt") if not content_div: return {"title": title, "date": date_text, "content": "", "attachments": []} parts = [] attachments = [] for child in content_div.find_all(["p", "table", "img"], recursive=True): if child.name == "p": if child.find_parent("table"): continue text = child.get_text(" ", strip=True) a_tag = child.find("a", href=True) if a_tag: ahref = a_tag["href"] if ahref.startswith("/"): ahref = BASE_URL + ahref if any(ahref.lower().endswith(ext) for ext in [".pdf", ".doc", ".docx", ".xls", ".xlsx", ".zip", ".rar"]): attachments.append({"name": a_tag.get_text(" ", strip=True) or ahref.split("/")[-1], "url": ahref}) if text: parts.append(text) elif child.name == 'table': tbl_html = html_table_to_html(child, BASE_URL) if tbl_html: parts.append(tbl_html) elif child.name == "img": src = child.get("src", "") alt = child.get("alt", "") if src: if src.startswith("/"): src = BASE_URL + src parts.append("![" + (alt or "image") + "](" + src + ")") content = "\n\n".join(parts) return {"title": title, "date": date_text, "content": content, "attachments": attachments} def main(): conn = sqlite3.connect(DB, timeout=60) c = conn.cursor() html = fetch_with_referer(BASE_URL + LIST_PATH) total_pages = get_page_count(html, LIST_PATH) if not total_pages: total_pages = 49 pages_to_fetch = min(total_pages, _MAX_PG) if _MAX_PG else total_pages print("Total pages:", total_pages, "Fetch:", pages_to_fetch) existing = set() for row in c.execute("SELECT page_url FROM gov_raw WHERE site_name=?", (SITE_NAME,)): existing.add(row[0]) print("Existing:", len(existing)) all_items = [] for pg in range(1, pages_to_fetch + 1): try: if pg == 1: pg_html = html url = BASE_URL + LIST_PATH else: url = BASE_URL + LIST_PATH + "_" + str(pg) pg_html = fetch_with_referer(url) items = parse_list(pg_html) print(" Page", pg, ":", len(items)) all_items.extend(items) except Exception as e: print(" Page", pg, "error:", e) print("Total items:", len(all_items)) new_count = 0 skip_count = 0 empty_count = 0 for item in all_items: url = item["url"] if url in existing: skip_count += 1 continue try: html = fetch_with_referer(url) detail = parse_detail(html) title = detail["title"] or item["title"] date_text = detail["date"] or item["date"] content = detail["content"] attachments = detail["attachments"] if not content: empty_count += 1 print(" WARN empty:", title[:30]) attachments_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]" text_only = re.sub(r"\s+", " ", content) text_only = re.sub(r"!\[.*?\]\(.*?\)", "", text_only) text_only = re.sub(r"\|.*?\|", "", text_only) summary = (text_only[:200] or title).strip() c.execute( "INSERT OR REPLACE INTO gov_raw (title, site_name, page_url, publish_date, content, summary, attachments, date_rank) VALUES (?,?,?,?,?,?,?,?)", (title, SITE_NAME, url, date_text, content, summary, attachments_json, date_text) ) conn.commit() new_count += 1 print(" OK", title[:30], "|", date_text) except Exception as e: print(" ERR", item["title"][:30], "|", e) print("\n" + "=" * 40) print(SITE_DISPLAY, "(", SITE_NAME, ")") print("New:", new_count, "Skip:", skip_count, "Empty:", empty_count, "Pages:", pages_to_fetch) conn.close() if __name__ == "__main__": main()