#!/usr/bin/env python3 """高唐县人民政府 - 环评审批信息""" import re, sys, os, json import urllib.request, urllib.error from bs4 import BeautifulSoup import sqlite3 BASE_URL = "http://www.gaotang.gov.cn" LIST_URL = BASE_URL + "/channel_x_0.0_8941/" SITE_NAME = "高唐县-环评审批信息" SITE_DISPLAY = "高唐县人民政府" GROUP_NAME = "环评平台" DB = os.environ.get("SEARCH_DB", "/root/search.db") HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"} _MAX_PG = None for i, a in enumerate(sys.argv): if a == "--pages" and i + 1 < len(sys.argv): _MAX_PG = int(sys.argv[i + 1]) break def html_table_to_html(table, base_url=""): """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL""" import urllib.parse tbl = BeautifulSoup(str(table), 'html.parser') for a in tbl.find_all('a'): href = a.get('href', '') if href and not href.startswith(('http', 'javascript', '#')): a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href for img in tbl.find_all('img'): src = img.get('src', '') if src and not src.startswith(('http', '//', 'data:')): img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src return str(tbl) def fetch(url): req = urllib.request.Request(url, headers=HEADERS) try: return urllib.request.urlopen(req, timeout=30).read().decode("utf-8", errors="replace") except Exception as e: print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr) return "" def extract_items(html): """Extract items from a list page""" soup = BeautifulSoup(html, "html.parser") items = [] for a in soup.select("div.gtxLbList a.box.clearfix"): title_p = a.find("p") if not title_p: continue title = title_p.get_text(strip=True) em = a.find("em") date_text = em.get_text(strip=True) if em else "" href = a.get("href", "").strip() if href and not href.startswith("http"): href = BASE_URL + href items.append({"title": title, "url": href, "date": date_text}) return items def get_total_pages(html): """Get total pages from laypage config""" m = re.search(r'count:\s*(\d+)', html) m2 = re.search(r'limit:\s*(\d+)', html) if m and m2: count = int(m.group(1)) limit = int(m2.group(1)) total = (count + limit - 1) // limit return total return None def parse_detail(html, url): """Parse detail page content""" soup = BeautifulSoup(html, "html.parser") # Title title = "" meta_title = soup.find("meta", attrs={"name": "ArticleTitle"}) if meta_title and meta_title.get("content"): title = meta_title["content"].strip() if not title and soup.title: title = soup.title.string.strip() title = re.sub(r"\s*\|\s*高唐县人民政府\s*$", "", title).strip() # Date date_text = "" meta_date = soup.find("meta", attrs={"name": "PubDate"}) if meta_date and meta_date.get("content"): m = re.search(r"(\d{4}-\d{2}-\d{2})", meta_date["content"]) if m: date_text = m.group(1) if not date_text: date_p = soup.select_one("div.gtxDetPart1 p:contains('发布日期')") if date_p: m = re.search(r"(\d{4}-\d{2}-\d{2})", date_p.get_text()) if m: date_text = m.group(1) # Content content_div = soup.select_one("div#sxNewsDetailContent") if not content_div: content_div = soup.select_one("div.sxNewsDetailContent") if not content_div: content_div = soup.select_one("div.content div.txt1") if not content_div: print(f"[WARN] No content found: {url}", file=sys.stderr) return {"title": title, "date": date_text, "content": "", "attachments": []} attachments = [] parts = [] # Extract attachments from the related attachments div attach_div = soup.select_one("div.txt2") if attach_div: for a in attach_div.find_all("a", href=True): href = a["href"].strip() atitle = a.get_text(strip=True) if href.startswith("/"): ahref = BASE_URL + href elif href.startswith("http"): ahref = href else: ahref = url.rsplit("/", 1)[0] + "/" + href attachments.append({"title": atitle, "url": ahref}) for el in content_div.find_all(["p", "table", "img"], recursive=True): if el.name == "p" and el.find_parent("table"): continue if el.name == "table" and el.find_parent("table"): continue if el.name == "p": text = el.get_text(" ", strip=True) # Check for links in p for a in el.find_all("a", href=True): href = a["href"].strip() if re.search(r'\.(pdf|docx?|xlsx?|rar|zip)$', href, re.I): atitle = a.get_text(strip=True) or a.get("download", "") if href.startswith("/"): ahref = BASE_URL + href elif href.startswith("http"): ahref = href else: ahref = url.rsplit("/", 1)[0] + "/" + href if not any(att["url"] == ahref for att in attachments): attachments.append({"title": atitle, "url": ahref}) if text: parts.append(text) elif el.name == 'table': tbl_html = html_table_to_html(el, url) if tbl_html: attachments.append(tbl_html) elif el.name == "img": src = el.get("src", "") alt = el.get("alt", "") if src: if src.startswith("//"): src = "http:" + src elif src.startswith("/"): src = BASE_URL + src elif not src.startswith("http"): src = url.rsplit("/", 1)[0] + "/" + src parts.append(f"![{alt}]({src})" if alt else f"![]({src})") # Deduplicate attachments seen = set() unique_attachments = [] for att in attachments: if att["url"] not in seen: seen.add(att["url"]) unique_attachments.append(att) content = "\n\n".join(parts) return {"title": title, "date": date_text, "content": content, "attachments": unique_attachments} def main(): html = fetch(LIST_URL) if not html: print("[ERROR] Cannot fetch list page", file=sys.stderr) sys.exit(1) total_pages = get_total_pages(html) if total_pages is None: total_pages = 1 print("[WARN] Could not determine total pages, assuming 1", file=sys.stderr) print(f"[INFO] Total pages: {total_pages}", file=sys.stderr) if _MAX_PG and total_pages > _MAX_PG: print(f"[INFO] Limiting to {_MAX_PG} pages (--pages={_MAX_PG})", file=sys.stderr) total_pages = _MAX_PG all_items = [] for pg in range(1, total_pages + 1): if pg == 1: url = LIST_URL else: url = f"{LIST_URL}?page={pg}" print(f"[INFO] Fetching page {pg}/{total_pages}: {url}", file=sys.stderr) html = fetch(url) if not html: print(f"[WARN] Empty response for page {pg}", file=sys.stderr) continue items = extract_items(html) print(f"[INFO] Found {len(items)} items on page {pg}", file=sys.stderr) all_items.extend(items) print(f"[INFO] Total items from list: {len(all_items)}", file=sys.stderr) conn = sqlite3.connect(DB) c = conn.cursor() new_count = 0 empty_content = 0 total_attachments = 0 for item in all_items: c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],)) if c.fetchone(): continue print(f"[INFO] Fetching detail: {item['title'][:50]}...", file=sys.stderr) html = fetch(item["url"]) if not html: print(f"[WARN] Cannot fetch detail, skipping: {item['url']}", file=sys.stderr) continue detail = parse_detail(html, item["url"]) title = detail["title"] or item["title"] date_text = detail["date"] or item["date"] content = detail["content"] attachments = detail["attachments"] plain = re.sub(r'<[^>]+>', '', content) plain = re.sub(r'\s+', ' ', plain).strip() summary = plain[:200] if plain else "" if not content: empty_content += 1 print(f"[WARN] Empty content: {title}", file=sys.stderr) date_rank = 0 if date_text: m = re.search(r"(\d{4})-(\d{2})-(\d{2})", date_text) if m: date_rank = int(m.group(1) + m.group(2) + m.group(3)) att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]" try: c.execute( "INSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank) VALUES (?,?,?,?,?,?,?,?,?)", (title, SITE_NAME, GROUP_NAME, item["url"], date_text, content, summary, att_json, date_rank) ) if c.rowcount > 0: new_count += 1 total_attachments += len(attachments) except Exception as e: print(f"[ERROR] DB insert failed: {title[:30]} - {e}", file=sys.stderr) conn.commit() conn.close() print(f"\n[RESULT] {SITE_NAME}: {new_count} new, {empty_content} empty content, {total_attachments} attachments") if __name__ == "__main__": main()