#!/usr/bin/env python3 """六安市裕安区-行政审批（环评审批）""" import re, sys, os, json, urllib.parse import urllib.request, urllib.error from bs4 import BeautifulSoup import sqlite3 BASE_URL = "https://www.yuan.gov.cn" API_URL = BASE_URL + "/site/label/8888" SITE_NAME = "裕安区-行政审批" SITE_DISPLAY = "六安市裕安区人民政府" GROUP_NAME = "环评平台" DB = os.environ.get("SEARCH_DB", "/root/search.db") HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"} _API_PARAMS = { "labelName": "publicInfoList", "siteId": "6788091", "organId": "6596771", "catId": "7260215", "pageSize": "15", "action": "list", "isDate": "true", "dateFormat": "yyyy-MM-dd", "length": "80", "type": "4", "result": "暂无相关信息", "file": "/c2/yuan/publicInfoList_newest" } _MAX_PG = None for i, a in enumerate(sys.argv): if a == "--pages" and i + 1 < len(sys.argv): _MAX_PG = int(sys.argv[i + 1]) break def html_table_to_html(table, base_url=""): """保留 HTML 表格结构，仅将相对链接/图片转绝对 URL""" import urllib.parse tbl = BeautifulSoup(str(table), 'html.parser') for a in tbl.find_all('a'): href = a.get('href', '') if href and not href.startswith(('http', 'javascript', '#')): a['href'] = urllib.parse.urljoin(base_url, href) if base_url else href for img in tbl.find_all('img'): src = img.get('src', '') if src and not src.startswith(('http', '//', 'data:')): img['src'] = urllib.parse.urljoin(base_url, src) if base_url else src return str(tbl) def fetch(url): req = urllib.request.Request(url, headers=HEADERS) try: return urllib.request.urlopen(req, timeout=30).read().decode("utf-8", errors="replace") except Exception as e: print(f"[WARN] fetch failed: {url} - {e}", file=sys.stderr) return "" def fetch_api(page_index): params = dict(_API_PARAMS) params["pageIndex"] = str(page_index) url = API_URL + "?" + urllib.parse.urlencode(params) req = urllib.request.Request(url, headers=HEADERS) try: return urllib.request.urlopen(req, timeout=30).read().decode("utf-8", errors="replace") except Exception as e: print(f"[WARN] API failed page {page_index}: {e}", file=sys.stderr) return "" def extract_items(html): soup = BeautifulSoup(html, "html.parser") items = [] for li in soup.select("li.clearfix"): a = li.find("a", class_="title", href=True) if not a: continue href = a["href"].strip() title = a.get_text(" ", strip=True) # Clean title: remove image alt text and leading special chars title = re.sub(r'^[【\[]?[^】\]]+[】\]]?\s*', '', title).strip() if not title: title = a.get_text(strip=True) span = li.find("span", class_="date") date_text = span.get_text(strip=True) if span else "" if href and not href.startswith("http"): href = BASE_URL + href items.append({"title": title, "url": href, "date": date_text}) return items def parse_detail(html, url): soup = BeautifulSoup(html, "html.parser") # Title title = "" if soup.title: title = soup.title.string.strip() title = re.sub(r"_六安市裕安区人民政府\s*$", "", title).strip() title = re.sub(r'\s+', ' ', title) # Date from table (发布日期) date_text = "" for th in soup.find_all("th"): if "发布日期" in th.get_text(): td = th.find_next("td") if td: m = re.search(r"(\d{4}-\d{2}-\d{2})", td.get_text()) if m: date_text = m.group(1) break # Content content_div = soup.select_one("div.wzcon.j-fontContent") if not content_div: print(f"[WARN] No content: {url}", file=sys.stderr) return {"title": title, "date": date_text, "content": "", "attachments": []} attachments = [] parts = [] for el in content_div.find_all(["p", "table", "img"], recursive=True): if el.name == "p" and el.find_parent("table"): continue if el.name == "table" and el.find_parent("table"): continue if el.name == "p": text = el.get_text(" ", strip=True) for a in el.find_all("a", href=True): href = a["href"].strip() if re.search(r'\.(pdf|docx?|xlsx?|zip|rar)$', href, re.I) or "/group2/M00" in href: atitle = a.get_text(strip=True) or a.get("download", "") ahref = BASE_URL + href if href.startswith("/") else (url.rsplit("/", 1)[0] + "/" + href if not href.startswith("http") else href) if not any(att["url"] == ahref for att in attachments): attachments.append({"title": atitle, "url": ahref}) if text: parts.append(text) elif el.name == 'table': tbl_html = html_table_to_html(el, url) if tbl_html: parts.append(tbl_html) elif el.name == "img": src = el.get("src", "") alt = el.get("alt", "") if src: src = BASE_URL + src if src.startswith("/") else (url.rsplit("/", 1)[0] + "/" + src if not src.startswith("http") else src) parts.append(f"![{alt}]({src})" if alt else f"![]({src})") seen = set() ua = [] for att in attachments: if att["url"] not in seen: seen.add(att["url"]) ua.append(att) return {"title": title, "date": date_text, "content": "\n\n".join(parts), "attachments": ua} def main(): first_html = fetch_api(1) if not first_html: print("[ERROR] Cannot fetch list page", file=sys.stderr) sys.exit(1) # Determine total pages from the HTML pagination script total_pages = 1 m = re.search(r'pageCount:\s*(\d+)', first_html) if m: total_pages = int(m.group(1)) print(f"[INFO] Total pages: {total_pages}", file=sys.stderr) if _MAX_PG and total_pages > _MAX_PG: print(f"[INFO] Limiting to {_MAX_PG} pages (--pages={_MAX_PG})", file=sys.stderr) total_pages = _MAX_PG all_items = extract_items(first_html) print(f"[INFO] Found {len(all_items)} items on page 1/{total_pages}", file=sys.stderr) for pg in range(2, total_pages + 1): print(f"[INFO] Fetching page {pg}/{total_pages}", file=sys.stderr) html = fetch_api(pg) if not html: continue items = extract_items(html) print(f"[INFO] Found {len(items)} items", file=sys.stderr) all_items.extend(items) print(f"[INFO] Total items: {len(all_items)}", file=sys.stderr) conn = sqlite3.connect(DB) conn.execute("PRAGMA busy_timeout = 60000") c = conn.cursor() new_count = empty_content = total_attachments = 0 for item in all_items: c.execute("SELECT id FROM gov_raw WHERE page_url=?", (item["url"],)) if c.fetchone(): continue html = fetch(item["url"]) if not html: continue detail = parse_detail(html, item["url"]) title = detail["title"] or item["title"] date_text = detail["date"] or item["date"] content = detail["content"] attachments = detail["attachments"] plain = re.sub(r'\s+', ' ', re.sub(r'<[^>]+>', '', content)).strip() summary = plain[:200] if not content: empty_content += 1 date_rank = 0 if date_text: m = re.search(r"(\d{4})-(\d{2})-(\d{2})", date_text) if m: date_rank = int(m.group(1) + m.group(2) + m.group(3)) att_json = json.dumps(attachments, ensure_ascii=False) if attachments else "[]" try: c.execute("INSERT OR IGNORE INTO gov_raw (title, site_name, group_name, page_url, publish_date, content, summary, attachments, date_rank) VALUES (?,?,?,?,?,?,?,?,?)", (title, SITE_NAME, GROUP_NAME, item["url"], date_text, content, summary, att_json, date_rank)) if c.rowcount > 0: new_count += 1 total_attachments += len(attachments) except Exception as e: print(f"[ERROR] DB: {title[:30]} - {e}", file=sys.stderr) conn.commit() conn.close() print(f"\n[RESULT] {SITE_NAME}: {new_count} new, {empty_content} empty, {total_attachments} attachments") if __name__ == "__main__": main()