#!/usr/bin/env python3 """批量生成爬虫脚本 Part3 — dataproxy/特殊列表/API 站""" import os, re as _re OUT = os.path.expanduser("~/Crawler/gov_crawler/gen_out") SKEL = open(os.path.join(OUT, "crawl_gdkfq_ezhou_hbgs.py"), encoding="utf-8").read() def write(name, content): path = os.path.join(OUT, name) with open(path, "w", encoding="utf-8") as f: f.write(content) print("WROTE", name) def re_replace_block(s, start_marker, new_block): idx = s.find(start_marker) if idx < 0: return s nxt = s.find("\ndef ", idx + 1) nxt2 = s.find('\nif __name__ == "__main__":', idx + 1) if nxt2 > 0 and (nxt < 0 or nxt2 < nxt): nxt = nxt2 if nxt < 0: return s return s[:idx] + new_block.rstrip("\n") + s[nxt:] def gen_plain(name, desc, url, base, list_url, site_name, group, category, page_tpl, list_selector, date_sel, detail_selector): """普通 HTML 列表站""" s = SKEL s = s.replace('"""\n葛店开发区 - 环保公示\nURL: http://gdkfq.ezhou.gov.cn/zwdt/hbgs/\n"""', '"""\n%s\nURL: %s\n"""' % (desc, url)) s = s.replace('BASE_URL = "http://gdkfq.ezhou.gov.cn"', 'BASE_URL = "%s"' % base) s = s.replace('LIST_URL = "http://gdkfq.ezhou.gov.cn/zwdt/hbgs/"', 'LIST_URL = "%s"' % list_url) s = s.replace('SITE_NAME = "葛店开发区-环保公示"', 'SITE_NAME = "%s"' % site_name) s = s.replace('GROUP_NAME = "湖北"', 'GROUP_NAME = "%s"' % group) s = s.replace('SCRIPT_NAME = "crawl_gdkfq_ezhou_hbgs.py"', 'SCRIPT_NAME = "%s"' % name) s = s.replace('CATEGORY = "环境保护"', 'CATEGORY = "%s"' % category) blk = "def build_list_url(page):\n if page == 1:\n return LIST_URL\n return BASE_URL + \"" + page_tpl + "\" % page\n" s = re_replace_block(s, "def build_list_url(page):", blk) blk2 = ("def parse_list(html_text, page_url):\n" " items = []\n" " if not BeautifulSoup:\n" " return items\n" " soup = BeautifulSoup(html_text, \"html.parser\")\n" " for li in soup.select(\"" + list_selector + "\"):\n" " a = li.find(\"a\", href=True)\n" " if not a:\n" " continue\n" " href = a[\"href\"].strip()\n" " title = a.get(\"title\", \"\") or a.get_text(strip=True)\n" " date = \"\"\n" " d = li.select_one(\"" + date_sel + "\")\n" " if d:\n" " date = d.get_text(strip=True)\n" " title = clean_title(title)\n" " if not title or len(title) < 4:\n" " continue\n" " if not href.startswith(\"http\"):\n" " href = urllib.parse.urljoin(page_url, href)\n" " items.append((href, title, date))\n" " return items\n") s = re_replace_block(s, "def parse_list(html_text, page_url):", blk2) blk3 = ("def parse_detail(html_text, page_url):\n" " title = \"\"; date = \"\"; content_html = \"\"\n" " if BeautifulSoup:\n" " soup = BeautifulSoup(html_text, \"html.parser\")\n" " d = soup.select_one(\"" + detail_selector + "\")\n" " if d:\n" " content_html = \"\".join(str(c) for c in d.contents)\n" " t = soup.find(\"meta\", attrs={\"name\": re.compile(\"ArticleTitle\", re.I)})\n" " if t and t.get(\"content\"):\n" " title = clean_title(t[\"content\"])\n" " md = soup.find(\"meta\", attrs={\"name\": re.compile(\"PubDate\", re.I)})\n" " if md and md.get(\"content\"):\n" " dm = re.search(r\"([0-9]{4}-[0-9]{2}-[0-9]{2})\", md[\"content\"])\n" " if dm:\n" " date = dm.group(1)\n" " if not title:\n" " h1 = soup.find(\"h1\") or soup.find(\"h2\")\n" " if h1:\n" " title = clean_title(h1.get_text())\n" " return title, date, content_html\n") s = re_replace_block(s, "def parse_detail(html_text, page_url):", blk3) return s if __name__ == "__main__": # ===== fengning 丰宁 (汉诺威, art 链接) ===== write("crawl_fengning_gsgg.py", gen_plain( "crawl_fengning_gsgg.py", "丰宁满族自治县 - 公告公示", "http://fengning.gov.cn/col/col4511/index.html?number=C20007C00011", "http://fengning.gov.cn", "http://fengning.gov.cn/col/col4511/index.html?number=C20007C00011", "丰宁满族自治县-公告公示", "河北", "公示公告", '/col/col4511/index_%d.html?number=C20007C00011', "ul li", "span", "div.article, div.content, div.TRS_Editor, #zoom" )) # ===== sthjt_sc 四川 ===== write("crawl_sthjt_sc.py", gen_plain( "crawl_sthjt_sc.py", "四川省生态环境厅 - 拟批准建设项目环境影响报告书公示", "https://sthjt.sc.gov.cn/sthjt/c103940/list_level2.shtml", "https://sthjt.sc.gov.cn", "https://sthjt.sc.gov.cn/sthjt/c103940/list_level2.shtml", "四川省生态环境厅-拟批准环评", "四川", "环评审批", '/sthjt/c103940/list_level2_%d.shtml', "div.list dl dd", "span.fr", "div.TRS_Editor, div.content, div.list_con, .Custom_UnionStyle" )) # ===== puershi 普洱 ===== write("crawl_puershi_hjxx.py", gen_plain( "crawl_puershi_hjxx.py", "普洱市 - 建设项目环境影响评价信息", "https://www.puershi.gov.cn/zwgk/zfxxgk2/zfbmxxgk/pessthjj/fdzdgknr/hjbh/jsxmhjyxpjxx.htm", "https://www.puershi.gov.cn", "https://www.puershi.gov.cn/zwgk/zfxxgk2/zfbmxxgk/pessthjj/fdzdgknr/hjbh/jsxmhjyxpjxx.htm", "普洱市-建设项目环评", "云南", "环评审批", '/zwgk/zfxxgk2/zfbmxxgk/pessthjj/fdzdgknr/hjbh/jsxmhjyxpjxx/%d.htm', "div.gkpt_rig ul li", "span", "div#vsb_content, div.content, div#content_div" )) # ===== yanshi 偃师 ===== write("crawl_yanshi_hjbh.py", gen_plain( "crawl_yanshi_hjbh.py", "偃师区 - 建设项目环境影响评价信息", "https://www.yanshi.gov.cn/zdlyxxgk/hjbhxx/jsxmhjyxpjxx/", "https://www.yanshi.gov.cn", "https://www.yanshi.gov.cn/zdlyxxgk/hjbhxx/jsxmhjyxpjxx/", "偃师区-建设项目环评", "河南", "环评审批", '/zdlyxxgk/hjbhxx/jsxmhjyxpjxx/index_%d.html', "div.news_list ul li, ul li", "span", "div#contents.newscontents, div.ny_content, div.content" )) print("DONE part3 (4 sites)")