uj d Z ddlZddlZddlZddlZddlZddlmZmZ ej j dej j ej j e ddl
mZ ej ed z
j! d Zdd iZd
Zd d d
ddd
ddd
ddd
ddd
dZd Zd Zd Zd%dZed k( r ej4 Zej9 d d e ej= ej9 d e d! ejA Z! ee!jD e!jF " Z$ e%d#e$ d$ yy)&ua crawl_shanxi_sthjt.py - 山西省生态环境厅-环境影响评价公示 (5栏目)
站点: https://sthjt.shanxi.gov.cn/zwgk/hjpj/{col}/
CMS: TRS WCM (UTF-8, 服务端渲染)
栏目(--col):
xmslgs = 项目受理公示
nspgs = 拟审批公示
hpwjspjdgg = 环评文件审批决定
nzcspyjgs = 拟作出审批意见公示
sljsxmhjyxpjwjqk = 受理建设项目环评文件情况
列表: 两种结构 —
A) ul.list-details > li > a[title=标题](href=./YYYYMM/tYYYYMMDD_xxx.shtml) + span[YYYY-MM-DD] (xmslgs/nspgs/hpwjspjdgg)
B) table.list-table > tr > td > a[title=标题] + ... + td 日期 (nzcspyjgs/sljsxmhjyxpjwjqk, 列: 项目名称/建设地点/建设单位/环评机构/日期)
分页: index_N.shtml (首页=index.shtml, 第2页=index_1.shtml, 每页10~20条)
详情: 标题 + div.content-box 正文(TRS_UEDITOR)
N)datetime timedelta)push_to_searchdbiG )daysz%Y-%m-%dz
User-Agentz6Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0zhttps://sthjt.shanxi.gov.cnu+ 山西省生态环境厅-项目受理公示zzwgk/hjpj/xmslgs)namepathu( 山西省生态环境厅-拟审批公示zzwgk/hjpj/nspgsu1 山西省生态环境厅-环评文件审批决定zzwgk/hjpj/hpwjspjdggu4 山西省生态环境厅-拟作出审批意见公示zzwgk/hjpj/nzcspyjgsu= 山西省生态环境厅-受理建设项目环评文件情况zzwgk/hjpj/sljsxmhjyxpjwjqk)xmslgsnspgs
hpwjspjdgg nzcspyjgssljsxmhjyxpjwjqkc & t | d }|dk( r
t d| d}nt d| d|dz
d} t j |dt d }d |_ t
j d
|j t j }g }|D ]b \ }}} }
t
j d d |xs | j } t | d
k r@t d| d| }|j ||
| d d |r|S t
j d|j t j } | D ]b \ }}} }
t
j d d |xs | j } t | d
k r@t d| d| }|j ||
| d d |S # g cY S xY w)Nr /z/index_z.shtml Ftimeoutheadersverifyutf-8zl<li>\s*<a href='\./(\d{6}/t\d+_\d+\.shtml)' title='([^']*)'>(.*?)</a>\s*<span>\[(\d{4}-\d{2}-\d{2})\]</span>\s+ )urlpub_datetitlez<tr>\s*<td><a[^>]*href='\./(\d{6}/t\d+_\d+\.shtml)'[^>]*title='([^']*)'>(.*?)</a></td>\s*<td[^>]*>.*?</td>\s*<td[^>]*>.*?</td>\s*<td[^>]*>.*?</td>\s*<td[^>]*>(\d{4}-\d{2}-\d{2})</td>)COLUMNSBASErequestsgetHEADERSencodingrefindalltextDOTALLsubstriplenappend)
colpage_idxr r ritemsresultshref
title_attr
title_textpdr rowss
'/root/gov_crawler/crawl_shanxi_sthjt.py
fetch_listr8 $ s 3< D1}avQ avWXaZL7 LLgwu M
JJw
E G,1 E(j*bvr:#;<BBD u:> avQtf%sUCD E
:: B D -1 E(j*bvr:#;<BBD u:> avQtf%sUCD E N5 s $F
Fc d t j || }|sy| j d|j }|dk( ry|dz }d}t j d| |d D ]J }|j d j
d r|dz
})|dz }|dk( s4| |||j z c S | |d S ) u 平衡 div 匹配: 从匹配到的 <div...> 的 '>' 之后, 到对应 </div>, 返回内部 HTML。
修复非贪婪 (.*?)</div> 在正文含嵌套 div(表格/附件区)时截断的问题r >r z<div[\s>]|</div>Nr z<div)r% searchfindstartfinditergroup
startswith)html open_tag_remgtidepthmms r7 extract_balanced_divrI I s +t$A
3 "B Rx
QA
Ekk-tABx8 .
88A; ! !& ) QJE QJEzAa"((*n--
. 8O c t d D ]( } t j | dt d }d|_ n yd}t j d|j t
j }|r[t j d d|j d
j }t j d | d j }|r|d
k( rt j d|j t
j }|rYt j dd|j d
j }t j d d| j }d}dD ]: }t |j | }|j s*|j } n t j dd|t
j t
j z }t j dd|t
j t
j z }t j dd|t
j t
j z }t j dd | }||fS # d }Y YxY w)N r Fr r )r r r z <title>\s*([^<]+?)\s*r r u [-–—|_]r u 山西省生态环境厅z]*>(.*?)z<[^>]+>)z ]*id="content-box"[^>]*>z-]*class="[^"]*text-details[^"]*"[^>]*>z0]*class="[^"]*trs_editor_view[^"]*"[^>]*>z)flagsz]*>.*?z0]*class="[^"]*td-title-2[^"]*"[^>]*>.*?z src="(?!https?://|//)(/[^"]+)"c 8 dt | j d dS )Nzsrc="r ")r r@ )rH s r7 z fetch_detail.. s" U4&QSQYQYZ[Q\P]]^C_ rJ )ranger! r" r# r$ r% r< r' r( r) r@ r* splitrI I)r _attemptr/ r rD contentpatcs r7 fetch_detailrX ] s !H S'75QA AJ y E
2AFFBIIFAvr1771:.446 /288: E 77 II,affbii @ FF:r1771:6<<>EFF62u-335EGE
! - 779ggiG
ff+R BDD@PQGff)2wbii"$$>NOGffH"g]_]f]fikimim]mnGff68_ahiG '>; As $I I%r c g }t }t | D ] }t | |dz }|st d|dz d nt d|dz dt | d |D ] }|d r
|d t
k r|d |v r |j
|d t |d \ }}|s|d }|j sZ|j ||d |d |d t | d |d
d t d t | d |D cg c] } | d
j s| }
} |
rt |
d| t |
S c c} w )Nr z Page z: 0 items (stop)z: z itemsr r r r r )r r
source_urlr site_namerU summaryz Total: rU
shx_sthjt_) setrQ r8 printr+ CUTOFFaddrX r* r, r r ) r- max_pagesrecordsseenpgr0 itr rU r/ valids r7 runrh sn G 5DI 3Q' GBqD6!12 3
1vRE
|623 B*~"Z.6"9%yD HHRY )"U) 4NE77 ==? NN r%y5 zN $S\& 1gRT
,
Ic'l^6
*+ 811Y<#5#5#7Q 8E 8 *SE 23 u:
9s E E__main__z--colr )defaultchoicesz--pages)typerj )rb zDone: z records)r )&__doc__sysosr! r% argparser r r insertdirnameabspath__file__ crawler_libr nowstrftimer` r# r r r8 rI rX rh __name__ArgumentParserap add_argumentlistkeysint
parse_argsargsr- pagescntr_ rJ r7 r sK ' & & ( 277??277??8#<= > (
(,,.9%0
0 : :: F Q
R$ EN`a@J[
\NXnoPZop!` kG H #J(#J > z "BOOGXtGLLN7KOLOOICO3
==?D
dhh$**
-C F3%x
!
rJ