-wjw! N d Z ddlZddlZddlZddlZddlZddlZddlmZ ej j dej j ej j e
ddlmZ ddlmZmZ ej$ ed z
j' d Zd Zd
Zd Zd Zd
eez dz dZd Zd Zd Z d Z d dZ e dk( rW ej@ Z!e!jE de#d e!jI Z% e e%jL Z' e(de de' d yy) u crawl_huizhou_hpslgg.py - 惠州市-环评受理公告
站点: https://shj.huizhou.gov.cn/zwgk/jsxmhpglxxgk/hpslgg/
CMS: NFCMS (广东南方网系网站群, window.NFCMS_SITE_ID=752011)
列表: UL.ul_art_row > LI.li_art_title > A[title] 标题 + LI.li_art_date 日期(YYYY-MM-DD)
分页: 首页 index.html + index_N.html (N=2..8, 无index_1; 每页20条)
⚠️ WAF: 分页页须 Chrome UA + Referer, 否则 nginx 直接 404
详情: TD.artTitle.xltd_9 标题 + meta PubDate 日期 + TD.artContent.xltd_10 > DIV#divZoom 正文(平衡div, UEditor表格)
附件: TD.xltd_11 > div.m-attachment > div.f-APPENDIX > a.pdf (绝对URL) → 转 附件名
段落嵌入
N)urljoin)push_to_searchdb)datetime timedeltaiG )daysz%Y-%m-%du 惠州市-环评受理公告u 广东zhttps://shj.huizhou.gov.cnz/zwgk/jsxmhpglxxgk/hpslgg/zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36
index.html)z
User-AgentRefererc t j | dt d }d|_ |j S # t
$ r+}t
d| d| t j Y d }~yd }~ww xY w) N) F)timeoutheadersverifyzutf-8z[WARN] fetch failed: z - )file ) requestsgetHEADERSencodingtext Exceptionprintsysstderr)urlres )/root/gov_crawler/crawl_huizhou_hpslgg.pyfetchr sZ LLgwu M
vv
%cU#aS1
Cs /2 A&!A!!A&c t j || t j }|sy| j d|j }|dk( ry|dz }d}t j
d| |d t j D ]j }|j
d j d s |j
d j d r|dz
}I|dz }|dk( sT| |||j z c S | |d S )
u& 平衡 div 匹配 (大小写不敏感)r > z]|Nr z .
88A; ! !& )RXXa[-C-CF-K QJE QJEzAa"((*n--
. 8O c , | dk( rt t z dz }nt t z d| dz dz }t | }|sg S g }t j d|t j
D ], }|j
d |j
d |j
d |j
d j f\ }}}}|xs t j d
d | j } t j d d
| j } t j dd | } t | d k rd|vsd|vrt t t z | }t j d| }
|
r|
j
d nd } |j || | d / |S )uA 列表页: 0=首页 index.html, N=index_N.html (N>=2, 无index_1)r r index_r# z.htmlz{\s*]*title="([^"]*)"[^>]*href="([^"]+)"[^>]*>(.*?).*?([^<]*) <[^>]+>r \s+ ^\s*(?:·| |·|\s)+z/content/post_z/hpslgg/(\d{4}-\d{2}-\d{2}))r pub_datetitle)
BASELIST_DIRr r$ r) DOTALLr* stripsublenr r% append) page_idxr r, itemsr.
title_attrhref
title_html date_textr@ mdpds r
fetch_listrP = ss 1}Xo ,Xo&A e << :D E
[[
459299F C 34''!*aggaj!''RS*VWV]V]^_V`VfVfVh2h/
D*iArvvj"j AHHJvsE*0028"eD u:> 4 ':T+A dXot,
YY-y
9 RXXa[B
Sb5AB!C" Lr4 c t }|syd}t j d|t j t j z }|rt j
dd|j
d t j t j z }t j
dd| j }t j
dd | j }|s7t j d
| }|r |j
d j }t j
d d| }t j
d d| }d}t j d
| }|r8t j d|j
d }|r|j
d }|s)t j d| }|r|j
d }d}t |d }|j r|j }|j s,t |d }|j r|j }t j
dd|t j t j z }t j
dd|t j t j z }t j
dd|t j }t j
d fd| }t j
d fd| }g } t j d|t j }|rt j d|j
d t j D ] }
|
j
d j t j
dd|
j
d j } } | sY| j d r t | z } | j d | d | xs d d | r%|j d!z d!j | z }t j
d"d| }t j
d#d| }|||j fS )$uY 详情页: artTitle 标题 + PubDate 日期 + divZoom 正文 + f-APPENDIX 附件转段落)r r r r z2]*class="[^"]*artTitle[^"]*"[^>]*>(.*?)z!]*artSource[^>]*>.*?r# )flagsr: r; r< z/ u" 发布时间：(\d{4}-\d{2}-\d{2})z]*id="divZoom"[^>]*>z*]*class="[^"]*artContent[^"]*"[^>]*>zzz ^]*id="divZoom"[^>]*>\s*z,href="(?!https?://|//|#|javascript:)([^"]+)"c B dt | j d dS )Nzhref="r# "r r* r2 r s r z fetch_detail.. s &bhhqk)B(C1 E r4 z src="(?!https?://|//)([^"]+)"c B dt | j d dS )Nzsrc="r# rT rU rV s r rW z fetch_detail.. s %RXXa[(A'B! D r4 z6]*class="[^"]*f-APPENDIX[^"]*"[^>]*>(.*?)$]*href="([^"]+)"[^>]*>(.*?)r7 /z 附件z
z%^(]*>(\s*()?\s*)?)+z%(]*>(\s*()?\s*)?)+$)r r$ r% rC r& rE r* rD r3 r) r+ rA rG rstripjoin)
r r, r@ r. segrM rN contentc attach_psamhts
` r fetch_detailrf [ sY :D E
Gryy[][_[_O_`Aff92qwwqzQSQZQZ]_]a]aQabz2s+113vsE*002 IIH$ O GGAJ$$&E FF4b% @E FF[]_af gEI
?FA
YY-qwwqz
:
I II;T B
IG T#ABAwwy'') ==? 'T U 779ggiGff+R BDD@PQGff)2wbii"$$>NOGff7WBDDQGffDEwPGff5DgOG I
KTSUS\S\]A++ EqwwqzSUS\S\] NB88A;$$&z2rxx{(K(Q(Q(SqA<<$ qA <s"Q](O8!LM N .."T)DIIi,@@ff=r7KGff=r7KG )W]]_ ,,r4 r c P g }t }t | D ] }t | }|st d|dz d nt d|dz dt | d |D ]j }|d r
|d t
k r|d |v r |j
|d t |d \ }}}|st d|d X|s|d }|xs |d } g }
t j d
|t j D ] } | j d t j d d | j d
j }
} t j d| t j sd| j! v s|
j# |
xs d| d |j# ||d |d | t$ |d |
rt' j( |
d ndt* d m t dt | d |D cg c] }|d j s| }}|r t- |d t | S c c}w )Nz Page r# z: 0 items (stop): itemsr? r z SKIP empty content: r@ rY r: r r7 z*\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps)$
attachmentr[ )r@ r F) ensure_asciiz[]) r@ r
source_urlr? site_namer` summary attachments
group_namez Total: r` huizhou_hpslgg)setrangerP r rF CUTOFFaddrf r$ r) rC r* rE rD r% r& lowerrG SITE_NAMEjsondumps
GROUP_NAMEr ) max_pagesrecordsseenpgrI itr@ rM r` rO attsrc rd re r valids r runr s G 5DI 2 GBF8#34 5
Qxr#e* V45 B*~"Z.6"9%yD HHRY (4RY(? %E9g0E <=7 ,bnBDkk"I7TVT]T]^
Dxx{BFF:r288A;$G$M$M$O199 JArttTXdhihohohqXqKK!-x BC
D
NN r%y5 Y7 VZdjjE.R`d (
'
>
Ic'l^6
*+ 811Y<#5#5#7Q 8E 8 01 u:
9s +H#H#__main__z--pages)typedefaultz [RESULT] rh ri )r ))__doc__r osr$ rx argparser urllib.parser pathinsertdirnameabspath__file__ crawler_libr r r nowstrftimert rw rz rA rB r r r3 rP rf r __name__ArgumentParserap add_argumentint
parse_argsargspagesnr r4 r r s # " " 277??277??8#<= > ( (
(,,.9'2
2 <