({j d Z ddlZddlZddlZddlZddlZddlZddlZddl m
Z
ej j dej j ej j e ddlmZ dZdZdZdZd Zd
Ze d e d e d
e ZdZdZdZdZ e de d e ddZ d Z d Z d Z d Z!d Z"e#dk( r e" yy)u
石家庄高新技术产业开发区 - 公示公告 爬虫
列表: http://www.shidz.gov.cn/columns/fc6b4309-d198-4db4-9c9a-8358ef6ba98c/index.html
首页静态渲染 15 条; 分页走 blocks AJAX 接口 (关键: 必须带 fix=0, 否则返回 0B):
/columns/{colid}/templates/{tid}/blocks/{blockid}?page=N&fix=0
colid=fc6b4309-d198-4db4-9c9a-8358ef6ba98c
tid=146f96b5-fcde-4c1c-8672-7399dfeb4a44
blockid=cd2eb42d-5993-49fc-a805-b3e2aaded14a
总数: 13385 条 / 893 页 (每页 15 条)
详情: /columns/UUID/YYYYMM/DD/UUID.html
标题: #biaoti 内 div; 日期: 发布时间：YYYY-MM-DD; 正文: #conN
用法: python3 crawl_shidz_gsgg.py [--pages=N] [--limit=N]
N)urljoin)push_to_searchdbzhttp://www.shidz.gov.cnu1 石家庄高新技术产业开发区-公示公告u 公示公告z$fc6b4309-d198-4db4-9c9a-8358ef6ba98cz$146f96b5-fcde-4c1c-8672-7399dfeb4a44z$cd2eb42d-5993-49fc-a805-b3e2aaded14az /columns/z /templates/z/blocks/ i} z
2020-01-01zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z*/*z /index.html)
User-AgentAcceptRefererc d} d}t j dd D ]` }|j d r t |j d d } 2|j d sDt |j d d }b | |fS )Nr z--pages==z--limit=)sysargv
startswithintsplit)pageslimitas %/root/gov_crawler/crawl_shidz_gsgg.py
parse_argsr & su
E
E
XXab\ ) <<
# Q(E
\\*
% Q(E )
%< c t j | xs d } t j dd| } t j dd| } t j dd| } | j S )N z [\u200b\u200e\u200f\ufeff\xa0]\s+ u ^[•·\-—]\s*)html_modunescaperesubstrip)ts r clean_titler! 1 sZ !'r"A
0"a8A
vsA A
"B*A 779r c | j t |ddt d }d|_ |j }|rt | dk r t
d| dt | d d
g S g }t }t j d | D ] }|j d
|j d |j d } }}||v r;|j | |j d st t | n|}t | }
t |
dk r|j! |
||d |S # t" $ r } t
d| d| d
g cY d} ~ S d} ~ ww xY w)u8 请求 blocks 分页接口, 返回 [{title, href, date}]r )pagefix )paramsheaderstimeoututf-8d z [WARN] page u 空响应 (zB)TflushzN(\d{4}-\d{2}-\d{2})\s*]*title=\"([^\"]+)\"r
http )titlehrefdatez [ERR] page : N)get BLOCK_URLHEADERSencodingtextlenprintsetr finditergroupaddr r BASE_URLr! append Exception) sessionr# rhtmlitemsseenmr3 r2 r1 r es r
fetch_pagerJ 9 s> KK 4*BG]_K `
vvs4y3 $TF,s4ykDD QIu nptu CA !
AGGAJ
$Dt| HHTN26//&2I78T*tDE"A1vz LL1dDA B C
vRs+48 s% A!D( $CD( ( E1E E Ec | j t t dz dd }d|_ |j }d}|j d }|dk\ r8t
j d |||d
z }|rt |j d }|sDt
j d | }|r,t |j d j d
d }d}t
j d| }|r|j d }|s_t
j d| }|rGd|j d t |j d t |j d fz }d}|j d }|dk\ r|j d| d z } d }
| } | t | k rJ|
dkD rE|| | dz dk( r |
d z
}
| dz
} n|| | dz dk( r |
d z }
| dz
} n| d z
} | t | k r|
dkD rE|| | dz
} t
j dd| } t
j dd| } t
j dfd | } t
j d fd | } t
j d d| } t
j d d| } | j }|rt | d!k rd}|||fS # t $ r }
t! d" d#|
d$% Y d&}
~
y'd&}
~
ww xY w)(uU 详情: #biaoti 标题 + 发布时间 + #conN 正文 (与 crawl_shidz_tzgg.py 同款)/)r r r% )r' r( r) r z id="biaoti"r z#]*>\s*([^<]{4,100})\s*i r
z([^<]+)_u, 发布时间[：:]\s*(\d{4}-\d{1,2}-\d{1,2})z(\d{4})-(\d{1,2})-(\d{1,2})z %s-%02d-%02dr- r. z id="conN">r0 zz
zzhref="([^"]*)"c d| j d j d s t | j d z S | j d z S )Nz href="%s"r
)r/ #
javascriptr> r r rH urls r z fetch_detail.. sq kefelelmneoezez |W fXWSRSRYRYZ[R\E] 7i ^_ ^e ^e fg ^h 7i r z
src="([^"]*)"c d| j d j d s t | j d z S | j d z S )Nzsrc="%s"r
)r/ zdata:rR rS rT s r rV z fetch_detail.. sq Zcdcjcjklcmcxcx zY dZ73PQPWPWXYPZC[ 6k `a `g `g hi `j 6k r z\sstyle="[^"]*"zM]*>||]*>||]*>||]*>| z [ERR] detail r4 Tr+ N)r r r )r5 UAr@ r8 r9 findr searchr! r> r r r: r r rB r; )rC rU rD rE r1 irH pub_datebodystartdepthjrawrI s ` r fetch_detailrc U s 3 KKB8c>%R\^K _
vv IIm $ 6 @$q3-PA#AGGAJ/ 3T:A#AGGAJ$4$4S$9!$<= IIEt L wwqzH 8$?A)QWWQZQWWQZ #aggVWj/,ZZ IIk " 6IIc1%)EEAc$i-EAI!A#;&(QJEFA!AaC[H,QJEFAFA c$i-EAI uQqS/C&&4b#>C&&2BJ# # K ,KK c $ t \ } }t dt d| d| d t j }|j
j
dt i g }t }| dk rd} t d| dz D ] }t || }d}|D ]4 }|d |v r |j |d |j | |dz
}6 t d
| d t | d | d
t | d |st d
| dd nt j d t | } |D cg c]! }|j! d xs dt" k\ s |# }}t | | k r$t dt" d| t | z
dd t dt | dd |rt | |kD r|d | }g }
t% | D ]j \ } }t' ||d \ } }
}| xs |d }|
xs |j! dd }|rt) j* dd| n|}t) j* dd| j- d d }g }|rjt) j. d |t( j0 D ]B }|j |j3 d j- |j3 d d D |
j t |d |d ||||t4 |rt7 j8 |d ! ndd" | dz d#z dk( r t d$| dz d%t | d t j d& m t d't |
d(d t; |
t ) t= d* |
D }t d+t |
d,| d y c c}w )-Nz[SHIDZ-GSGG] SITE=z pages=z limit=Tr+ r r r
r2 z [page z] u 条(新u ), 累计u ] 空页, 停止翻页g?r3 9999z CUTOFF u 过滤 u 条u 列表完成: u 条, 抓详情...r1 r z<[^>]+>r r i, zQ]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar|ofd|wps))"[^>]*>([^<]*)r- )namerU F) ensure_ascii) site_name
source_urlrU r1 r] contentsummarycategory attachments
z detail rL g333333?z
pushing z items to searchdb...) batch_labelc 3 h K | ]* }|j d xs dj r'd , yw)rj r r
N)r5 r ).0rD s r zmain.. s) Ka155+;+Ar*H*H*JKs (22z DONE. pushed=u 空正文=) r r; SITE_NAMErequestsSessionr' updaterY r< rangerJ r? rA r: timesleepr5 CUTOFF enumeraterc r r r r= Ir> CATEGORYjsondumpsr sum)r r rC all_itemsrG pgrF new_countitbeforeresultsr\ d_titled_dater^ r1 r] rk attsrH emptys r mainr s *A)B$GtT N3y>**<
=TJ Y %' fu% G9% 2 ,Wbj A &2g;/RVVFB /37"&&S$/U&&g.446t< [[!uw{} ~B ~B C
M QWWQZ%5%5%7
KL
M""V*RZT HCG4::d?R
ER<1 IacU!C N#34D A
3%(
Js7|n$9
:$GW)4 K7K KE OCL>UG
r s 1 0 0 277??277??8#<= > ( $ ? .,
0j % C5 J vuH:YugU`9a
b85p=J@ zF r