uj,
d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl m
Z
ddl m Z ej dd ZdZd Zd
Zd Ze d Zd
dddddddddddd Zdddd Zd Zd Zd Zd Zd! Zd" Zd5d#Z d6d$Z d% Z e d&k( r e ejB d'kD rejB d' nd(Z"e"d(k( r e ye"d)k( r e ye"dk( r~ ed' Z#e#re#jI d*d ndZ% ed+e% e#jI d, xs g dd- D ]9 Z& e'd.e&jI d/ xs d0dd1 d2e&jI d3d0 dd4 ; yyy)7uT
Crawler: 芜湖市湾沚区人民政府 - 行政审批 (Epoint 基层政务公开)
N)datetime timedelta)
BeautifulSoup)urljoin SEARCH_DBz/root/search.dbu 湾沚区生态环境分局u 行政审批zhttps://www.wanzhi.gov.cnzhttps://sthjj.wuhu.gov.cnz/wuhu/site/label/8888grassrootsContentPageList67878916603391151truez
yyyy-MM-dd504list1141240) labelNamesiteIdorganIdpageSize pageIndexisDate
dateFormatlengthtypeactionisJsoncatIdzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z!application/json, text/plain, */*z0application/x-www-form-urlencoded; charset=UTF-8)z
User-AgentAcceptz Content-Typec 2 t dt d| d y )N[z] T)flush)printSITE)msgs !/root/gov_crawler/crawl_wanzhi.pylogr& % s AdV2cU
4( c i t dt | i} t j t |t
dd }|j
S # t $ r}t d| Y d }~y d }~ww xY w)Nr F)dataheaderstimeoutverifyz API error: ) API_DATAstrrequestspostAPI_URLHEADERSjson Exceptionr& )
page_indexr* res r% fetch_list_pager9 ) s` 5h 5 S_ 5D MM'grRW Xvvx k! s 1A A*A%%A*c t j | t dd }d|_ |j | fS # t
$ r }t
d| d| d| fcY d}~S d}~ww xY w)zFetch detail page.r) F)r+ r, r- zutf-8zFetch error for : N)r0 getr3 encodingtextr5 r& )
detail_urlr7 r8 s r% fetch_detailr@ 3 sb LLWb O
vvz!! zl"QC 01Z s 14 A AA A c B | j d }|dk r| j d }|dk ry| j dd| }d}| j d| }|dkD r|t | k r|dz
}| ||dz d k( r | j d
| }|dk\ r|d z }@| | dk( r| |dz |d z d
k7 rn`| ||d z dk( rU| j d| }||kD r>| |d z | j d }|r|dk7 r|}|dz }|dk( rt | ||dz S |dkD r|t | k ry)z5Extract content from whjhq.gov.cn - gkwz_contnet div.z"class="j-fontContent gkwz_contnet"r gkwz_contnet < > / s`
))8
9C
Qwii'
Qw JJsAs #E
E #sA
!)CI
Q !A#;& %#AAvE 7c>d1Q3qsms 2
!AaC[D
iiQ'G{ !G,224Q7E 1 A
A:)$uWQY*?@@% !)CI
& r' c B | j d }|dk r| j d }|dk ry| j dd| }d}| j d| }|dkD r|t | k r|dz
}| ||dz d k( r | j d
| }|dk\ r|d z }@| | dk( r| |dz |d z d
k7 rn`| ||d z dk( rU| j d| }||kD r>| |d z | j d }|r|dk7 r|}|dz }|dk( rt | ||dz S |dkD r|t | k ry)zAExtract content from sthjj.wuhu.gov.cn - newscontnet minh500 div.z)class="j-fontContent newscontnet minh500"r znewscontnet minh500rC rD rE rF rG rH rI rJ rK rL rM rN rO rU s r% get_content_sthjjr` ^ sa
))?
@C
Qwii-.
Qw JJsAs #E
E #sA
!)CI
Q !A#;& %#AAvE 7c>d1Q3qsms 2
!AaC[D
iiQ'G{ !G,224Q7E 1 A
A:)$uWQY*?@@% !)CI
& r' c | syt | d }|j dd D ]R }|j dd }|j d xs d}t t
| }|j
d | d
| d T |j g d D ] }|j g }|j d
dg D ]u }|j d
k( r|j t | -|j dk( s=|j d
rO|j dd } | se|j | w |rdj | S | j S )zgConvert content HTML to clean text with markdown links, preserving tables as HTML in original position.rC z html.parseraT)hrefrc )stripu 附件r z]())spanbstrongfontemiustablep) separatorrd z
)r find_allr< get_textr STHJJ_URL replace_withunwrapnameappendr/ find_parentjoinrd )
content_htmlsoupa_tagrc r> full_urltagpartselts
r% rT rT ~ s6 } 5D s . 4yy$~~D~)5X9d+
QtfBxj23 4
}} QR
EmmWcN+
77g LLR !
WW^BNN7$; b 5A Q
"'6;;u @L,>,>,@@r' c dddd}t j d| }|rn|j d j }t j dd| j }t j dd| j }||d< t j d| }|r%|j d j d d
|d < |d k( rt | |d
< |S t
| |d
< |S )z Extract title, date, content.rC title publish_datecontentz(.*?)rE u* [_\-—]\s*芜湖市政务公开平台\s*$u' [_\-—]\s*芜湖市生态环境局\s*$r zPubDate.*?content="([^"]+)"N
r whjhqr )researchgrouprd subr^ r` )rV site_typeresultmr s r% parse_detailr s 2"
=F
)40A
"Db%PVVXA2uMSSU w
0$7A!"!1!1!3CR!8~ G -d 3y M .d 3y Mr' c
t j t | dz z
j d }t d| d t d }|sy |j
dd }t t d }||z dz
|z }t d
| d | g }t dt |dz d D ]/ }t | }|s n |j
d
g }|D ] } | j
d xs dd d }
|
|k r!| j
dd xs6 | j
dd xs" | j
dd xs t d| d } | j
d xr- t | j
d xs | j
d } |j | d | j
dd |
| | d t d| dt | d t j d 2 t d t | d g }
t# |d D ] \ }} t% | d \ }}|r8| d rd nd }t' || }|d r|d | d< |d! r|d! | d"< |d# | d#< nd| d#< |
j | d | d | d" | j
d#d t( d$t* t* d% |dz dk( rt d&| d't | t j d d}d}dd l}|j/ t0 }|j3 d( |j3 d) |j5 }|
D ]V } |j3 d*| d+ | d, | d, | d | d! | d# | d# xs dd d- | d. f |j6 dkD r|dz
}n|dz
}X |j; |j= t d0| d1| d2 y # t8 $ r }t d/| |dz
}Y d }~d }~ww xY w)3Nr) days%Y-%m-%dzFull crawl (cutoff: re rE totalr r Total: z , pages: i r* publishDaterC r urlDomain redirectLinklink/grassroots/content/ contentIdr )
content_idr dateurl is_redirectz Page r; z items ?zAfter filter: r r sthjjr r r r -r page_urlr r site_namecolumnz
Detail: rL PRAGMA journal_mode=WALPRAGMA busy_timeout=10000INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category) VALUES (?,?,?,?,?,?,?,?)r r r
DB error: Result: new, skipped) r nowr strftimer& r9 r< intr. rangeminBASE_URLboolrw rR timesleep enumerater@ r r# COLUMNsqlite3connectDB_PATHexecutecursorrowcountr5 commitclose) months_backcutoffr* r page_size total_pages all_itemspgitemsitempub_dater? r enrichedrW rV _stypedetailstoredskippedr conncr8 s r% crawl_allr sR llnykB.> ??
I
I*
UF
vha() 1 D HHWa EHZ()I9$q(Y6K'% + /0IAs;?C01 r" $
D /52s;H& +r2 cdhh~r6R cVZV^V^_egiVj cqypz {O PT U` Pa Ob ocJ "hh{ 33jTXXn=U=iY]YaYabhYi8jK "; / '2. ! *
gbTCJ"23 4
3/2 FG ??7 #DLL*+LL,-
A
II, k"D$4d:6Fg ^ 4d9oy/'R#.X@
A zzA~! 1 KKMJJL(6(& 23 *QC ! qLG s +AN33 O<OOc
t j t d z
j d } t d| d t d }|sy |j
dg }|D cg c] }|j
d xs d d d
| k\ s |" }}g }|D ]$ }|j
d d xs6 |j
d d xs" |j
d
d xs t d|d }|j
d xr- t |j
d xs |j
d
}t | \ } }
|rdnd} | r t | | nd d d d} |j | d xs |j
dd ||j
d xs d d d
| d t dt t d t j d ' d}
d}dd l}|j# t$ }|j' d |j' d |j) }|D ]V } |j' d|d |d |d |d |d |d |d xs d d d |d f |j* dkD r|
dz
}
n|dz
}X |j/ |j1 t d"|
d#| d$ y c c}w # t, $ r }t d!| |dz
}Y d }~d }~ww xY w)%N r r zIncremental (since re rE r* r rC r r r r r r r r r r r r r r r r r r r r r r r r r r r )r r r r r& r9 r< r r r@ r rw r# r r r r r r r r r r5 r r )r r* r itfilteredr r r? r rV r r r r r r r r r8 s r% crawl_incrementalr s llnya 00
:
::
FF
fXQ'( 1 D HHVR E "Srrvvm'<'BCR&HF&RSHSH XXk2. _$((>22N _RVRZRZ[aceRf _mulv wK LP Q\ L] K^ k_
((;//fD.9Q9eUYU]U]^dUe4f z*a&G.2 dE*"VXeg8hG_="(=" !XXm 4 :CR @ i( 66(+
3 FG ??7 #DLL*+LL,-
A
II, k"D$4d:6Fg ^ 4d9oy/'R#.X@
A zzA~! 1 KKMJJL(6(& 23Y TN *QC ! qLG s% $ JJ AJ J?"J::J?__main__rE incrementalfullr r r* z r rC r z | r < )r )$ )(__doc__sysosr4 r r r0 r r bs4r urllib.parser getenvr r# r r rs r2 r. r3 r& r9 r@ r^ r` rT r r r __name__rR argvmoder* r< r r r" r' r% r s ) ( ( (
"))K!2
3$ & '
J+
,,
D1F ) @ @A.&W4t34l z chh-!+388A; D }
q!(,!$! geW 88F#)r2A. YB B}-3Sb9:#bffWR>PQTRT>U=VW X Y
r'