j; d Z ddlZddlZddlZddlZddlZddlZddlZdZ e dz Z
dZ dZ dZ
dZd Z dd
lmZ ej& j) d d Zej& j) d
d ZdddddZdZ eej4 D ]k \ ZZ e j; d r e e j? dd d Z4e dk( s:edz e!ej4 k sS e ej4 edz Zm edk\ rendZ"dZ# ejH d ejJ ej e#dz z
Z&h d Z'd Z(d Z)d Z*d Z+d&d!Z,d" Z-d# Z.d$ Z/e0d%k( r e/ yy# e$ r dZY 9w xY w# e $ r Y w xY w# e $ r Y w xY w)'u^ 舒城县人民政府 - 基层重点领域信息公开 (site/label/8888 JSON API 聚合栏目)
https://www.shucheng.gov.cn/site/tpl/5186?pid=7367495&id=7367498&organId=6596321
Lonsun 龙讯 CMS: label API 返回全部子栏目(惠民惠农/新闻发布/政府开放日等) 共7935条
用户要求: 新增站点只测前5页, 不跑全量 (2026-08-18) Nzhttps://www.shucheng.gov.cnz/site/label/8888u4 舒城县人民政府-基层重点领域信息公开u 安徽zcrawl_shucheng_jcdd.pyu+ 基层重点领域信息公开-行政审批7367498)
BeautifulSoupDB_PATHz/root/search.db
JSONL_PATH zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36zzh-CN,zh;q=0.9zPhttps://www.shucheng.gov.cn/site/tpl/5186?pid=7367495&id=7367498&organId=6596321XMLHttpRequest)z
User-AgentzAccept-LanguageRefererzX-Requested-With z--pages== z--pagesiG z%Y-%m-%diQ >) pdddldth1h2h3h4h5h6hrlioltdthtruldivnavpreformmainasidetabletbodytfoottheadcenterdialogfigurefooterheaderhgroupaddressarticlecaptiondetailssectionfieldset
blockquote
figcaptionc | yddl }ddlm g fd | dj }|j dd| }|j dd | }|j S )
ul 块级边界出换行、行内节点直接拼接、
出换行（≈ 浏览器看到的换行结构）。Nr r )NavigableStringc F | j D ] }t | rj t | *t |dd dk( rj d Lt |dd t
v r+j d | j d | y )Nnamebr
)children
isinstanceappendstrgetattr _BLOCK_TAGS)nodechr7 outwalks (/root/gov_crawler/crawl_shucheng_jcdd.pyrE zbody_text..walkE s --
B"o.
3r7#VT*d2
4 VT*k9
4 R
4 R
z[ \t\r\f\v]*\n[ \t\r\f\v]*r; z\n{3,}
)rebs4r7 joinsubstrip)el_retr7 rD rE s @@@rF body_textrQ = sb z#
C H
A -tQ7A 61%A 779rG c | sydd l }|j | } t j dd| } | j S )Nr r z\s+ )htmlunescaperI rL rM )rP html_libs rF clean_titlerW X s9 ! A
vsA A 779rG c dd l }dd l}|j |j j dddt
dt
| dddd d
d ddd } |j t |t d
}|j dk7 r t d|j dd g dfS |j }|j dg }|j dd }||fS # t $ r }t d| d g dfcY d }~S d }~ww xY w)Nr publicInfoList6788831659632120truez
yyyy-MM-dd504listr ) labelNamesiteIdorganIdcatIdspageSize pageIndexisDate
dateFormatlengthtypeactionresultisJson
isSetValue )paramsheaderstimeout z [API status ]Tflushdatatotalz
[API fail] )requestsurllib3disable_warnings
exceptionsInsecureRequestWarningCAT_IDSr? getAPI_URLHEADERS status_codeprintjson Exception) pagery rz rp rrw itemsrx es rF fetch_apir a s W / / F FG% Y" F LL"L M ==C N1==/34 @q5Lvvx$!$e|
aS!.1u s%
A
C 7C C7C2,C72C7c dd l }dd l}|j |j j |j | t d }|j dk( rXd|j j d xs dj vr |j j dd
S |j S y # t $ r}t d | d
Y d }~y d }~ww xY w)Nr ro )rq rr rs charsetz Content-Typer utf-8replace)errorsz [fetch fail] Tru )ry rz r{ r| r} r r r rq lowercontentdecodetextr r )urlry rz r r s rF fetchr s W / / F FG1 LLgrL : ==C ~!>!D" K K MMyy'' 'BB66M 1
s#400 1s A2B0 " B0 0 C9CCc t yt t | d }|j d D ]P }|j dd }|s|j d r*|r t
j j || n||d< R |j d D ]P }|j dd }|s|j d r*|r t
j j || n||d< R t | S ) Nr html.parserahref)http
javascript#imgsrc)r zdata:r )r r? find_allr
startswithurllibparseurljoin)r$ base_urltblr r r r s rF html_table_to_htmlr s
E
M
2C
\\#
SuuVR (CD@H ,,Xtu 查看原文z)r
r r$ r ) recursive)r
r r$ r
r r )r appendixr downloadOLDSRC/r z./ r )r r zz
r r altz//zhttps:z 277c>
77g ".."9
77l ";;s$;/Dww% }[[4$[?
Dy($/H155R3Hx,393D3DS3IX .PSPZPZ[^`aPbcdPehkPkntPtEt, #
3 21 5 ;d12h F - (4 0 $ #
3 21 5 ;d B""VE#BC
D s4& -.
WW
)"c2H X&
WW
bnnW&=&&#C&&#C>>$'"S.C^^C("S.C / **S!,Q/#5;C c3?RZ? [\W+]Z
!
!#D
!
1 Dy II5tRTT BdooV\F]ghglglmwgxZZdZ+DquuZ/DF$$,s2sgr2s/sx,393D3DS3IX .PSPZPZ[^`aPbcdPehkPkntPtEt, #
3 21 5 ;d12h F - (4 0 $ #
3 21 5 ;d B""VE#BC#D( $$S$5I Y{'7'78T'U
Y
%%c%5
bAV9??$Dv&.i s 4!QuX 4s r r -r Fr c 3 , K | ] }|d r d yw)r r Nr r r s rF r zmain..v s 7a!I,7
c 3 , K | ] }|d s d yw)r r Nr r s rF r zmain..w s ;!!M*:A;r z5[RESULT] %s: %d new, %d empty content, %d attachments)r r range_PAGESr rW r r r r> r _CUTOFF_DATEr r rK rI r intr r r r sumr ) all_items
total_recordsr r rx r r r r filteredr resultsrT dattsatt_text
date_finalrankr empty att_counts rF r" r" E s
"Y
./IMa!$ X u GD6 / 0
2B w 34E66&"%DFF=)GRVVL-AGR"MD??6* $ dE4 0 1 2 vRE
|>- JRVW X" H & -eT D<' ud+,-
/3y>3x=2Q
QRG % eTT{ NN4%RTeg ~B QR S
T t $uu]B'99 4t 44UU6]*d
II,j 9 qwwqz))#r23D!%%."9E:uuY+H $
4e D
(
777 7E;w;;I
AYPST[P\^cenDo
opW rG __main__)r )1__doc__sysosrI timer urllib.parser r r r r r r CATEGORYr~ rJ r ImportErrorenvironr r r r
_MAX_PAGES enumerateargvir r r r
ValueErrorr r _DAYS_CUTOFFstrftime localtimer rA rQ rW r r r r r r" __name__r rG rF r' s F 6 5 5 5 (
'
' B
& 8
!
**..$5
6
ZZ^^L"
-
@'a(
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA t}}Z lUZFZ8Z)[\ * 6 B HfV$44 n zF S M$
s5 E& "E4 E?&E10E14E<;E<?FF