ʂj[ h d Z ddlZddlZddlZddlZddlZej j dej j ej j e
ddl m Z dZ
dZdZdZdZe
d e d
e d e Ze
d e d
e dZdZddZd Zedk( r ej0 e yy)uO
crawl_huangshan_xzsp.py - 黄山市-行政许可执法决定信息 (organId=6615714, catId=6717937)
站点: www.huangshan.gov.cn (瑞数4代 + 帝联NWAF 双WAF, 与歙县 ahshx 同款模板)
过盾: playwright-stealth (必需)
列表: label/8888 API (与 ahshx 完全同款):
labelName=publicInfoList&siteId=6793339&organId=6615714&pageSize=15&pageIndex=N
&isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&isJson=true&catId=6717937
total=3357, 详情 /zwgk/public/6615714/{contentId}.html
详情: meta ArticleTitle/PubDate/ContentSource, 正文 div.wzcon.j-fontContent
N)push_to_searchdbz https://www.huangshan.gov.cnu( 黄山市-行政许可执法决定信息661571467179376793339zO/huangshanzwgk/zwgk/site/label/8888?_=0.{rand}&labelName=publicInfoList&siteId=z &organId=z&pageSize=15&pageIndex={page}&isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&result=&isJson=true&keyWords=&isSetValue=true&catIds=&catId=z/zwgk/public/column/z?type=4&catId=z&action=list&nav=3zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c
K ddl m} ddlm} | 4 d { }|j j ddg d d { }|j
t dd d
d d
d { } | }|j | d { |j d { } |j t dd d { }|j d d { |j d { } t j d| }
t! d|r |j" nd d|
r|
j% d d d nd dt' | |
r[d|
j% d v sd|
j% d v r5t! d |j) d { d d d d { yg } t/ }
d}t1 d| dz D ]X }t2 j5 t- t7 t9 d | d d | }|j; d!| d" d { }d#|v rt! d$|d# n|j= d%d }|j= d&g xs g }|s nt! d'| d(t' | d)| d* |D ] }|j= d+d, }|s||
vs |
j? | | jA |jC d- r|ntD |z |j= d.d, jG |j= d/ xs d,d d0 d1 |d2z |k\ sY n t! d3| d4t' | d5 g }tI | D ] \ }}t! d6|dz d7t' | d8|d. d d9 d: |j |d; dd< d { }|j d= d { |j d { } |d. }t j d@| }
|
r>|
j% d jG r |
j% d jG }|dA }t j dB| }
|
r7t j dC|
j% d }|r|j% }d,}t j dD| }
|
r |
j% d jG }|s3| jK dE }|dkD r | jK dF| dz } | | | dGz }|jG s
t! dH t jL dIdJtD z d7z | }t jL dKdLtD z d7z | }|jA ||d; |tN |d,dM |j) d { d d d d { t! dNt' dO |r tQ |dP t' | S 7 7 7 `7 C7 .7 7 7 7 F7 8# t* $ rN} t! d t- | d d |j) d { 7 Y d } ~ d d d d { 7 yd } ~ ww xY w7 7 7 7 }# t* $ r%} t! d>t- | d d? Y d } ~ #d } ~ ww xY w7 7 # 1 d { 7 sw Y xY ww)QNr )async_playwright)StealthTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNi i )widthheightz
Asia/Shanghai)
user_agentlocaleviewport timezone_iddomcontentloadediȯ )
wait_untiltimeouti: z([^<]*)z [LOAD] status=?z title= ( z len=u 请稍候 Environmentu [WAF] 仍被拦截z
[LOAD ERR] x hs_i)randpagezOasync () => {
try {
const r = await fetch('zv');
return await r.json();
} catch(e) { return {error: e.message}; }
}errorz [API ERR] totaldatau 第u 页: u 条 (total=)link httptitle publishDate
)urlr% date z [LIST] total=u , 去重后 u 条z [/z] & z...r( i i z [DETAIL ERR] d z/]*>([\s\S]*?)\s*zclass="wzcon j-fontContent">i N u [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")r% r( pub_date site_namecontentsummaryu 共 u
条有效huangshan_xzsp))playwright.async_apir playwright_stealthr chromiumlaunch new_contextUAapply_stealth_asyncnew_pagegotoLIST_URLwait_for_timeoutr1 researchprintstatusgrouplenclose ExceptionstrsetrangeAPI_TPLformatabshashevaluategetaddappend
startswithBASEstrip enumeratefindsub SITE_NAMEr ) pagesr r pbrowserctxstealthr resphtmlme items_allseenr pgapi_urlr itemsitr" recordsidxr% r/ dmr1 idx2starts )/root/gov_crawler/crawl_huangshan_xzsp.py run_asyncrn sf 5* ! i i Q
)) \n *
''2g;?31O]l ( n n)))#...\\^#
88JTYZZD''. . . 'D 3T:A $DT[[c$B']^!''RS*UXVX/dgIhhmnqrvnwmxy zkQWWQZ/=AGGAJ3N,- mmo%%-i i i : u519% B nn#c$RDz2B.C*DST*JQSnTG ,,,39 5( D $ T']O45HHWa(EHHVR(.BE E"U3u:,l5'C D
vvfb)D ,HHTN $$'+v'>tD4K!#!4!:!:!2B$GH + ( GC CAwaI /r"W+cr2B1C3G H
!YYr%y=OY^Y __++D111 !\\^+ wKE LdSAQWWQZ%%'
((*&zH GNAYY 3QWWQZ@ !xxzHG ^`deA''!***,yy!>?!8 IIc4 01 4E "5u 5G==?,-ffZD 3)>HGffY$(Y9Y9 A>Y9 AY99YX;Y,X>-YYY F1Y9:Y5;Y9? Z
Y7 6ZY9Y9 Y9 Y9W W W W Z X5'-X0XX0Y9 Z)X,*Z0X55Y9;Y>YY Y2
Y-'Y9-Y22Y97Z9Z ?Z Z Zc \ dd l } | j }|j dt d |j dd |j }|j
rdn |j }t d t d
| t j t | }t d | d
y)Nr z--pages )typedefaultz--all
store_true)action z z pages=)rY zDone: z records) argparseArgumentParser add_argumentint
parse_argsallrY rA rX asynciorunrn )rv parserr rY cnts rm mainr s
$
$
&F
Q7
5 D88CE Byk
()
++ie,
-C F3%x
! __main__)rp )__doc__sysosr? r| jsonpathinsertdirnameabspath__file__ crawler_libr rS rX ORGAN_IDCAT_IDSITE_IDrJ r= r9 rn r __name__exit r rm r s " ! ! 277??277??8#<= > (% 6
V //6iy
^^d]e g V'z xGY Zvqh
z CHHTV r