j " l d Z ddlZddlZddlZddlZddlZddlZddlZej j dej j ej j e ddl
mZ dZdZdZddd d
dd d d
d
d dd
dZdZddZddZd Zedk( r ej0 e yy)u
crawl_ahhz_public.py - 徽州区政府 public 栏目 (www.ahhz.gov.cn)
站点: 帝联NWAF + 瑞数4代双WAF (与 ahshx 同款)
过盾: playwright + playwright-stealth
列表: GET /huangshanzwgk/zwgk/site/label/8888
labelName=publicInfoList&siteId=6793339&organId={organ}&pageSize=15&pageIndex=N
&isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&isJson=true&catId={cat}
详情: /zwgk/public/{organ}/{contentId}.html, meta ArticleTitle/PubDate, 正文 div.wzcon.j-fontContent
栏目 (--col):
hpysp_a : 徽州区-环评第一次公示/意见反馈 organ=6616219 cat=6726923
hpysp_b : 徽州区-环评公示(询价/第一次公示) organ=6616219 cat=6726922
nlj : 徽州区农业农村局-公开公示 organ=6616193 cat=6726922
进程锁: flock 站点级串行化
N)push_to_searchdbzhttps://www.ahhz.gov.cn6793339z/tmp/ahhz_public.lock66162196726923u 徽州区-环评第一次公示)organcatname6726922u 徽州区-环评公示6616193u% 徽州区农业农村局-公开公示)hpysp_ahpysp_bnljzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c , t t d } t j |t j t j
z |S # t $ r t d| d t j | z }t j |k rv t j |t j t j
z |cY S # t $ r t j d Y nw xY wt j |k rvt d Y yw xY w)u3 站点级互斥锁, 阻塞等待最多 max_wait 秒wu< [LOCK] 另一 ahshx 实例运行中, 等待释放 (最多 zs)...
u [LOCK] 等待超时, 放弃N)
open LOCK_FILEfcntlflockLOCK_EXLOCK_NBBlockingIOErrorprinttimesleep)max_waitfddeadlines &/root/gov_crawler/crawl_ahhz_public.py acquire_lockr # s
i B
B
56
LXJV[\]99;)iikH$
B
=> "
2
iikH$ -.
s< 6A AD6C D C*'D)C**D DDc
K t | }|d }|d }|d }t dt d| d| }t d| d| d }d
d lm} d
d lm} | 4 d { }
|
j j d
dg d d { } | j t ddddd d { } | }
|
j | d { | j d { } |j |dd d { }|j d d { |j d { }t! j" d| }t% d |r |j& nd d |r|j) d nd d t+ | |r[d!|j) d v sd"|j) d v r5t% d# | j- d { d d d d { y
g }t3 }d
}t5 d |d z D ]W }|j7 t1 t9 t; d&| d'| d(d |) }|j= d*| d+ d { }d,|v rt% d-|d, n|j? d.d
}|j? d/g xs g }|s nt% d0| d1t+ | d2| d3 |D ] }|j? d4d5 }|s||vs |jA | |jC |jE d6 r|nt |z |j? d7d5 jG |j? d8 xs d5d d9 d: |d;z |k\ sX n t% d<| d=t+ | d> g } tI | D ]- \ } }t% d?| d z d@t+ | dA|d7 d dB dC |j |dD ddE d { }|j dF d { |j d { }|d7 } t! j" dI| }|r>|j) d jG r |j) d jG } |dJ } t! j" dK| }|r7t! j" dL|j) d } | r| j) } d5}!t! j" dM| }|st! j" dN| }|r |j) d jG }!|!s3|jK dO }"|"d
kD r |jK dP|" d z }#||#|#dQz }!|!jG s
t% dR t! jL dSdTt z d@z |! }!t! jL dUdVt z d@z |! }!| jC | |dD | ||!d5dW 0 | j- d { d d d d { t% dXt+ dY | rtO | d&| t+ | S 7 7 7 o7 R7 =7 #7
7 7 \7 N# t. $ rN}t% d$t1 | d d% | j- d { 7 Y d }~d d d d { 7 y
d }~ww xY w7 57 7 7 # t. $ r%}t% dGt1 | d dH Y d }~:d }~ww xY w7 7 # 1 d { 7 sw Y xY ww)ZNr r r zO/huangshanzwgk/zwgk/site/label/8888?_=0.{rand}&labelName=publicInfoList&siteId=z &organId=z&pageSize=15&pageIndex={page}&isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&result=&isJson=true&keyWords=&isSetValue=true&catIds=&catId=z/zwgk/public/column/z?type=4&catId=z&action=list&nav=3r )async_playwright)StealthTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNi i )widthheightz
Asia/Shanghai)
user_agentlocaleviewport timezone_iddomcontentloadedi )
wait_untiltimeouti. z([^<]*)z [LOAD] status=?z title= z len=NWAF Environmentu [WAF] 仍被拦截z
[LOAD ERR] x ahshx__i)randpagezOasync () => {
try {
const r = await fetch('zv');
return await r.json();
} catch(e) { return {error: e.message}; }
}errorz [API ERR] totaldatau 第u 页: u 条 (total=)link httptitle publishDater )urlr@ date z [LIST] total=u , 去重后 u 条z [/z] ( z...rB i0u i z [DETAIL ERR] d z/]*>([\s\S]*?)\s*\s*]*>([\s\S]*?)\s*zclass="wzcon j-fontContent">i N u [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")r@ rB pub_date site_namecontentsummaryu 共 u
条有效)(COL_MAPBASESITE_IDplaywright.async_apir" playwright_stealthr# chromiumlaunch new_contextUAapply_stealth_asyncnew_pagegotowait_for_timeoutrK researchr statusgrouplenclose Exceptionstrsetrangeformatabshashevaluategetaddappend
startswithstrip enumeratefindsubr )$col_keypagescolorgan_idcat_idrJ API_TPLLIST_URLr" r# pbrowserctxstealthr8 resphtmlme items_allseenr: pgapi_urlr; itemsitr= recordsidxr@ rI dmrK idx2starts$ r run_asyncr 6 s
'
C7|H
ZFF I 33:)9XJbbhaikG +H:^F8K]^H5* ! n n Q
)) \n *
''2g;?31O]l ( n n)))#...\\^#
88JTYZZD''. . . 'D 3T:A $DT[[c$B'XY!''RS*_bIcchilmqirhst uf
*mqwwqz.I,- mmo%%-n n n : u519% B nn#c$y"7N2O.P*QRURV*W^`naG ,,,39 5( D $ T']O45HHWa(EHHVR(.BE E"U3u:,l5'C D
vvfb)D ,HHTN $$'+v'>tD4K!#!4!:!:!2B$GH + - GC CAwaI /r"W+cr2B1C3G H
!YYr%y=OY^Y __++D111 !\\^+
wKE LdSAQWWQZ%%'
((*&zH GNAYY 3QWWQZ@ !xxzHG suyzAII bdhi''!***,yy!>?!8 IIc4 01 4E "5u 5G==?,-ffZD 3)>HGffY$(X?#["X# [X[X[ X)7X8X)X X)(X )B X)X#X)
[X&[ A0[ZA>[[A>[A[-ZZZ Z !Z8Z 9Z=G[[ [ [[9[[[[[X) X) X)#X)&[) Z 2-Y; Y" Y;%[) [4Y75[;Z [Z Z Z Z=Z82[8Z==[[[
[
[[c
dd l } | j }|j ddddj t j z |j dt d |j d
d |j }|j t vr7t d
|j dt t j yt }|y t |j }t d|d d|d d|d d|j t j t |j |j s |j nd }t d| d t# j$ |t" j& |j) y# t# j$ |t" j& |j) w xY w)Nr z--colxitou_zbu 栏目键: ,)defaulthelpz--pagesr1 )typer z--all
store_true)actionu [ERR] 未知栏目 u
, 可选: z r z (organ=r z cat=r z) pages=2 )rq zDone: z records)argparseArgumentParser add_argumentjoinrM keysint
parse_argsrr r listr rq asynciorunr allr r LOCK_UNr_ )r parserr& lock_fdrr cnts r mainr sr
$
$
&F
-#((SZS_S_SaJb:bc
Q7
5 D xxw
#DHH:ZW\\^8L7MNOnGdhh
3v;-xG ~U3u:,htzzl[\kk)DHHdhhDJJTVWX
se8$%
GU]]+
GU]]+
s
BF# #6G__main__)i )r1 )__doc__sysosrZ r jsonr r pathinsertdirnameabspath__file__ crawler_libr rN rO r rM rU r r r __name__exit r r s / . . . 277??277??8#<= > (
# #9>_`"9>VW"9>ef w&AH2 z CHHTV r