jD 4 d Z ddlZddlZddlZddlZddlZddlZddlZej j dej j ej j e ddl
mZ dZdZdZdZdZd Zd
Zdd Zdd Zd
Zedk( r e yy)u{
crawl_ahhz_hpgs.py - 徽州区生态环境分局-建设项目环评审批 (www.ahhz.gov.cn)
站点: 帝联NWAF + 瑞数4代双WAF (与 ahshx/黄山 同款)
过盾: playwright + playwright-stealth
列表: GET /huangshanzwgk/zwgk/site/label/8888 labelName=grassrootsContentPageList
siteId=6793339&organId=6615882&catId=1000260&pageSize=15&pageIndex=N
&isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&isJson=true
返回 JSON {data:[{title, contentId, publishDate, fileNum, author}], total, pageCount}
详情: /zwgk/grassroots/{organ}/{contentId}.html
meta ArticleTitle/PubDate/ContentSource, 正文 div.wzcon.j-fontContent (与 ahshx 一致)
栏目: 徽州区生态环境分局-建设项目环评文件（报告书、报告表）审批 (基层政务公开-生态环境)
内容: 环评批复/批前公示/受理公示, total=306, 21页
进程锁: flock 站点级串行
N)push_to_searchdbzhttps://www.ahhz.gov.cn679333966158821000260u4 徽州区生态环境分局-建设项目环评审批z/tmp/ahhz_hpgs.lockzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c , t t d } t j |t j t j
z |S # t $ r t d| d t j | z }t j |k rv t j |t j t j
z |cY S # t $ r t j d Y nw xY wt j |k rvt d Y y w xY w)Nwu; [LOCK] 另一 ahhz 实例运行中, 等待释放 (最多 zs)...
u [LOCK] 等待超时, 放弃)
open LOCK_FILEfcntlflockLOCK_EXLOCK_NBBlockingIOErrorprinttimesleep)max_waitfddeadlines $/root/gov_crawler/crawl_ahhz_hpgs.py acquire_lockr s
i B
B
56
KH:UZ[\99;)iikH$
B
=> "
2
iikH$ -.
s< 6A AD6C D C*'D)C**D DDc b
K t dt dt dt d}t dt dt }ddlm} dd lm} | 4 d { }|j j d
d g d
d { }|j t ddddd d { } | }|j | d { |j d { } | j |dd d { }
| j d d { | j! d { } t# j$ d| } t' d|
r |
j( nd d| r| j+ d nd d t- | | r[d | j+ d v sd | j+ d v r5t' d |j/ d { d d d d { yg }t5 }d}t7 d | d z D ]L }|j9 t3 t; t= d#| d$d |% }| j? d&| d' d { }d(|v rt' d)|d( n|jA d*d }|jA d+g xs g }|s nt' d,| d-t- | d.| d/ |D ] }|jA d0 }|s||vs |jC | t d1t d2| d3}|jE ||jA d4d5 jG |jA d6 xs d5d d7 d8 |d9z |k\ sM n t' d:| d;t- | d< g }tI | D ] \ }}t' d=|d z d2t- | d>|d4 d d? d@ | j |dA ddB d { }
| j dC d { | j! d { } |d4 }t# j$ dF| } | r>| j+ d jG r | j+ d jG }|dG }t# j$ dH| } | r7t# j$ dI| j+ d } | r| j+ }d5} t# j$ dJ| } | r | j+ d jG } | s3| jK dK } | dkD r | jK dL| d z } | | | dMz } | jG s
t' dN t# jL dOdPt z d2z | } t# jL dQdRt z d2z | } |jE ||dA |tN | d5dS |j/ d { d d d d { t' dTt- dU |r tQ |dV t- | S 7 7 n7 M7 07 7 7 7 7 :7 ,# t0 $ rN}
t' d!t3 |
d d" |j/ d { 7 Y d }
~
d d d d { 7 yd }
~
ww xY w7 7 7 7 }# t0 $ r%}
t' dDt3 |
d dE Y d }
~
#d }
~
ww xY w7 7 # 1 d { 7 sw Y xY ww)WNzd/huangshanzwgk/zwgk/site/label/8888?_={rand}&action=list&labelName=grassrootsContentPageList&siteId=z5&isDate=true&dateFormat=yyyy-MM-dd&length=50&organId=z&catId=zR&pageSize=15&pageIndex={page}&type=4&keyWords=&isSetValue=true&isJson=true&result=z/zwgk/grassroots/column/z?&catId=r )async_playwright)StealthTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNi i )widthheightz
Asia/Shanghai)
user_agentlocaleviewport timezone_iddomcontentloadedi )
wait_untiltimeouti@ z([^<]*)z [LOAD] status=?z title= z len=NWAF Environmentu [WAF] 仍被拦截z
[LOAD ERR] x ahhz_i)randpagezOasync () => {
try {
const r = await fetch('zv');
return await r.json();
} catch(e) { return {error: e.message}; }
}errorz [API ERR] totaldatau 第u 页: u 条 (total=) contentIdz/zwgk/grassroots//z.htmltitle publishDater )urlr6 date z [LIST] total=u , 去重后 u 条z [z] ( z...r9 i0u i z [DETAIL ERR] d z/]*>([\s\S]*?)\s*zclass="wzcon j-fontContent">i N u [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")r6 r9 pub_date site_namecontentsummaryu 共 u
条有效 ahhz_hpgs))BASESITE_IDORGAN_IDCAT_IDplaywright.async_apir playwright_stealthr chromiumlaunch new_contextUAapply_stealth_asyncnew_pagegotowait_for_timeoutrA researchr statusgrouplenclose Exceptionstrsetrangeformatabshashevaluategetaddappendstrip enumeratefindsub SITE_NAMEr ) pagesAPI_TPLLIST_URLr r pbrowserctxstealthr/ resphtmlme items_allseenr1 pgapi_urlr2 itemsitcidr9 recordsidxr6 r? dmrA idx2starts r run_asyncr / s JJQFxjPWX^W_egG /z&JH5* ! l l Q
)) \n *
''2g;?31O]l ( n n)))#...\\^#
88JTYZZD''- - - 'D 3T:A $DT[[c$B'XY!''RS*_bIcchilmqirhst uf
*mqwwqz.I,- mmo%%-l l l : u519% B nn#c$rd|2D.E*Fst*LSUnVG ,,,39 5( D $ T']O45HHWa(EHHVR(.BE E"U3u:,l5'C D
ff[)3d?HHSM !F"3H:Qse5IC $$ "!#!4!:!:!2B$GH + * GC CAwaI /r"W+cr2B1C3G H
!YYr%y=OY^Y __++D111 !\\^+ wKE LdSAQWWQZ%%'
((*&zH GNAYY 3QWWQZ@ !xxzHG ^`deA''!***,yy!>?!8 IIc4 01 4E "5u 5G==?,-ffZD 3)>HGffY$(D KK
3 4
qc"#
s 8B' '%C __main__)i )r )__doc__sysr rR r jsonr r pathinsertdirnameabspath__file__ crawler_libr rD rE rF rG rg r rM r r r __name__ r r s
/ . . . 277??277??8#<= > (
B ! v$zz zF r