!j d Z ddlZddlZddlZddlZddlZddlmZ ddlm Z dZ
e
dZ dZ ej j dd dd lmZ d
Zd Zed k( r ej( e yy)
u
威海南海新区政务服务平台 - 通知公告 (whnhzwfw.sd.gov.cn/nanhai/govservice/notice)
站点: 山东政务服务网 威海市·南海新区 (icity 框架)
列表: /nanhai/govservice/notice -> JS onQuery(N) 翻页 (limit=8, 实际渲染10含置顶)
API: POST /nanhai/api-v2/app.icity.govservice.GovProjectCmd/getPoliciList/execute
(AES-ECB加密 + URL签名, 直接在页面上下文执行 onQuery 绕开)
详情: /nanhai/icity/publishdetail?id={uuid}
标题 #title, 发布机关 #deptname, 成文日期 #writetime, 发布日期 #ctime, 正文 #contentDetail
注意: https 被 TLS 拦截, 必须用 http + playwright stealth
N)async_playwright)Stealthzhttp://whnhzwfw.sd.gov.cnz/nanhai/govservice/noticeu 威海南海新区-通知公告z/root/gov_crawler)push_to_searchdbc K |4 d{ t d D ] } t j | j |dd d d{ | j d d{ | j d d{ }|d
s|d r|c cddd d{ S ddd d{ y7 7 k7 T7 =7 !# t $ ra}|d k( r1t d
t | j dt | dd n | j d|dz z d{ 7 Y d}~ d}~ww xY w7 |# 1 d{ 7 sw Y yxY ww)u" 抓详情页: 标题/日期/正文N domcontentloadedȯ
wait_untiltimeout2 r i u () => {
const g = (id) => { const el = document.getElementById(id); return el ? el.textContent.trim() : ""; };
const body = document.getElementById("contentDetail");
let html = "";
if (body) {
// 移除脚本/style, 保留表格与段落
const clone = body.cloneNode(true);
clone.querySelectorAll("script,style,iframe,object").forEach(e => e.remove());
html = clone.innerHTML;
}
return { title: g("title"), dept: g("deptname"), ctime: g("ctime"), writetime: g("writetime"), html: html };
}htmltitle z [DETAIL ERR] x i ) rangeasynciowait_forgotowait_for_timeoutevaluate Exceptionprinttype__name__str)pageurlsemattemptinfoes $/root/gov_crawler/crawl_whnh_tzgg.py fetch_detailr' sN Qx FG
F &&tyyAS]by'cmoppp++C000 !]] , <4= K% " $1! F. 1 q0 &
Fa< -d1g.>.>-?qQ OP //! 0DEEE
F' s EB3ED,,B= B5
B="B7#B=;B9<B= D,
EB;E D," E-D*.E5B=7B=9B=;E= D'AD"D
D" D,"D''D,*E,D>2D5 3D>:Ec
K t j } | j dt dd | j dt dd | j }g }t 4 d { }|j j dd g
d { }|j d d d
dd d { }t }|j | d { |j d { }t d D ] } t j |j t dd d d { |j# d d { |j% d d { } | r nt' d|dz d t1 } t d|j2 dz D ]L } | dkD r6|j% d"| d# d { |j# d$ d { |j5 d%d& d { }
d}|
D ]e }|d' j7 }|d( j7 }|r|s.d)|v r3|| v r8| j9 | |j; |||d* d+ |dz
}g t' d,| d-t= |
d.| d/t= | |j> rt= | |j> k\ r n<|dk( r| dkD r n0|j% d0 d { }|s=t= | |k\ sM n t' d1t= | d2 |j d { }t j@ d3 }d}|D ]r }tC ||d4 | d { }|dz
}|r)||d5< t' d6| d7t= | d8|d9 d d: Kd |d5< t' d6| d7t= | d;|d9 d d: t |jE d { d d d d { g }|D ] }|jG d5 xs i }|jG d9 xs |d9 }|jG d< xs |d* xs d=} |jG d>d= } | s d?|d9 d@} |j; tH ||d4 |d4 | | |d dA |jG dBd= dC tK |dDE t' dFt= | dG y 7 87 7 7 7 7 7 m7 W# t( $ rg}
t' d |dz d t+ |
j, d t/ |
d d |d k( r |j# d!|dz z d { 7 Y d }
~
+d }
~
ww xY w7 l7 V7 ?7 b7 7 7 ~7 q# 1 d { 7 sw Y xY ww)HNz--pages u 最大页数)r defaulthelpz--maxr u 最大条数(0=全部)Tz --no-sandbox)headlessargszoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36iV i )widthheightzzh-CN)
user_agentviewportlocale r r r
r
r i z typeof onQuery === 'function'z [RETRY] page r z loaded but no onQueryz
[RETRY] z/5 r d i zonQuery()i z#roll_list1 tra els => els.map(e => {
const a = e.querySelector("a");
const tds = e.querySelectorAll("td");
return { t: a ? a.textContent.trim() : "", h: a ? a.href : "", date: tds.length > 1 ? tds[tds.length-1].textContent.trim() : "" };
})thzwhzwfw.sd.gov.cn/wh/date)r r! r9 z [LIST] page z: rows=z new=z total=zlayerpageAllTotal || 0u 共 u 条列表, 开始抓详情...r r! detailz [/z] r ( z] FAIL ctime r zz
dept) site_namer
source_urlr! contentpub_datesummarysource whnh_tzgg) batch_labelzDone: z records)&argparseArgumentParser add_argumentint
parse_argsr chromiumlaunch new_contextr apply_stealth_asyncnew_pager r r r LIST_URLr r r r r r r setpageseval_on_selector_allstripaddappendlenmax Semaphorer' closeget SITE_NAMEr ) parserr- resultspbrowserctxstealthr r# okr% seenpgrowsnew_cntrr7 r8 totalpage2r" doneitemr$ recordsitdr r9 rC s r& mainrs 4 s
$
$
&F
RnM
c1;ST DG ! I I Q
))4~>N)OO'' I #s3G ( E E )))#...\\^# Qx BG
B &&tyyFXbgy'hrtuuu++D111 ==)HII {2HIJ B u4::>* BAvmmhrd!$4555++D111 223C F D
G
cFLLNcFLLN )Q. 9 1ai HI1
LGCI;eG9GCPWL>Z [xxCLDHH 4!|Q --(@AAETe+? B S\N"@ABlln$ " ND%eT%[#>>D AID!%XD63w<.4="3E2FGH!%XD63w<.W
cr8J7KLM N mmo SI I X G FF8 "g-"W+uuW~1F1r%%#BwK=-G" U)e9 Tc{eeFB'
" W+6 F3w<.
)*I OE /#
v1I
B
719+Sa1A1A0B!CF4CL>RSa<++DGaK,@AAA
B 612 B % ? SI I I I s A'U)R!*U-"UR$ U.R'/"UR*U)R-*U=0R9-R0.R9R3R9 R6 R9&U(R9:AU T,UT/U3T24C UT5U U,.UT83UT;A1U T>U UUCU$U'U*U-U0R93R96R99 T)AT$T
T$ U$T))U/U2U5U8U;U>UUU
U
UU__main__)__doc__r jsonresysrI playwright.async_apir playwright_stealthr BASErS r_ pathinsert crawler_libr r' rs r run r& r ss ( ' ' 1 &"V, - - & ' (8f+P zGKK r