j* , d Z ddlZddlZddlZddlZddlZddlZddlZej j dej j ej j e ddl
mZ dZdZdZdZdZd
d Zdd
Zd Zed k( r e yy)u
crawl_zixing_hjzl.py - 资兴市-环境治理及建设项目公示 (www.zixing.gov.cn)
站点: MMB1 防护 (Server: core, MM_ cookie) — 非瑞数4, stealth goto 后 DOM 即真实页面
列表: 纯静态分页 default.htm + default_N.htm (N=1..31), total=622, 每页 20 条
列表项: 标题YYYY-MM-DD
详情: content_XXXX.html, meta ArticleTitle/PubDate/ContentSource, 正文 div.content
N)push_to_searchdbzhttp://www.zixing.gov.cnz%/zwgk/ztbd/zwgkgzydzl/zlhms/hjbf/hjzlz/tmp/zixing_hjzl.locku+ 资兴市-环境治理及建设项目公示zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c , t t d } t j |t j t j
z |S # t $ r t d| d t j | z }t j |k rv t j |t j t j
z |cY S # t $ r t j d Y nw xY wt j |k rvt d Y y w xY w)Nwu= [LOCK] 另一 zixing 实例运行中, 等待释放 (最多 zs)...
u [LOCK] 等待超时, 放弃)
open LOCK_FILEfcntlflockLOCK_EXLOCK_NBBlockingIOErrorprinttimesleep)max_waitfddeadlines &/root/gov_crawler/crawl_zixing_hjzl.py acquire_lockr s
i B
B
56
MhZW\]^99;)iikH$
B
=> "
2
iikH$ -.
s< 6A AD6C D C*'D)C**D DDc x
K ddl m} ddlm} | 4 d { }|j j ddg d d { }|j
t dd d
d d
d { } | }|j | d { |j d { }t t z dz } |j |dd d { } |j d d { |j d { }
t j d|
} t# d| r | j$ nd d| r| j' d nd dt) |
| r[d| j' d v sd| j' d v r5t# d |j+ d { d d d d { yg }
t1 }t3 d| dz D ]. }|dk( r|}nt t z d |dz
d z } |j |dd d { } |j d! d { |j d { }
|j5 d% d { }|st# d&| d' nt# d(| d)t) | d* |D ]B }|d+ s |d+ |vs|j7 |d+ |
j9 |d+ |d, |d- d. D |j5 d/| d0 d { }|r t# d&| d1 n t# d2t) |
d* g }t; |
D ]' \ }}t# d3|dz d4t) |
d#|d5 d d6 d7 |j |d8 dd d { } |j d! d { |j d { }
|d5 }t j d:|
} | r>| j' d j= r | j' d j= }|d; }t j d<|
} | r7t j d=| j' d }|r|j' }d>}t j d?|
} | r | j' d j= }|j= s3|
j? d@ }|dkD r |
j? dA| dz }|
||dBz }|j= s
t# dC t j@ dDdEt z d4z | }t j@ dFdGt z d4z | }|j9 ||d8 |tB |d>dH * |j+ d { d d d d { t# dIt) dJ |r tE |dK t) | S 7 7 i7 H7 +7 7 7 7 7 %7 # t, $ rN} t# d t/ | d d |j+ d { 7 Y d } ~ d d d d { 7 yd } ~ ww xY w7 7 7 # t, $ r)} t# d"| d#t/ | d d$ Y d } ~ Wd } ~ ww xY w7 7 }7 7 7 # t, $ r%} t# d9t/ | d d$ Y d } ~ rd } ~ ww xY w7 <7 /# 1 d { 7 sw Y @xY ww)LNr )async_playwright)StealthTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNi i )widthheightz
Asia/Shanghai)
user_agentlocaleviewport timezone_idz /default.htmdomcontentloadedi )
wait_untiltimeoutip z([^<]*)z [LOAD] status=?z title= z len=NWAFzEnvironment Checkingu [WAF] 仍被拦截z
[LOAD ERR] x z /default_z.htmi0u i z
[LIST ERR pz] d a () => {
const out = [];
document.querySelectorAll('li').forEach(li => {
const a = li.querySelector('a[href*="content_"]');
if (a) {
const sp = li.querySelector('.time');
out.push({t: a.innerText.trim(), h: a.href, d: sp ? sp.innerText.trim() : ''});
}
});
return out;
}z [pu ] 无列表项, 停止u 第u 页: u 条htd)urltitledatez2() => document.querySelectorAll('a[href*="default_z.htm"]').length > 0u ] 无下一页u [LIST] 去重后 z [/r. ( z...r- z [DETAIL ERR] z/]*>([\s\S]*?)zclass="content">i N u [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")r. r- pub_date site_namecontentsummaryu 共 u
条有效 zixing_hjzl)#playwright.async_apir playwright_stealthr chromiumlaunch new_contextUAapply_stealth_asyncnew_pageBASELIST_DIRgotowait_for_timeoutr6 researchr statusgrouplenclose Exceptionstrsetrangeevaluateaddappend enumeratestripfindsub SITE_NAMEr ) pagesr r pbrowserctxstealthpage first_urlresphtmlme items_allseenpgr- itemsithas_nextrecordsidxr. r4 dmr6 idx2starts r run_asyncrm & s 5* ! v v Q
)) \n *
''2g;?31O]l ( n n)))#...\\^# 8On4
99KUZ[[D''- - - 'D 3T:A $DT[[c$B'XY!''RS*_bIcchilmqirhst uf
*.DPQ
.R,- mmo%%/v v v < u519% $ BQw Xo)BqD6(>>
!YYs7ISXY YY++D111 !\\^+
--
)
E RD 678 E"U3u:,d3 4
Zc7r#wd 2HHRW% $$RWr#wPRSVPW%XY
Z
"]]-`ac`ddx+y zzHRD/0I$ L #C N#3489 + + GC CAwaI /r"W+cr2B1C3G H
!YYr%y=OY^Y __++D111 !\\^+ wKE LdSAQWWQZ%%'
((*&zH GNAYY 3QWWQZ@ !xxzHG GNA''!***,==?yy!23!8 IIc4 01 4E "5u 5G==?,-ffZD 3)>HGffY$(:;
( { `1+
)#a&#, 89
L mv v v v s4 Z:V(Z:$Z$V+#Z$&V.' Z$V1Z$ V4 Z$4W V7 W$V:%W<V==B WW W! Z:,W-Z:2>Z$1X)X X)!X#"X)9X&:X)>Z$Y D KK
3 4
qc"#
s 8B' '%C __main__)i )ro )__doc__sysry rE rw jsonr r pathinsertdirnameabspath__file__ crawler_libr rA rB r rV r> r rm r~ __name__ r r s / . . . 277??277??8#<= > (! 2 # 9 v$~B zF r