j $ d Z ddlZddlZddlZddlZddlZddlZej j dej j ej j e ddl m
Z
dZdZdZdZdZd
d Zdd
Zd Zed k( r e yy)u
crawl_hngy_tzgg.py - 桂阳县-通知公告 (www.hngy.gov.cn)
站点: 湖南郴州桂阳县 (MMB1 防护, 与 zixing 同款; Server: core, MM_ cookie)
过盾: playwright-stealth, goto 后 DOM 即真实页面
列表: 纯静态分页 default.htm + default_N.htm, total=9512 条 (475页), 每页 20 条
列表项: YYYY-MM-DD标题
详情: content_XXXX.html, meta ArticleTitle/PubDate/ContentSource, 正文 div.xl-content (截断 div.xl-weixin 二维码)
N)push_to_searchdbzhttps://www.hngy.gov.cnz/zwgk/zwdt/tzggz/tmp/hngy_tzgg.locku 桂阳县-通知公告zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c , t t d } t j |t j t j
z |S # t $ r t d| d t j | z }t j |k rv t j |t j t j
z |cY S # t $ r t j d Y nw xY wt j |k rvt d Y y w xY w)Nwu; [LOCK] 另一 hngy 实例运行中, 等待释放 (最多 zs)...
u [LOCK] 等待超时, 放弃)
open LOCK_FILEfcntlflockLOCK_EXLOCK_NBBlockingIOErrorprinttimesleep)max_waitfddeadlines $/root/gov_crawler/crawl_hngy_tzgg.py acquire_lockr s
i B
B
56
KH:UZ[\99;)iikH$
B
=> "
2
iikH$ -.
s< 6A AD6C D C*'D)C**D DDc
K ddl m} ddlm} | 4 d { }|j j ddg d d { }|j
t dd d
d d
d { } | }|j | d { |j d { }t t z dz } |j |dd d { } |j d d { |j d { }
t j d|
} t# d| r | j$ nd d| r| j' d nd dt) |
| r[d| j' d v sd| j' d v r5t# d |j+ d { d d d d { yg }
t1 }t3 d| dz D ]D }|dk( r|}nt t z d |dz
d z } |j |dd d { } |j d! d { |j d { }
|j5 d% d { }|st# d&| d' nd}|D ]G }|d( s |d( |vs|j7 |d( |
j9 |d( |d) |d* d+ |dz
}I t# d,| d-t) | d.| d/t) |
d0 |j5 d1| d2 d { }|r6t# d&| d3 n t# d4t) |
d5 g }t; |
D ] \ }}t# d6|dz d7t) |
d#|d8 d d9 d: |j |d; dd d { } |j d! d { |j d { }
|d8 }t j d=|
} | r>| j' d j= r | j' d j= }|d> }t j d?|
} | r7t j d@| j' d }|r|j' }dA}t j dB|
} | r | j' d j= }|s7t j dC|
} | r | j' d j= }|j= s
t# dD t j> dEdFt z d7z | }t j> dGdHt z d7z | }|j9 ||d; |t@ |dAdI |j+ d { d d d d { t# dJt) dK |r tC |dL t) | S 7 7 u7 T7 77 "7 7 7 7 17 ## t, $ rN} t# d t/ | d d |j+ d { 7 Y d } ~ d d d d { 7 yd } ~ ww xY w7 $7 7 # t, $ r)} t# d"| d#t/ | d d$ Y d } ~ Md } ~ ww xY w7 7 s7 7 7 # t, $ r%} t# d/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNi i )widthheightz
Asia/Shanghai)
user_agentlocaleviewport timezone_idz /default.htmdomcontentloadedi )
wait_untiltimeoutip z([^<]*)z [LOAD] status=?z title= z len=NWAFzEnvironment Checkingu [WAF] 仍被拦截z
[LOAD ERR] x z /default_z.htmi0u i z
[LIST ERR pz] d a () => {
const out = [];
document.querySelectorAll('li').forEach(li => {
const a = li.querySelector('a[href*="content_"]');
if (a) {
const sp = li.querySelector('span');
out.push({t: a.innerText.trim(), h: a.href, d: sp ? sp.innerText.trim() : ''});
}
});
return out;
}z [pu ] 无列表项, 停止htd)urltitledateu 第u 页: u
条, 新增 u (累计 )z2() => document.querySelectorAll('a[href*="default_z.htm"]').length > 0u ] 无下一页u [LIST] 去重后 u 条z [/r. ( z...r- z [DETAIL ERR] z/]*>([\s\S]*?)\s*]*class="[^"]*xl-weixinz-class="xl-content[^"]*"[^>]*>([\s\S]*?)u [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")r. r- pub_date site_namecontentsummaryu 共 u
条有效 hngy_tzgg)"playwright.async_apir playwright_stealthr chromiumlaunch new_contextUAapply_stealth_asyncnew_pageBASELIST_DIRgotowait_for_timeoutr6 researchr statusgrouplenclose Exceptionstrsetrangeevaluateaddappend enumeratestripsub SITE_NAMEr )pagesr r pbrowserctxstealthpage first_urlresphtmlme items_allseenpgr- itemsnew_rowsithas_nextrecordsidxr. r4 dmr6 s r run_asyncrk ' s 5* ! s s Q
)) \n *
''2g;?31O]l ( n n)))#...\\^# 8On4
99KUZ[[D''- - - 'D 3T:A $DT[[c$B'XY!''RS*_bIcchilmqirhst uf
*.DPQ
.R,- mmo%%/s s s < u519% $ BQw Xo)BqD6(>>
!YYs7ISXY YY++D111 !\\^+ --
)
E RD 678H
"c7r#wd 2HHRW% $$RWr#wPRSVPW%XY MH
"
E"U3u:,mH:YsS\~N^^_` a !]]-`ac`ddx+y zzHRD/0I$ L #C N#3489 + ( GC CAwaI /r"W+cr2B1C3G H
!YYr%y=OY^Y __++D111 !\\^+ wKE LdSAQWWQZ%%'
((*&zH GNAYY 3QWWQZ@ !xxzHG kmqrA''!***,II NPTU ggaj . . 0G==?,-ffZD 3)>HGffY$(:;
* { `1+
)#a&#, 89
F gs s s s s4 [V4[$Z0V7#Z0&V:' Z0V=Z0 W Z04W W W$W%W<W =B WW W! [,W-[2>Z01X5X, X5!X/"X59X2:X5>Z0Y*#Z07Z0?A;Z0:Y-;Z0A%Z0)Y9Y0Y9 Y3 Y94Y65Y99F5Z0.Z*/Z03 [>Z-?6[7Z0:Z0=Z0 Z0WW W W[ X)-X$X X$Z0 [ X [$X))Z0,X5/X52X55 Y'> Y"Z0"Y''Z0-Z00Y93Y96Y99 Z'Z" Z0"Z''Z0-[0[6Z9 7[>[c dd l } | j }|j dt d |j }t }|sy t
j t |j }t d| d t j |j y # t j |j w xY w)Nr z--pages )typedefault)rV zDone: z records)argparseArgumentParser add_argumentint
parse_argsr asynciorunrk rV r osrJ fileno)rp parserr lockns r mainr| s
$
$
&F
Q7 D >D KK
3 4
qc"#
s 8B' '%C __main__)i )rm )__doc__sysrw rE ru r r pathinsertdirnameabspath__file__ crawler_libr rA rB r rU r> r rk r| __name__ r r s ) ( ( 277??277??8#<= > ( ! $ v${| zF r