j , d Z ddlZddlZddlZddlZddlZddlZddlZej j dej j ej j e ddl
mZ dZdZdZdZdZd
d Zdd
Zd Zed k( r e yy)u~
crawl_ahtxq_tzgg.py - 屯溪区-通知公告 (www.ahtxq.gov.cn)
站点: 帝联NWAF + 瑞数4代双WAF (与 ahshx/黄山 同款)
过盾: playwright + playwright-stealth
列表: 首页 /zxzx/tzgg/index.html (10条/页) + 翻页 API GET /content/column/6793870?pageIndex=N
返回 HTML 片段 (li 列表), 页面内 fetch 即可
列表项: YYYY-MM-DD标题
详情: /zxzx/tzgg/{id}.html, meta ArticleTitle/PubDate/ContentSource, 正文 div.wzcon.j-fontContent
内容含: 突出生态环境问题整改验收销号公示 / 接访安排 / 帮扶名单公示 等
N)push_to_searchdbzhttps://www.ahtxq.gov.cnz/https://www.ahtxq.gov.cn/content/column/6793870u 屯溪区-通知公告z/tmp/ahtxq_tzgg.lockzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c , t t d } t j |t j t j
z |S # t $ r t d| d t j | z }t j |k rv t j |t j t j
z |cY S # t $ r t j d Y nw xY wt j |k rvt d Y y w xY w)Nwu< [LOCK] 另一 ahtxq 实例运行中, 等待释放 (最多 zs)...
u [LOCK] 等待超时, 放弃)
open LOCK_FILEfcntlflockLOCK_EXLOCK_NBBlockingIOErrorprinttimesleep)max_waitfddeadlines %/root/gov_crawler/crawl_ahtxq_tzgg.py acquire_lockr s
i B
B
56
LXJV[\]99;)iikH$
B
=> "
2
iikH$ -.
s< 6A AD6C D C*'D)C**D DDc n
K ddl m} ddlm} | 4 d { }|j j ddg d d { }|j
t dd d
d d
d { } | }|j | d { |j d { } |j t dz dd d { }|j d d { |j d { } t j d| }
t! d|r |j" nd d|
r|
j% d nd dt' | |
r[d|
j% d v sd|
j% d v r5t! d |j) d { d d d d { yg } t/ }
d }|j d { } || }|s|j1 d d { }t! d t' | d! |D ]5 }|d" s |d" |
vs|
j3 |d" | j5 | 7 t7 d#| dz D ] }t8 d$| }|j1 d%| d& d { }t; |t, r'|j= d' rt! d(| d)|d d* n || }|st! d(| d+ npt! d,| d-t' | d! |D ]5 }|d" s |d" |
vs|
j3 |d" | j5 | 7 |j d. d { t! d/t' | d! g }t? | D ] \ }}t! d0|dz d1t' | d2|d3 d d4 d5 |j |d" dd6 d { }|j d7 d { |j d { } |d3 }t j d:| }
|
r>|
j% d jA r |
j% d jA }|d; }t j d<| }
|
r7t j d=|
j% d }|r|j% }d>}t j d?| }
|
r |
j% d jA }|s3| jC d@ }|dkD r | jC dA| dz }| ||dBz }|jA s
t! dC t jD dDdEt z d1z | }t jD dFdGt z d1z | }|j5 ||d" |tF |d>dH |j) d { d d d d { t! dIt' dJ |r tI |dK t' | S 7 7 7 |7 _7 J7 )7 7 7 b7 T# t* $ rN} t! d t- | d d |j) d { 7 Y d } ~ d d d d { 7 yd } ~ ww xY w7 7 i7 7 #7 7 7 # t* $ r%} t! d8t- | d d9 Y d } ~ ,d } ~ ww xY w7 7 # 1 d { 7 sw Y xY ww)LNr )async_playwright)StealthTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNi i )widthheightz
Asia/Shanghai)
user_agentlocaleviewport timezone_idz/zxzx/tzgg/index.htmldomcontentloadedi )
wait_untiltimeouti@ z([^<]*)z [LOAD] status=?z title= z len=NWAF Environmentu [WAF] 仍被拦截z
[LOAD ERR] x c X g }t j d| D ] }|j d j |j d j |j d j }}}|j d r t
|z }|j
|||d |S )NzQ]*>\s*]*>([^<]*)\s*]*href="([^"]*)"[^>]*title="([^"]*)"r& /)urltitledate)refinditergroupstrip
startswithBASEappend)houtmmr0 hrefr/ s r extract_from_htmlz$run_async..extract_from_htmlL s Ckk"vxyz
H$&HHQK$5$5$7!9J9J9LbhhWXkN_N_NaEd??3' $;D
4%FG
H
J a () => {
const out = [];
document.querySelectorAll('ul li').forEach(li => {
const a = li.querySelector('a');
if (a && /\d{6,}\.html/.test(a.href)) {
const sp = li.querySelector('.date');
out.push({url: a.href, title: (a.getAttribute('title')||a.innerText||'').trim(), date: sp ? sp.innerText.trim() : ''});
}
});
return out;
}u 第1页: u 条r. r+ z ?pageIndex=zfetch('z/').then(r=>r.text()).catch(e=>'ERR:'+e.message)ERRz [pz
] fetch ERR: P u ] 无更多u 第u 页: i u [LIST] 去重后 z [r- z] r/ ( z...i0u i z [DETAIL ERR] d z/]*>([\s\S]*?)\s*zclass="wzcon j-fontContent">i N u [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")r/ r. pub_date site_namecontentsummaryu 共 u
条有效
ahtxq_tzgg)%playwright.async_apir playwright_stealthr chromiumlaunch new_contextUAapply_stealth_asyncnew_pagegotor6 wait_for_timeoutrF r1 searchr statusr3 lenclose Exceptionstrsetevaluateaddr7 range
COLUMN_API
isinstancer5 enumerater4 findsub SITE_NAMEr ) pagesr r pbrowserctxstealthpageresphtmlme items_allseenr< itemsitpgapi_urltxtrecordsidxr/ rD dmrF idx2starts r run_asyncry ) s 5* ! ~ ~ Q
)) \n *
''2g;?31O]l ( n n)))#...\\^#
4*A#AN`joppD''- - - 'D 3T:A $DT[[c$B'XY!''RS*_bIcchilmqirhst uf
*mqwwqz.I,- mmo%%-~ ~ ~ : u \\^#!$' --
)
E CJ?!8 IIc4 01 4E "5u 5G==?,-ffZD 3)>HGffY$(D KK
3 4
qc"#
s 8B' '%C __main__)i )r{ )__doc__sysr r1 r jsonr r pathinsertdirnameabspath__file__ crawler_libr r6 r] rb r rN r ry r __name__ r= r r s / . . . 277??277??8#<= > (!
>
$ " v$FR zF r=