j $ d Z ddlZddlZddlZddlZddlZddlZej j dej j ej j e ddl m
Z
dZdZdZdZdZd
d Zdd
Zd Zed k( r e yy)u
crawl_hanchuan_tzgg.py - 汉川市-通知公告 (www.hanchuan.gov.cn)
站点: 湖北孝感汉川市 (JPAAS 系统 .jhtml, JS混淆挑战 JbU — 真实浏览器执行后过盾)
过盾: playwright-stealth, goto 后 DOM 即真实页面
列表: index.jhtml (第1页) + index_N.jhtml (N=2..), total=160 条, 每页 10 条
列表项: li > a[href=/tzgg/{id}.jhtml] 标题 + 日期/来源
详情: /tzgg/{id}.jhtml, meta ArticleTitle/PubDate/ContentSource, 正文 div.p-xwxq-content
附件: /u/cms/hanchuan/... 相对路径转绝对
N)push_to_searchdbzhttp://www.hanchuan.gov.cnz/tzggz/tmp/hanchuan_tzgg.locku 汉川市-通知公告zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c , t t d } t j |t j t j
z |S # t $ r t d| d t j | z }t j |k rv t j |t j t j
z |cY S # t $ r t j d Y nw xY wt j |k rvt d Y y w xY w)Nwu? [LOCK] 另一 hanchuan 实例运行中, 等待释放 (最多 zs)...
u [LOCK] 等待超时, 放弃)
open LOCK_FILEfcntlflockLOCK_EXLOCK_NBBlockingIOErrorprinttimesleep)max_waitfddeadlines (/root/gov_crawler/crawl_hanchuan_tzgg.py acquire_lockr s
i B
B
56
OPXzY^_`99;)iikH$
B
=> "
2
iikH$ -.
s< 6A AD6C D C*'D)C**D DDc K ddl m} ddlm} | 4 d { }|j j ddg d d { }|j
t dd d
d d
d { } | }|j | d { |j d { }t t z dz } |j |dd d { } |j d d { |j d { }
t j d|
} t# d| r | j$ nd d| r| j' d nd dt) |
t) |
dk sd|
d d v rt# d |j d d { |j d { }
t j d|
} t# d | r| j' d nd dt) |
t) |
dk r*|j+ d { d d d d { yg }
t1 }t3 d| dz D ] }|dk( r|}nt t z d | d!z } |j |dd" d { } |j d# d { |j d { }
g }t j d'|
}|r||j' d }t j4 d(| }t j4 d)| }t7 | D ]1 \ }\ }}|t) | k r|| nd*}|j9 |||f 3 d}|D ]w \ }}}|j; d+ r|nt |z }|j= d,d* }|j? }|sH||vsM|jA | |
j9 |||d- |dz
}y t# d.| d/t) | d0| d1t) |
d2 t) | d3k st# d4| d5 n t# d6t) |
d7 g } t7 |
D ] \ } } t# d8| dz d9t) |
d%| d: d d; d< |j | d= dd" d { } |j d# d { |j d { } | d: }t j d?| } | r>| j' d j? r | j' d j? }| d@ } t j dA| } | r7t j dB| j' d }!|!r|!j' } d*}"t j dC| } | r | j' d j? }"|"j? s
t# dD t jB dEdFt z d9z |" }"t jB dGdHt z d9z |" }"| j9 || d= | tD |"d*dI |j+ d { d d d d { t# dJt) dK | r tG | dL t) | S 7 V7 17 7 7 7 7 7 7 7 7 7 v# t, $ rN} t# d t/ | d d |j+ d { 7 Y d } ~ d d d d { 7 yd } ~ ww xY w7 z7 d7 O# t, $ r)} t# d$| d%t/ | d d& Y d } ~ d } ~ ww xY w7 7 7 # t, $ r%} t# d>t/ | d d& Y d } ~ /d } ~ ww xY w7 <7 /# 1 d { 7 sw Y @xY ww)MNr )async_playwright)StealthTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNi i )widthheightz
Asia/Shanghai)
user_agentlocaleviewport timezone_idz /index.jhtmldomcontentloadedi@ )
wait_untiltimeoutiX z([^<]*)z [LOAD] status=?z title= z len=i@ JbUi u [WAF] 挑战未过u 再等: title=z
[LOAD ERR] x z/index_z.jhtmli0u i z
[LIST ERR pz] d z class="news-list"[\s\S]*?z0]*href="([^"]*\.jhtml)"[^>]*title="([^"]*)"z(\d{4}-\d{2}-\d{2}) httpz:80)urltitledateu 第u 页: 解析 u
条, 新增 u (累计 ) z [pu ] 列表结束u [LIST] 去重后 u 条z [/r- ( z...r, z [DETAIL ERR] z/]*>([\s\S]*?)u [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")r- r, pub_date site_namecontentsummaryu 共 u
条有效
hanchuan_tzgg)$playwright.async_apir playwright_stealthr chromiumlaunch new_contextUAapply_stealth_asyncnew_pageBASELIST_DIRgotowait_for_timeoutr5 researchr statusgrouplenclose Exceptionstrsetrangefindall enumerateappend
startswithreplacestripaddsub SITE_NAMEr )#pagesr r pbrowserctxstealthpage first_urlresphtmlme items_allseenpgr, rowsmsegseglinksdatesiutdnew_rowsr- recordsidxitdhr3 dmr5 s# r run_asyncrt ( s. 5* ! r r Q
)) \n *
''2g;?31O]l ( n n)))#...\\^# 8On4 99KUZ[[D''- - - 'D 3T:A $DT[[c$B'XY!''RS*_bIcchilmqirhst u4y4 ET%4[$8,-++D111 !\\^+II 7>(qc(B%D {STt9t#!--/)) 9r r r F u519% " BQw Xo'"V(<<
!YYs7ISXY YY++D111 !\\^+
D99 >EDjjm
#VX[\
#93?!*5!1 +IAv1$%E
NaAKKAq *+ H
"1a <</aTAXkk%, S_HHSM $$S5!%LM MH
"
E"\#d)M(9UXYbUcTddef g4y1}RD/0E" H #C N#3489 + $ GC CAwaI /r"W+cr2B1C3G H
!YYr%y=OY^Y __++D111 <<>) wKE LbQAQWWQZ%%'
((*&zH GLAYY 3QWWQZ@ !xxzHG NPRSA''!***,==?,-ffZD 3)>HGffY$(:;
D `1)
)#a&#, 89
> er r r r s< ^Y0^$],Y3#],&Y6' ],Y9], Y< ],4Z Y? Z$Z%Z<Z=B Z Z
Z!Z "A(Z
Z Z ^Z ^!;], [74[.5[7
[1[7%[4&[7*C!], ],A%],8A%], \58\,9\5\/\5)\2*\5.E<],*]&+],/ ^:]);6^3],6],9],<],?ZZZZ ZZ^ [+ -[&
[
[&], ^ [" ^&[++],.[71[74[77 \) \$ ],$\))],,\5/\52\55 ]#>] ], ]##],)^,]?2]5 3]?:^c dd l } | j }|j dt d |j }t }|sy t
j t |j }t d| d t j |j y # t j |j w xY w)Nr z--pagesr0 )typedefault)rW zDone: z records)argparseArgumentParser add_argumentint
parse_argsr asynciorunrt rW r osrI fileno)rx parserr lockns r mainr s
$
$
&F
Q7 D >D KK
3 4
qc"#
s 8B' '%C __main__)i )r0 )__doc__sysr rD r} r r pathinsertdirnameabspath__file__ crawler_libr r@ rA r rV r= r rt r __name__ r r s ) ( ( 277??277??8#<= > (# % $ v$zz zF r