ʂj 8 d Z ddlZddlZddlZddlZddlZddlZej j dej j ej j e ddl m
Z
dZdZdZdZd dZd Zed
k( r ej* e yy) u
crawl_cnqc_zdly.py - 青川县人民政府-重点领域信息公开 (cnqc.gov.cn)
站点: 四川广元青川县 (瑞数4代+帝联NWAF 双WAF, 与 ahshx/gdqy 同版本)
过盾: playwright-stealth, 全程 goto (fetch 被瑞数拒, goto 过盾后 ~7s/页)
列表: /New/List/20171128112728617.html?page=N (重点领域信息公开, 50页)
ul > li > a[title]+span日期, 详情 /New/Detail/xxx.html
详情: meta ArticleTitle/PubDate/ContentSource, 正文 div.msg-content (截断到二维码 div_div 前)
限速: 详情间 3-8s 随机抖动 (用户要求)
注意: 用户原URL(20171201155541049.htmlz)已404, 用重点领域信息公开替代
N)push_to_searchdbzhttp://cnqc.gov.cnu" 青川县-重点领域信息公开20171128112728617zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c
K ddl m} ddlm} | 4 d { }|j j ddg d d { }|j
t dd d
d d
d { } | }|j | d { |j d { } |j t dz dd d { }|j d d { |j d { } t j d| }
t! d|r |j" nd d|
r|
j% d d d nd dt' | |
rHd|
j% d v r5t! d |j) d { d d d d { yg } t/ }
t1 d| dz D ]; } |j t d t2 d | dd d { }|j d! d { |j d { } t j4 d%| }|st j4 d&| }d}|D ]` \ }}}|j7 d' r|nt |z }||
vs(|
j9 | | j; ||j= |d( |dz
}b t! d)| d*t' | d+| d,t' | d- t' | d.k s1t! d/ n g }t? | D ]u \ }}t! d0|dz dt' | d#|d1 d d2 d3 |j |d4 dd d { }|j d5 d { |j d { }d|d d6 v r t! d7 |d1 }t j d9| }
|
r>|
j% d j= r |
j% d j= }|d: }t j d;| }
|
r7t j d<|
j% d }|r|j% }d=} t j d>| }
|
r |
j% d j= } | s7t j d?| }
|
r |
j% d j= } | j= s
t! d@ t j@ dAdBt z dz | } t j@ dCdDt z dz | } |j; ||d4 |tB | d=dE |t' | dz
k sEtE jF tI jJ dFdG d { x |j) d { d d d d { t! dHt' dI |r tM |dJ t' | S 7 7 7 r7 U7 @7 7 7 7 h7 Z# t* $ rN} t! d t- | d d |j) d { 7 Y d } ~ d d d d { 7 yd } ~ ww xY w7 j7 T7 ?# t* $ r)} t! d"| d#t- | d d$ Y d } ~ d } ~ ww xY w7 ?7 )7 # t* $ r%} t! d8t- | d d$ Y d } ~ d } ~ ww xY w7 Q7 97 ,# 1 d { 7 sw Y =xY ww)KNr )async_playwright)StealthTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z --no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)headlessexecutable_pathargszzh-CNi i )widthheightz
Asia/Shanghai)
user_agentlocaleviewport timezone_id/domcontentloadediȯ )
wait_untiltimeouti: z([^<]*)z [LOAD] status=?z title= ( z len=u 请稍候u [WAF] 挑战未过z
[LOAD ERR] x z
/New/List/z .html?page=i@ ip z
[LIST ERR pz] d zx\s*]*title="([^"]*)"[^>]*>[\s\S]{0,200}?\s*(\d{4}-\d{2}-\d{2})zq]*title="([^"]*)"[^>]*>[\s\S]{0,200}?\s*(\d{4}-\d{2}-\d{2})http)urltitledateu 第u 页: 解析 u
条, 新增 u (累计 ) u [END] 列表结束z [r $ z...r i i u [WAF] 详情被拦z [DETAIL ERR] z/([\s\S]*?)\s*([\s\S]*?)
u [SKIP] 正文空zhref="/zhref="zsrc="/zsrc=")r r pub_date site_namecontentsummaryg @g @u 共 u
条有效 cnqc_zdly)'playwright.async_apir playwright_stealthr chromiumlaunch new_contextUAapply_stealth_asyncnew_pagegotoBASEwait_for_timeoutr$ researchprintstatusgrouplenclose ExceptionstrsetrangeLIST_IDfindall
startswithaddappendstrip enumeratesub SITE_NAMEasynciosleeprandomuniformr ) pagesr r pbrowserctxstealthpageresphtmlme items_allseenpgrowsnew_rowsutdr recordsidxitdhr r" dmr$ s $/root/gov_crawler/crawl_cnqc_zdly.py run_asyncrb s 5* ! g g Q
)) \n *
''2g;?31O]l ( n n)))#...\\^#
4#::LV[\\D''. . . 'D 3T:A $DT[[c$B']^!''RS*UXVX/dgIhhmnqrvnwmxy z[AGGAJ.,- mmo%%-g g g : u519% B
!YY$z'+bT'R_q |AY B B++D111 !\\^+ :: Z \` aDzz #W Y] ^H
"1a <</aTAXd?HHSM $$S1779a%PQ MH
"
E"\#d)M(9UXYbUcTddef g4y2~,-+ 0 + . >GC CAwaI /r"W+cr2B1C3G H
!YYr%y=OY^Y __++D111 <<>) "Ud)+23 , wKE LbQAQWWQZ%%'
((*&zH GLAYY 3QWWQZ@ !xxzHG Z\^_A''!***,II JBO ggaj . . 0G==?,-ffZD 3)>HGffY$(^ mmo Og g R
F3w<.
+,+. w<Yg
n /# ] .'
&+g . M#a&#, 0 1--/ ! !5g g g . B1+
bTCF4CL>:;
. `1)
)#a&#, 89
J > Og g g g s) [!W[!$[ W#[ &W' [ W[ W [ $ W0W!W0W$ W03W'4BW0W*W0 [!W-[! [ 9'Y Y
!Y9Y
:YYYA[ 0A3[ %A [ /Z Z
Z"Z #Z:Z;ZF2[ ,[ 1[2[ [
[ [! [ 6[![ [ [ [ !W0$W0'W0*W0-[!0 Y9-Y&X)'Y,[ 0 [!;X><[!Y[
Y
YY Z Z 9[ Z[ Z ZZ Z?Z:4[ :Z??[ [ [! [ [ [ [!c \ dd l } | j }|j dt d |j dd |j }|j
rdn |j }t d t d
| t j t | }t d | d
y)Nr z--pages )typedefaultz--all
store_true)action2 z z pages=)rJ zDone: z records) argparseArgumentParser add_argumentint
parse_argsallrJ r4 rE rF runrb )rj parserr
rJ cnts ra mainrs s
$
$
&F
Q7
5 D((B
E Byk
()
++ie,
-C F3%x
! __main__)rd )__doc__sysosr2 rF rH timepathinsertdirnameabspath__file__ crawler_libr r0 rE r= r, rb rs __name__exit rt ra r s * ) ) 277??277??8#<= > ( 0
vod
z CHHTV rt