Gj, d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl Z dZ
e
dz Z dZ dZ
dZdd Zd
Zd Zd Zed
k( r e yy)u 弋阳县人民政府 (www.jxyy.gov.cn) 通知公告爬虫 — UCAP CMS + 电信NWAF + 瑞数双层防护
用法: python3 crawl_jxyy_tzgg.py [--pages N] [--out /tmp/jxyy_tzgg.jsonl]
过盾: 服务器 playwright + playwright_stealth.Stealth (NWAF 405 拉黑本地IP, 服务器IP可过)
列表: li > h4 > a[title] + span.time, 分页 list.shtml / list_{N}.shtml
详情: div#zoomcon > ucapcontent, meta ArticleTitle/PubDate/ContentSource
N)urljoinzhttp://www.jxyy.gov.cnz/jxyy/tzgg/list.shtml jxyy_tzggzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36)u 请稍候NWAF405u 访问u 拦截c h ddl m} |xs t } || d }|j d D ] }|j d xs d}|s%|j d }|r|j dd }|s)|j dd d
i }|r|j d d }|s%|j d
} | r| j dd }|rK|j
d r|n t || }
|j d|
} d| _ |j | |j |g d D ] } | j |j d D ]J } dD ] }
|
| j v s| |
= t | j D ] }
|
j
d s| |
= L |j d D ] }|j d xs |j d xs d}|rK|j
d r|n t || }
|j d|
} d| _ |j | x|j |j g d D ] } | j |j d D ]] } | j dd }|r4|j
d s#|j
d r|n t || | d< K|sN| j _ |j d D ]I }|j d r|j d r(|j d r:|j K t! | S ) u 白名单清洗: 保留 /
/, unwrap 装饰标签, 清 inline style, 链接绝对化
base_url: 详情页 URL (图片/附件相对路径以此为基址)r
BeautifulSoup html.parservideosrc sourceparamnameurlattrsvalueembedhttpahrefu 查看视频)scriptstyleiframeobjectr T)r classalignvalignborder cellpadding cellspacingwidthheightbgcolorfacecolorsizelangdir setedariatabindexrolezaria-imgzdata-srcu 查看图片)divspanfontcenterbstrongemiuslabelh1h2h3 ucapcontentr
javascriptpstriptable)bs4r BASEfind_allgetfind
startswithr new_tagstring replace_with decomposer listunwrapget_textstr)htmlbase_urlr basesoupvr r9 r@ eabs_srcr tattrr/ r s crawl_jxyy_tzgg.py
clean_htmlr\ s " tD } -D
]]7
# eeEl bx AeeE2&wvuo6AeeGR(wAeeE2& ^^F 3cs9KG Sw /A%AH
NN1
KKM)* B
C
]]4
6 D qwwdG
M Dw'dG
}}U# ggen9
39r ^^F 3cs9KG Sw /A%AH Q MMO ]] K L
]]3
uuVR 5 $ 7WT4=PAfI
KKM ]]3
zzz%affWoaffSk
KKM t9 c ddl m} || d }g }|j d D ] }|j dd }|s|j dd }t j d
| sB|j d xs |j d xs d j }|j d
d }|r|j d nd } |s|j d s|j |t t | | d t g } }
|D ]/ } | d |
vs |
j | d | j | 1 | S )u/ 列表页解析: li > h4 > a[title] + span.timer r r
lir Tr r r
z(/jxyy/tzgg/20\d{2}\d{2}/[a-f0-9]+\.shtmltitlerA r1 timeclass_.shtml)r` r dater )rD r rF rH rG researchrP rB endswithappendr rE setadd)
rR r rU itemsr_ r r r` r1 re seenoutits
r[ parse_list_htmlrp U s- ! } -D EmmD! U GGCdG # uuVR yyDdK w ?1::D:#9?RFFHwwvfw-,0t}}4}(b T]]8, LL5t1DdS T U r#D
e9D HHRY JJrN Jr] c ddl m} || d fd} |d xs d} |d xs d} |d xs d}j d d
}|sj d d
}d}|r|j d } | s|} g }
| j dd D ] } | d j } t j d| s-| j d xs d}
| j d r| n t || }|
j |
|f | j t t | | }|
r.dj d |
D }|dz |z j }||r|dd nd|||r
D
cg c]
\ }
}|
|d c}}
d S g d S c c}}
w ) u5 详情页解析: meta 字段 + #zoomcon > ucapcontentr r r
c | j dd| i }|r#|j d xs dj S dS )Nmetar r contentr
)rH rG rB )r mrU s r[ rs z parse_detail_html..metat s= IIfVTNI 334i &B--/<" r Tr r z>\.(pdf|doc|docx|xls|xlsx|zip|rar|caj|wps|et|jpg|jpeg|png|gif)$rA u 附件r c 3 4 K | ] \ }}d | d| d yw)z z
N ).0rY fus r[ z$parse_detail_html.. s# TBl2$b8 <Ts
N
)fileNamefileUrl)r` publish_datert r attachments)rD r rH rF lowerrf rg rP rI r ri rM r\ rQ joinrB )rR r r rs r` pubr zoombodyucap att_linksr hrY r~ att_prU s @r[ parse_detail_htmlr p s ! } -D=
&BE
y/
RC
/
"
(bF 99Uy9 )D yy'=y>
D yy 'D s . A& !AyyZ\]^JJTJ*6h,,v.QGCO !R)
#d)S) GGT)TTE4K%'..0D$'CRRRVIN51bQ26N
]_
Os .Fc t j } | j dt dd | j dd | j dt d
| j }dd lm} dd lm} dddddd
}t |j dd } | 5 }|j j dg d }|j t ddddd }|j | j! dR fd } | t" s't% dd d d d |j' y j) }
d } t+ j, d |
} | rt/ d | D } |j0 dkD rt3 | |j0 } t% d | d! j5 d d" d g }
t7 d | d z D ] }|d k( rt" nt" j9 d#d$| d% }|d kD r+ | | st% d&| d'd n j) }
t; |
}|
j= | t% d&| d(t? | d)t? |
d tA jB tE jF d*d+ t? |
|d,< t% d-t? |
d.d tI } dd l%}tL jN jQ d/d0 }|jS | }|jU d1 |jU d2tV f D ] }|jY |d |j' t% d3t? | d4t? |
t? |tI d5 |
D z z
d.d t] |
d D ] \ }}|d6 |v r | |d6 d78 }|sR|d9xx d z
cc< t% d:| d;t? |
d<|d6 d |d9 |j^ k\ rt% d=d nEpta j) |d6 }|d> s|d> |d>< |d? s|d@ |d?< |dA }t? | d k r
|dBxx d z
cc< |d> |d? ||d6 |d6 tV t+ jb dCdD| je d dE d|dF |dF |dG dH }|jg ti jj |dIJ dKz |dLxx d z
cc< |d z dk( s|t? |
k( r(t% d:| d;t? |
dM|dL dN|d9 d tA jB tE jF dOdP |j' d d d |j' t% dQti jj |dIJ d y # tZ $ r Y _w xY w# 1 sw Y OxY w# |j' w xY w)SNz--pagesr u 0=自动探测全量)typedefaulthelpz--outz/tmp/jxyy_tzgg.jsonl)r z--max-detail-failr )r r )sync_playwright)Stealth)list_okrl detail_ok detail_failemptywzutf-8)encodingT)z---disable-blink-features=AutomationControlledz --no-sandboxz--disable-dev-shm-usage)headlessargsiV i )r$ r% zzh-CNz
Asia/Shanghai)
user_agentviewportlocale timezone_idc j | dd t | D ]V }t j d j xs dt
fdt D rEj sV y y# t $ r Y pw xY w) u+ goto + 等待过盾 (title 不含屏蔽词)i` domcontentloaded)timeout
wait_until r
c 3 & K | ] }|v
y wNr| )r} r4 rY s r[ r z(main..wait_ok.. s .wait_ok s} IIc5=OIP u $AJJqM
*A <| << # $
! s A= = B B u 过盾失败 (列表页))flush zlist_(\d+)\.shtmlc 3 2 K | ] }t | y wr )int)r} xs r[ r zmain.. s !4Q#a&!4s u 过盾成功, 总页数=z, title=( z
list.shtmllist_rd u 页u
过盾失败z: u
条, 累计 g333333?g @rl u 列表完成: u 条 SEARCH_DBz/root/search.dbzPRAGMA busy_timeout=10000z.SELECT page_url FROM gov_raw WHERE site_name=?u
DB 已有 u
条, 待抓 c 3 & K | ] }|d yw)r Nr| )r} r7 s r[ r zmain.. s ^{lm_`af_g^{s r )r r u 详情 /u : 过盾失败 u 失败过多, 中止r` r re rt r z<[^>]+> r r ) r` r rt page_url
source_url site_namesummary date_rankauthorcontent_sourcer F) ensure_asciir r z: ok=z fail=g?g ?u 完成: ) )6argparseArgumentParser add_argumentr
parse_argsplaywright.sync_apir playwright_stealthr openrn chromiumlaunch new_contextUAnew_pageapply_stealth_syncLIST_URLprintclosert rf findallmaxpagesminr` r replacerp extendlenra r randomuniformrj sqlite3osenvironrG connectexecute SITE_NAMErk r enumeratemax_detail_failr subrB writejsondumps) apr r r statsout_fpr@ browserctxr rR pages_totalru all_itemspgr rl existingr dbpconnrowidxro okdrt recr s @r[ mainr s "BOOIC9OOPOOG%;O<OO'c2O>
==?D3*AAaRS TE
$((C'
2Fi
f !jj'' q ( sG %% $#(F O & =C <<>D I ( ( .
8$0=5f P [ <<>DK
/6A !!4!!4 4 zzA~ !+tzz : ,[M$**,sPRBSATU]a b IA{Q/
5"$'hx/?/? PUVXUYY_N`/a6"3< B4} 5TB <<>D'- 'B4r#e* ]3y>:JKSWX
6>>#s34
5 !^E'N N3y>"2$7t D uH
jjnn[2CD s+ 89 <<(X[dZfg )C LLQ()
Js8}o]3y>CPX[^^{qz^{[{P{L|;|:} ~B C KO
P$Y2 $
5Re9( RYa0-(A-(GC5#i.)9 E T\`a ]+t/C/CC 4DA %dllnbi@z!#GAgJ ((*6
An%I,w<"$'Na'N wZ$%n$5 & "5 "$U)!* !vvj#w ? E E G M!" k&'k#$]#3 TZZ% @4 GHk"a'"8q=C3y>$9GC5#i.)9u[?Q>RRXY^_lYmXnow{|
6>>#s34I$
5J
MMOMf P HTZZE:;
BV2 V> 9FV2 BV"HV2$V> " V/+V2.V//V22V;7V> >W__main__r )__doc__r r rf sysra r urllib.parser r rE r r r r r\ rp r r __name__r| r] r[ r sg - , , ) ) v? =@6%PvJr zF r]