j9 d Z ddlZddlZddlZddlZddlZddlZddlZddlZ ej j dej j ej j e ddlmZ dZdZdZdZdZd Z eej0 D ]k \ ZZej7 d
r e ej; d d d Z4ed k( s:ed z e ej0 k sS e ej0 ed z Zm ed k\ rend Z dd
l!m"Z" ddl#m$Z$ d Z%d Z&d Z'd Z(d Z)d Z*d Z+ddZ,e-dk( r& ej\ e,e Z/ e0de/ d yy# e $ r Y w xY w# e $ r Y w xY w) u`
旌德县人民政府 — 基层政务公开专题 环评审批 (www.ahjd.gov.cn)
列表: /Jczwgk/showList/0/111001001/page_{N}.html (26页, 15条/页)
YYYY-MM-DD标题
详情: /Jczwgk/show/{id}.html
标题: meta ArticleTitle | 日期: meta PubDate(取前10)
正文: div.m-dttexts.f-clearfix.j-fontContent#zoom (深度计数闭合)
入库: crawler_lib.push_to_searchdb → /root/search.db (FTS触发器自动维护)
注意: 列表链接是 /Jczwgk/show/ 而非 /OpennessContent/show/ (与 hpgs 脚本不同)
N)push_to_searchdbu 旌德县环评审批zhttps://www.ahjd.gov.cn 111001001z
2020-01-01zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 z--pages==z--pages)async_playwright)Stealthc ( t dt d| dS )Nz/Jczwgk/showList/0/z/page_z.html)BASE COLUMN_ID)pages &/root/gov_crawler/crawl_ahjd_jczwgk.pylist_urlr ) s V&ykvU CC c g }t j d| t j D ] }|j d j |j d j |j d j }}}|d d }|rt j
d| s~|j
t |z ||d |S )NzZ\s*([^<]*)\s*]*title="([^"]*)"[^>]*>r
z^20\d{2}-\d{2}-\d{2}$)urltitledate)refinditerSgroupstripmatchappendr
) html_textitemsmr hrefr s r
parse_listr" - s E
[[e244 I GGAJ , , .
0@0@0BAGGAJDTDTDVEdCRyBHH%=t D
TD[5$GHI Lr c R t j d| }|st j d| }|sy| |j d }d}t j d| D ]G }|j d j d r |dz }|dk( s.|d|j
c S |dz
}I |S ) u- 提取 id=zoom 容器（深度计数闭合）z8]*class="[^"]*m-dttexts[^"]*"[^>]*id="zoom"[^>]*>z]*id="zoom"[^>]*> Nr ]*>|r r searchendr r
startswithstart)r m_opensegdepthmms r
extract_zoomr0 ; s
YYRT]
^F 6 B
FJJLM
"C
Ekk0#6
88A; ! !( + QJEz;BHHJ'' QJE
Jr c * t j dd| t j t j z } t j dd| t j t j z } t j dd| t j } d } || d } t j dd| t j } t j d d| } d
D ] }| j | }|d kD s| d | } | j S ) Nzr$ flagsz
c | } t j || }|s |S ||j d }d}|j }t j d| D ]U }|j d j d r.|dz }|dk( s.|j |j z } n|dz
}W |d |j
||d z })Nr r% r r& r' )htmlpatoutr r- r. r) r/ s r
strip_blockz clean_body..strip_blockS s #s#A
aeegh-CE%%'Ckk"8#>
88A;))(3QJE z eeg 0 QJE
jqwwy/CI-C r z\]*class="[^"]*(?:wzewm|relInfo|fxd_close|m-share|m-related-info|m-dtcode)[^"]*"[^>]*>z%]*class="clear"[^>]*>\s*z
\s*
)u 中央政府网站u 省级政府网站u 市级政府网站u 关于我们u 主办单位：u 网站标识码u 本网站支持IPv6u 政务新媒体矩阵u 旌德县人民政府u 用微信扫描二维码u 分享至好友和朋友圈u 打印u 关闭窗口u! 扫一扫在手机打开当前页r )r subr Ifindr )bodyr9 kwis r
clean_bodyr@ N s
66("d"$$+ FD
66&Drtt DD
66-T 6D( t | }D
66:BBDD QD
66-T *D4
IIbM q58D
::<r c g }t }t j d| t j D ]} }|j d }t
j j || }||vs9|j | |j |t j j | xs df t j d| t j }|r#t j d|j d t j D ] }|j d }t j dd|j d j! }t j d d| j! }t
j j || }||vs|j | |j ||xs5 t j j |j# d
d xs df |S ) u[ 从详情页提取 PDF object / 下载链接 作为附件 (审批结果公示页正文=PDF)zG]*data="([^"]+\.(?:pdf|docx?|xlsx?|pptx?|zip|rar|ofd))"[^>]*>r 附件z8]*class="[^"]*m-dtdownload[^"]*"[^>]*>(.*?)z$]*href="([^"]+)"[^>]*>(.*?)<[^>]+>r$ r u 【.*?】?r )setr r r; r urllibparseurljoinaddr ospathbasenamer( r r: r split) r page_urlattsseenr r! abs_urlm_dltxts r
extract_attachments_from_htmlrT t s
D 5D
[[cenprptpt
u Jwwqz,,&&x6 $ HHW KK"''"2"27";"GxH I J 99PR[]_]a]a bD DdjjQRmUWUYUYZ cA771:D&&R4::PQR>S-T&`X` ab c Kr c d}t j d| }|st j d| }|r4t j dd|j d j }d}t j d| }|st j d| }|r"|j d j d d }t | }|r t
| nd}|s&t | d }|rd
j d |D }|sy |||| d S )
Nr$ z'name="ArticleTitle"\s+content="([^"]+)"z!ArticleTitle"\s+content="([^"]+)"\s+ r z"name="PubDate"\s+content="([^"]+)"z PubDate"\s+content="([^"]+)"r
c 3 4 K | ] \ }}d | d| d yw)
N ).0r names r
z parse_detail.. s, C se#5dV8D s )r pub_dater= raw) r r( r: r r r0 r@ rT join)r r r ra r= rO s r
parse_detailrd s E
]*>.*?z]*>.*?c < | j d }| j d }t j dd| }|j }t j dd| j }|s5t j
j
|j d d xs d}t j j | }t j d |t j }|rj ||f d
t d }d | d
| d|< |S )Nr r rC r$ u ^附件[:：]\s*rD r rB z:\.(pdf|docx?|xlsx?|pptx?|zip|rar|wps|et|dps|txt|ofd)(\?|$)__LINK____rZ r[ r\ )r r r: r rJ rK rL rM rF rG rH r( r; r len)
r r! innerrS rQ is_attachkey attachments link_protectrN s
r
_link_replz html_to_text.._link_repl s wwqz
ffZU+iikff("c288:''""4::c?1#56B(C,,&&x6II []dfhfjfjk ~ .\*+2. *7)3EcU( S S
r z&]*href="([^"]+)"[^>]*>(.*?)c | j d }t j j | }t j
d|t j ryd| dS )Nr z'ueditor|editor|icon|logo|qrcode|ewm|btnr$ rZ u& " target="_blank">查看图片)r rF rG rH r r( r; )r srcabs_srcrN s r
_img_replz html_to_text.._img_repl sN ggaj,,&&x5
99?"$$ O gY&LMMr z ]*src="([^"]+)"[^>]*>c h j | j d dt dz
dS )Nr __TBL__r rh )r r ri )r
table_protects r
_tbl_replz html_to_text.._tbl_repl s3 QWWQZ(]+A-.b11r z]*>.*?z ]*>r z
z
z(]*>rX rC rW z z [\s\u200b\u200c\u200d\ufeff]+ru rh z
__LINK__\d+__z __TBL__\d+__z[ \t]+\nz\n{3,}z [ \t]{2,}rV )r r: r r; r( rM r replacer enumerater rc rE rI )contentrN content2ro rs rw has_tablepartsblockbplainkvr? tblr8 rP finalprl rm rn rv s ` @@@r
html_to_textr s 1byff]BrttAI vvgz8244@HvvlD("$$?HvvA2xWYW[W[\H E '
KKM z2q)
fc*228SA7UC &&( DAq !QA
. 0FAs GA3b/3/A 0
QWWY
++e
C ""$ 1kk!Q M* 03kkGA3b/3/0
&&!2s
+C
&& "c
*C
&&dC
(C
&&FC
(C
&& sC
(C 5D E
YYv
ffVR
B!:;cTk
Q ++e
"
"
$C ; &&r c
K t 4 d { }|j j dddg d { }|j t dddd d { }t }|j
| d { |j d { }d }t d
D ] } |j t d d d
d { }|j d d { |j d { } d| v sd| v rd} n;t d| d|r |j nd d |j d d { |s5t d |j# d { d d d d { yg } d } | }
t d | d z D ]# }t | } |j% d| d { } t' | }t d | d t) | d |s|d kD r nΌZt+ j, d | }|r7t/ |
t1 |j3 d xs |j3 d! }
|D ]> }|d" r&|d" t4 k rd} t d#| d$t4 d% n| j7 | @ | r n7|j t9 j: d&d' d { ||
k\ s$ n t d(t) | d)|
| s*|j# d { d d d d { yg }t= | d D ]` \ }}|d* } |j% d| d { } t? | }|s$t d+| d,t) | d.|d/ d d0 X|d/ xs |d/ }|d1 xs |d" }tA |d2 | \ }}}t) t+ jB d3d4| jE }d5|v }|d6k r(|s&t d+| d,t) | d7|d d0 d8| d9}|j7 tF ||||tI jJ |d : d; t d+| d,t) | d<|d d d= |j t9 j: d>d? d { c tM |d@A } |j# d { | cd d d d { S 7 7 w7 W7 77 "7 7 7 # t $ r(}
t d| dt! |
d d Y d }
~
d }
~
ww xY w7 7 7 7 G# t $ r }
t d | d|
Y d }
~
d }
~
ww xY w7 7 D7 67 # t $ r(}
t d+| d,t) | d-|
Y d }
~
Td }
~
ww xY w7 7 7 # 1 d { 7 sw Y y xY ww)BNTz---disable-blink-features=AutomationControlledz --no-sandbox)headlessargsiV i )widthheight)
user_agentignore_https_errorsviewportF r domcontentloadedi` )
wait_untiltimeouti z Jczwgk/showz m-liststyle2z [WAF] attempt=z status=rD u 无列表特征z [WAF] goto retry z: < u [FATAL] 过盾失败r zasync (url) => {
const r = await fetch(url, {credentials: 'include'});
return await r.text();
}z [LIST-ERR] page=u 第u 页: u 条u# 共\s*(\d+)\s*页|pagecount="(\d+)"r r u [CUTOFF] 第u 页出现 < u 日期, 停止翻页i i u [LIST] 共收集 u 条, total_pages=r z [/z] [ERR] z ] [NO-BODY] r ( ra r= rC r$ zrx ) ensure_ascii) site_namer ra r r} rm u
] 最新: z...i i ahjd_jczwgk) batch_label)'r chromiumlaunch new_contextUAr apply_stealth_asyncnew_pagerangegotor wait_for_timeoutr} printstatus Exceptionstrcloseevaluater" ri r r( maxintr CUTOFFr randomrandintr| rd r r: r SITE_NAMEjsondumpsr ) pagesr browserctxstealthr okr? respr e items_all
cutoff_hit total_pagespgr r m_pcitrecordsdr ra r} r rm plain_len has_linksnew_cs r
run_asyncr s ! i iQ
))4 ;^?
) ''t #s3 (
)))#...\\^# q .A
@ !YYx{?Q[`Y aa++D111"&,,. 0 I- 91L B(84DKKS3QQabc ''- - - . * +--/ ! !;i i i@
519% B2,C
"&-- 1 # y)E E"U3u:,d3 4699 CYOD !+s4::a=3QDJJqM/R S
%f:"V*v"5!%JN2$l6(BWXY $
% ''tT(BC C C[ ; > #C N#33Ek]ST--/ ! !Ki i iP y!, DEArU)C
"&-- 1 # Y'AA3aI /|BwK??
@ -
"9iL
*2$b45
(
D
"IiX
A3aI /xs;<
*
D Qi i i is VR,V#U2R/"U2R2 "U2>R5?U2R8U2, S R;
S%R>&S=S>S U2 S.U2S8&U2)S;*U2/ V:S>;V $U2%T:T;T?C.U2-T,. U292U2+T/,U21 V<T2=VU2 T82T53T87D5U2,U,-'U2U.U2 V&U0'V/U22U25U28U2;S>SS S5
S0*U20S55U2;U2>VT T)
T$ U2$T))U2/U22V5T88 U) U$ U2$U))U2.U20V2V8U; 9V V__main__)r zDone: z records)r )1__doc__sysrJ r asyncior r time urllib.parserF rK insertdirnameabspath__file__ crawler_libr r r
r r r
_MAX_PAGESr| argvr? ar* r rM
ValueErrorri _PAGESplaywright.async_apir playwright_stealthr r r" r0 r@ rT rd r r __name__runnr r] r r
r s > = = = 277??277??8#<= > ( # v
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA 1 &D &#L2R8]'@jZ z IF+,A F1#X
u
s$ -E )E( E%$E%(E0/E0