ujn d Z ddlZddlZddlZddlZddlZddlmZmZmZ ej j dej j ej j e
ddlmZ dZdZdZ ej& ej( ed z
j+ d
Zd Zdd Zed
k( r eej6 dkD xr ej6 d dv Z dZ e ej6 D ]< \ Z Z e dk( s e dz eej6 k s% e!ej6 e dz Z > ej Z" ee e e#d ej e"z
dd yy)u
荣盛石化-新闻专区 (www.cnrspc.com/xwzq)
万网/阿里云建站 CMS，需 Playwright 渲染
列表: 标题
日期: .w-al-date
详情: .w-detail 正文, .w-createtime-date 日期
分页: N 点击翻页，共~10页
N)datetimetimezone timedelta)push_to_searchdbu 荣盛石化-新闻专区zhttps://www.cnrspc.com/xwzqzwww.cnrspc.comiG )daysz%Y-%m-%d c
H t dd dt dd t d ddlm} g }t } | 5 }|j
j
d }|j }|j t d d
t j d t }d
} t d|dz D ]X }
| r nSt d|
ddd |j d } d} | D ] }
|
d |v r |
d r|
d t k rd} n|j! |
d |
d j# }t dt% | dz d|d d ddd |j }|j |
d d d
t j d |j' }|r%t) j* d d| j# n|}|
d } |j- d }|j- d }|rU|j/ j# }|r |j/ j# nd}| d| j# }d} |j- d }|r|j1 }t) j* d!d"t2 d#| }t) j* d$d| }t) j* d%d| }t) j* d&d| }t) j* d'd(| }|j# }|j5 |sd)| d*}t) j* d+d| j# d d, }|rt) j* d-d| n|}|j7 t ||
d ||r|d d. nd|t d/ | dz
} t d0t% | d1d2 |st% | |k\ sd} n t d5| d6dd | r nE|
|k s |
dz } |j d7| } | s
t d8 nt j d [ |j5 d d d |r%t= |d9 t d:t% | d; y t d< y # Y xY w# Y qxY w# t8 $ r'}t d3t; | d d4 d2 Y d }~d }~ww xY w# 1 sw Y xY w)=N
z2==================================================u
🏠 u ⏳ 启动 Playwright...r )sync_playwrightT)headless networkidlei0u )
wait_untiltimeout F u
📄 第u 页: )endflusha () => {
const results = [];
const links = document.querySelectorAll('a[href*="/newsinfo/"]');
const seen = new Set();
for (const a of links) {
const href = a.href;
const title = a.textContent.trim();
if (!title || title.length < 5 || seen.has(href)) continue;
seen.add(href);
const itemEl = a.closest('li, div');
const dateEl = itemEl ? itemEl.querySelector('.w-al-date') : null;
const date = dateEl ? dateEl.textContent.trim() : '';
results.push({href, title, date});
}
return results;
}hrefdatetitlez
[z] - z... u- \s*[-—|]\s*荣盛石化股份有限公司.*z.w-createtime-datez.w-createtime-timez .w-detailzsrc="(?!https?://)(/[^"]+)"z
src="https://z\1"z src="(?!https?://)([^/][^"]*)"z style="[^"]*"z\s+height="[^"]*"z
width="100%"z- style="width:100%;height:auto;display:block"zz
z<[^>]+>i, z\s+
) site_namer urlcontentpub_datesummarytagsu ✅ (z chars))r u ❌ < z +u 条a pn => {
const links = document.querySelectorAll('a');
for (const a of links) {
if (a.textContent.trim() === String(pn) && a.href === 'javascript:void(0);') {
a.click();
return true;
}
}
return false;
}u (无下一页)cnrspcu
✅ 完成! 共 u 条u
⏭ 无新数据) print SITE_NAMEplaywright.sync_apir setchromiumlaunchnew_pagegotoLIST_URLtimesleep MAX_PAGESrangeevaluateTHREE_YEARS_AGOaddstriplenr resubquery_selector text_content
inner_htmlDOMAINcloseappend Exceptionstrr ) incrementallimitr all_items seen_urlspwbrowserpage total_pagesstoppgitemspage_newitemr detail_page
page_title final_titler date_eltime_eldtr detail_elhtmlr enext_pnclickeds !/root/gov_crawler/crawl_cnrspc.pycrawlrZ s Bvhgi[6(
34
$%3II Fb++$$d$3 ! (}e D
1
;?+ { B LE*$ ? MM # E$ H K
<9, ABI 4 4 6 < < >rA*+AaSz'7'7'9H
!G $/$>$>{$K $#,#7#7#9D#%66*H][aZbbeJfhl#mD#%66*KRQU#VD#%66*;R#FD#%66*>D#ID#%66*: D D Ft LG>EbffVS' :;G $$%.!, #F|#*5=HSbM2#* )& MHE#g, w7tD S^u 4 DWK
Z
Bxj$"D 9 K q& -- - +,
1
w{ z
MFP H-
"3y>"2$78
"#Y , ! DQ
.d; OF Fs} DR A:Q%A9Q Q%B.Q 1B%Q%RR* R ARQ Q% Q" Q%% R . R
RR RR!__main__r )1z
--incrementalz--limit)r@ rA u ⏱ 耗时: z.1fs)FN)$__doc__sysosr6 r- jsonr r r pathinsertdirnameabspath__file__ crawler_libr r% r, r; nowutcstrftimer2 r/ rZ __name__r5 argvr@ rA enumerateiaintt0r$ rY rt sM 2 2 277??277??8#<= > ( ' ( 8<< - u0EEOOPZ[
U$n zchh-!#M 7M(MK E#((# '1 >acCM 11
&E'
B k/ LR ,A
./ rs