}jP D d Z ddlZddlZddlZddlZddlZddlZddlZej j dej j ej j e ddl
mZ dZdZd dZd Zd Zd Zed
k( r ej, e yy) u 本地 Mac 抓取 yanshou100 (绕过服务器 IP 限流) → JSONL
用法: python3 crawl_yanshou100_local.py --pages 5 --type yanshou|qita --out /tmp/x.jsonl
N)push_to_searchdbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36zhttps://www.yanshou100.comc K d|rdnd dt j | dt j | dt d| d } | j | d { S 7 # t $ r}d t | d d
d cY d }~S d }~ww xY ww) Nz
async () => {
const opts = {
method: 'POST',
headers: {'X-Requested-With': 'XMLHttpRequest', 'Accept': 'application/json, text/javascript, */*; q=0.01'}
};
if (truefalsez) {
opts.headers['Content-Type'] = 'application/x-www-form-urlencoded';
opts.body = new URLSearchParams(z).toString();
} else {
opts.headers['Content-Type'] = 'application/json';
opts.body = JSON.stringify(z,);
}
const r = await fetch('z' + 'z', opts);
const ct = r.headers.get('content-type') || '';
if (ct.includes('application/json')) {
return await r.json();
}
return {'code': 'notjson', 'msg': (await r.text()).slice(0, 120)};
}
errd )codemsg)jsondumpsBASEevaluate Exceptionstr)pageurlpayloadformjses +/root/gov_crawler/crawl_yanshou100_local.pyapi_postr
s
V ) *--1ZZ-@,A B( )-
7(;'< = $vU3% 0
B*4]]2&&&& 4 c!fTcl334sH ABA A A B A B(A>8B9B>BBc ddl m} ddlm} || d } |g d D ] }|j |j d D ] }|j
d xs |j
d xs d }|rL|j d
r|n |t | }|j d | }d
|_
|j | y|j |j ddi D ] }|d= |j g d D ] }|j |j d D ]p }|j
dd } | r5| j d s$| j d
r| n |t | |d< L| sO| j d sa|j r |j d D ]I }
|
j d r|
j d r(|
j d r:|
j K t | S )uM 白名单清洗: 保留 /
/, unwrap 装饰标签, 链接绝对化r )
BeautifulSoup)urljoinz html.parser)scriptstyleiframeobjectimgsrczdata-src httpa)hrefu 查看图片hiddenT)attrs) divspanfontcenterbstrongemiuslabelr%
javascriptp)striptable)bs4r urllib.parser decomposefind_allget
startswithr
new_tagstring replace_withunwrapget_textfindr ) htmlr r souptr r! abs_srcr$ r% r4 s r
clean_htmlrG ( s !$ } -D
9
:
}}U# ggen9
39r ^^F 3cs9KG Sw /A%AH Q MMO ]](D!1]
2
hK
]]i
j
]]3
uuVR 5 $ 7WT4=PAfI
dool3
KKM ]]3
zzz%affWoaffUm
KKM t9 c T K t d| }t d D ]> } | j |dd d { | j d d { | j d { t
j d }|r@|j d j r!|j d j d v rt
j d
}|r4t
j d d |j d j nd
}|j dd
j }|r n&t j d|dz z d { A s|dxx dz
cc< y t
j d }|r"|j d j d d nd
} fd}
|
d } d
} | j d d { } | s+t
j d }
|
r|
j d nd
} | r t | nd
} g }t
j d D ]D }|j d |j d }}|j# |t d| d| d F t% }g }|D ]/ }|d |vs |j' |d |j# | 1 d
}|D ] }|d |d d |d d z
} |r| d!z |z j } |||| | | ||d"}|d#xx dz
cc< |S 7 # t $ r Y w xY w7 7 7 7 I# t $ r Y Sw xY ww)$Nz/item_detail.html?id= domcontentloaded0u
wait_untiltimeouti z#id="itemName1"[^>]*>\s*([^<]+?)\s*< ) z z#id="itemName2"[^>]*>\s*([^<]+?)\s*<\s+ r" rQ g @ detail_failu3 id="createDate"[^>]*>\s*时间[：:]\s*([^<]+?)\s*<
c t j d| z dz }|r4t j dd|j d j S dS )Nzid="z"[^>]*>\s*([^<]+?)\s*.field^ sH IIgm&?? C:;rvvfc1771:.446CCrH companyNamea_
(() => {
const info = document.getElementById('itemInfo');
if (!info) return '';
let tbl = info.closest('table');
if (!tbl) return '';
const attTr = tbl.querySelector('tr#tr_attachment');
if (attTr) attTr.remove();
return tbl.outerHTML;
})()
z#id="itemInfo"[^>]*>([\s\S]*?)z!jumpToFile\('([^']+)','([^']+)'\) z/item/showFile?fileId=z
&fileName=)fileNamefileUrlrb z ra z
)titler page_url publish_datecontentauthor site_name attachmentsok)r
rangegotor wait_for_timeoutrg rW rX rZ r5 rY replaceasynciosleepr rG finditerappendsetadd)r item_idstatsri r attemptrE rd dmpubr^ companybodyimattsr\ fafnseenatts2r$ att_pitemr] s @r fetch_detail_pager H s?
F'y
1C8 1 ))C,>)N N N ##D))),,. II.>.@D^.^ @!DA;<vsAGGAJ/557"
h+113 mmC7Q; /000 1
m ! I1 MB&("((1+
cr
"bCD M"G D
]]
$
YY=q
A rxx{b #:d D
D
[[=q
A dQWWQZB 4&8NrdR\]_\`0abcd 5D E
Y
==?D )IQ 'E
$((C'
2F"9#% " " JJ - -t - <dKf
+ 5Bvvh'w6 r(|45 B4r#e* ]3w<.QRQVQVW^Q_P`aimn mmA&&&
'
DW k2$ ?#G,
t3 .tS% KKLLDu!E!LMLLN mmFNN3$<===ER<1$GAaC5#g, uU4[MPUVcPdOefnrs
t --/ ! != " "@ Kd }F5 +?*@
ANC "<T' U - B ' L >
"= " " " "@ s, BO*O #N $O ' O N" O &N%'O >N(?"O !N+"O :N.;A
O N1A0O 7A(O N4 AO #N7$A+O N:O A O N<!O % O 0N>1O 5*O* O "O %O (O +O .O 1O 4O 7O :O <O >O OO OO O''O*__main__)F)__doc__r rp r rW sysr ospathinsertdirnameabspath__file__ crawler_libr r r
r rG r r __name__run rH r r s 4 3 3 3 277??277??8#<= > (v#46 @JX,O\ zGKK rH