sj& d Z ddlZddlZddlZddlZddlZddlZddlZdZdZ d
dZ
d Z d Z d Z
ed k( r ej e
yy) u yanshou100.com 水土保持公示网 - 增量抓取 v3 (修复表格/段落/附件格式)
v3 修复 (2026-08-19 用户反馈):
1. 正文保留 段落 +
表格 HTML (不再剥平)
- clean_html 用 bs4: 去 script/style/iframe/object, div/span/font/center unwrap,
a 转绝对 URL, img 转查看链接, 表格结构完整保留
2. 附件转正文内嵌 附件名
段落 (多附件独立成段)
- attachments 字段同时保留 [{fileName,fileUrl}] 数组
3. 导入不走 import_jsonl_v2 的 strip_html 剥平逻辑 → 直接写 gov_raw (保留 HTML)
- push_to_searchdb 保留 content 原样
用法: python3 crawl_yanshou100_inc.py --pages 5 --type yanshou|qita --out /tmp/x.jsonl [--existing /tmp/ids.txt]
NzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36zhttps://www.yanshou100.comc K d|rdnd dt j | dt j | dt d| d } | j | d { S 7 # t $ r}d t | d d
d cY d }~S d }~ww xY ww) Nz
async () => {
const opts = {
method: 'POST',
headers: {'X-Requested-With': 'XMLHttpRequest', 'Accept': 'application/json, text/javascript, */*; q=0.01'}
};
if (truefalsez) {
opts.headers['Content-Type'] = 'application/x-www-form-urlencoded';
opts.body = new URLSearchParams(z).toString();
} else {
opts.headers['Content-Type'] = 'application/json';
opts.body = JSON.stringify(z,);
}
const r = await fetch('z' + 'z', opts);
const ct = r.headers.get('content-type') || '';
if (ct.includes('application/json')) {
return await r.json();
}
return {'code': 'notjson', 'msg': (await r.text()).slice(0, 120)};
}
errd )codemsg)jsondumpsBASEevaluate Exceptionstr)pageurlpayloadformjses %/root/gov_crawler/crawl_yanshou100.pyapi_postr s
V ) *--1ZZ-@,A B( )-
7(;'< = $vU3% 0
B*4]]2&&&& 4 c!fTcl334sH ABA A A B A B(A>8B9B>BBc ddl m} ddlm} || d } |g d D ] }|j |j d D ] }|j
d xs |j
d xs d }|rL|j d
r|n |t | }|j d | }d
|_
|j | y|j |j ddi D ] }|d= |j g d D ] }|j |j d D ]p }|j
dd } | r5| j d s$| j d
r| n |t | |d< L| sO| j d sa|j r |j d D ]I }
|
j d r|
j d r(|
j d r:|
j K t | S )uS 白名单清洗: 保留 /
//![]()
, unwrap 装饰标签, 链接绝对化r )
BeautifulSoup)urljoinz html.parser)scriptstyleiframeobjectimgsrczdata-src httpa)hrefu 查看图片hiddenT)attrs) divspanfontcenterbstrongemiuslabelr$
javascriptp)striptable)bs4r urllib.parser decomposefind_allget
startswithr new_tagstring replace_withunwrapget_textfindr ) htmlr r souptr r abs_srcr# r$ r3 s r
clean_htmlrF 1 s !$ } -D
9
:
}}U# ggen9
39r ^^F 3cs9KG Sw /A%AH Q MMO ]](D!1]
2
hK
]]i
j
]]3
uuVR 5 $ 7WT4=PAfI
dool3
KKM
]]3
zzz%affWoaffUm
KKM t9 c T K t d| }t d D ]> } | j |dd d { | j d d { | j d { t
j d }|r@|j d j r!|j d j d v rt
j d
}|r4t
j d d |j d j nd
}|j dd
j }|r n&t j d|dz z d { A s|dxx dz
cc< y t
j d }|r"|j d j d d nd
} fd}
|
d } d
} | j d d { } | s+t
j d }
|
r|
j d nd
} | r t | nd
} g }t
j d D ]D }|j d |j d }}|j# |t d| d| d F t% }g }|D ]/ }|d |vs |j' |d |j# | 1 d
}|D ] }|d |d d |d d z
} |r| d!z |z j } |||| | | ||d"}|d#xx dz
cc< |S 7 # t $ r Y w xY w7 7 7 7 I# t $ r Y Sw xY ww)$Nz/item_detail.html?id= domcontentloaded0u
wait_untiltimeouti z#id="itemName1"[^>]*>\s*([^<]+?)\s*< ) z z#id="itemName2"[^>]*>\s*([^<]+?)\s*<\s+ r! rP g @ detail_failu3 id="createDate"[^>]*>\s*时间[：:]\s*([^<]+?)\s*<
c t j d| z dz }|r4t j dd|j d j S dS )Nzid="z"[^>]*>\s*([^<]+?)\s*.fieldm sH IIgm&?? C:;rvvfc1771:.446CCrG companyNamea_
(() => {
const info = document.getElementById('itemInfo');
if (!info) return '';
let tbl = info.closest('table');
if (!tbl) return '';
const attTr = tbl.querySelector('tr#tr_attachment');
if (attTr) attTr.remove();
return tbl.outerHTML;
})()
z#id="itemInfo"[^>]*>([\s\S]*?)z!jumpToFile\('([^']+)','([^']+)'\) z/item/showFile?fileId=z
&fileName=)fileNamefileUrlra z r` z
)titler page_url publish_datecontentauthor site_name attachmentsok)r rangegotor wait_for_timeoutrf rV rW rY r4 rX replaceasynciosleepr
rF finditerappendsetadd)r item_idstatsrh r attemptrD rc dmpubr] companybodyimattsr[ fafnseenatts2r# att_pitemr\ s @r fetch_detail_pager W sC
F'y
1C8 1 ))C,>)N N N ##D))),,. II.>.@D^.^ @!DA;<vsAGGAJ/557"
h+113 mmC7Q; /000 1
m ! I1 MB&("((1+
cr
"bCD M"G D
]]
$
YY=q
A rxx{b #:d D D
[[=q
A dQWWQZB 4&8NrdR\]_\`0abcd 5D E
Y d |
jY d { d d d d { |jY t d?|d< d=|d> d@|d0 d y # 1 sw Y xY w7 f7 E7 )7 7 7 7 7 c c}w 7 ,7 7 s7 e# 1 d { 7 sw Y uxY w# |jY w xY ww)ANz--pages )typedefaultz--typeyanshouzyanshou/jiance/fangan/qita)r helpz--outz/tmp/ys100_inc.jsonl)r z
--existingr! u 已存在 itemId 集合文件 yanshou100_zutf-8)encodingu 已有 itemId: T)flushr )rj rS
skip_existingw)async_playwright)headless)
user_agentignore_https_errors/rJ rK rL i rO )
r limit
publicTypeitemNamestartendcityareaparTypesunTypez /item/searchr )0200u 页u : API失败 dataitemIdz: u
条, 累计 z, total=countu
列表共 u 个 itemIdr u 需抓 u (跳过已有 )F) ensure_asciirb g ?g?rT u 详情 z: ok=rj z fail=rS u 完成: ok=z skip_existing=)-argparseArgumentParser add_argumentint
parse_argsr args_typers existingospathexistsopenr4 rt printlenoutplaywright.async_apir chromiumlaunch new_contextUAnew_pagerl r rm rk pagesr r r: rr ro rp enumerater writer
r r randomuniformclose)apargsrh existing_idsflinerv out_fpr r3 browserctxr all_idspgr ditemsitr. new_idsiidr s r mainr s "BOOICO3OOHi6ROSOOG%;O<OOL"3ROS
==?D I )I5L }}
6
$--'
2 +a
+zz| $$T*
+ +
OC -.
/t<Q ;E
$((C'
2F(9#% $ " $ " JJ - -t - <dKf
+ 5Bvvh'w6 r(|45 B4r#e* ]3w<.QRQVQVW^Q_P`aimn mmA&&&
'
Js7|nK8 E")HQSV<-GqHGH%(\CL%@E/ " GCL> 9O8PPQRZ^ _#G,
t3 .tS% KKLLDu!E!LMLLN mmFNN3$<===ER<1$GAaC5#g, uU4[MPUVcPdOefnrs
t --/ ! !I$ " $ "L Kd }F5 +?*@PUVePfOg
hptue + +$ "<T' U - B ' I
L >
"I$ " $ " $ " $ "L s\ C)T +SS?T T
)S*T
- S5
S S5,S-S5S"S5'S(S5 S A
S5 S! A0S5=A(S5%S$&%S5 S' S'"A S5 S,A+S5,S/-S5=A S5=S1>S5 T
S3T
0T S T T
S5S5S5S5 S5!S5$S5'S5/S51S53T
5T;S> <TT
T T __main__)F)__doc__r ro r
rV sysr r r r r rF r r __name__run rG r r sZ 4 3 3 3v#48#LOd?vB zGKK rG