yj) R d Z ddlZddlZddlZddlZddlZddlZddlm Z ej j dej j ej j e ddlmZ dZedz ZdZdZd Zd
Zd Zd Zd
Zd Zd Zd Z d Z d Z e dk( r e yy)u
惠安县人民政府 - 公示信息 (gsxx) 爬虫
列表: https://www.huian.gov.cn/zwgk/xwzx/gggs/gsxx/
avalon.js 前端渲染 (terton.dynamicAndStaticData), 静态 HTML 含 JS 数组(每页15条)
翻页: 点页码按钮 (ms-controller=list_pagebar)
详情: /zwgk/xwzx/gggs/gsxx/YYYYMM/tYYYYMMDD_ID.htm
标题: meta ArticleTitle / h1
正文: TRS_Editor/TRS_UEDITOR 等容器 (平衡 div)
用法: python3 crawl_huian_gsxx.py [--pages=N]
N)urljoin)push_to_searchdbzhttps://www.huian.gov.cnz/zwgk/xwzx/gggs/gsxx/u" 惠安县人民政府-公示信息u 公示信息 z
2023-08-10zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c d} t j dd D ]M }|j d r t |j d d } 2|j sCt | } O | S )Nr z--pages==)sysargv
startswithintsplitisdigit)pagesas %/root/gov_crawler/crawl_huian_gsxx.py
parse_argsr s[
E
XXab\ <<
# Q(E
YY[FE
L c t j | xs d } t j dd| } t j dd| } t j dd| } | j S )N z [\u200b\u200e\u200f\ufeff\xa0]\s+ u ^[•·\-—]\s*)html_modunescaperesubstrip)ts r clean_titler ' sZ !'r"A
0"a8A
vsA A
"B*A 779r c g }t j d| t j }|s%t j d| t j }|s|S |j d }t j d| D ] }|j d t |j d |j d }}}t
| dk rN|j d rt t |dd }n+|j d
r
t |z }nt t | }|j t s|j |||d |S ) u@ 从内嵌 dynamicAndStaticData JSON 提取列表 (无 JS 执行)z7dynamicAndStaticData\(\s*\{[^}]*?list:\s*\[(.*?)\]\s*\)zlist:\s*\[(.*?)\]\s*\}r zo\{\s*\"docreltime\"\s*:\s*'([^']*)'\s*,\s*\"doctitle\"\s*:\s*'([^']*)'\s*,\s*\"docpuburl\"\s*:\s*'([^']*)'\s*\} ./N/)titlehrefdate) r searchSgroupfinditerr lenr r LIST_URLBASE_URLappend)htmlitemsmblobrecr' r% urls r extract_json_itemsr6 / s E
LdTVTXTXYA II/rtt < 771:D{{ N PT U B 99Q<SYYq\)BCIIaLSe u:> >>$ (CG,C
^^C
S.C(C(C~~h'
uc4@A B Lr c J | j d| }|S # t $ r Y yw xY w)u8 点击页码按钮 (数字按钮, 勿点 下一页 span)a (t) => {
const links = document.querySelectorAll('a[href="#"], .page_base a, .pagination a');
for (const a of links) {
if (a.textContent.trim() === String(t)) { a.click(); return true; }
}
return false;
}F)evaluate Exception)pagepgclickeds r
click_pager= J s8
-- !
s ""c | j d }|D ]\ }|d }|j d rt t |dd }n+|j d r
t |z }nt t | }||d< ^ |S )u4 从渲染后页面提取列表 (avalon 渲染完成)aX () => {
const out = [];
const seen = new Set();
const anchors = document.querySelectorAll('a[href*=".htm"]');
for (const a of anchors) {
const href = a.getAttribute('href') || '';
if (!href.includes('.htm') || seen.has(href)) continue;
const title = (a.textContent || '').trim();
if (title.length < 4) continue;
seen.add(href);
let date = '';
const li = a.closest('li');
const scope = li || a.parentElement;
if (scope) {
const m = scope.textContent.match(/(\d{4})[-\/](\d{1,2})[-\/](\d{1,2})/);
if (m) date = m[1] + '-' + m[2].padStart(2,'0') + '-' + m[3].padStart(2,'0');
}
out.push({href: href, title: title, date: date});
}
return out;
}r& r# r Nr$ )r8 r r r- r. )r: r1 itr& s r
extract_itemsr@ Y s~ MM
E* &z ??4 8T!"X.D
__S
!d?D8T*D6
Lr c @ | j dd | j d | j }d}t j d| }|rt |j
d }|s2t j d| }|rt |j
d }|sDt j d | }|r,t |j
d j d
d }d}t j d | }|r"|j
d j d
d }|s_t j d| }|rGd|j
d t |j
d t |j
d fz }d}dD ] }t j dt j | z dz | }|s6|j }|j d| dz } d}
| } | t | k rE|| | dz dk( r |
dz
}
| dz
} n || | dz dk( r|
dz }
| dz
} |
d k( rn| dz
} | t | k rE|| | dz
} t j dd| } t j d d| } t j d fd | } t j d fd | } t j d!d| } t j d"d| } t j d#d| } t j d$d%| } t j d&d| } t j d'd| } d%| vr d(| vrd%| z d)z } | j }t | d*k\ s n |rt | d*k rd}|||fS # t $ r }
t! d+ d,|
d-. Y d
}
~
y/d
}
~
ww xY w)0u- 详情页: 标题 + 正文容器 (平衡 div)commit0u
wait_untiltimeouti r z+]*>([^<]+)z([^<]+)_r z&]*\bz\b[^"\'>]*["\']?>r" zz
zzhref="([^"]*)"c d| j d j d s t | j d z S | j d z S )Nz href="%s"r )http#
javascriptr* r r m2r5 s r z fetch_detail.. sq {gigogopqgrg}g} Z h[gcSUS[S[\]S^F_ 8m ac ai ai jk al 8m r z
src="([^"]*)"c d| j d j d s t | j d z S | j d z S )Nzsrc="%s"r )rS zdata:rU rV rW s r rY z fetch_detail.. sq jegememnoepe{e{ }\ f]GCQSQYQYZ[Q\D] 7o ce ck ck lm cn 7o r z\sstyle="[^"]*"zM]*>||]*>||]*>||]*>|z^\s*]*>\s*z]*>zz
\s* \s*
z
\s*
z z [ERR] detail : Tflush)r r r )gotowait_for_timeoutrL r r( r r* r
r r escapestartfindr, r r9 print)r: r5 r0 r% r2 pub_datebodycidirb depthjrawes ` r fetch_detailrm | s E #(E : d#||~ IIDd K
+E 2D9A#AGGAJ/ 3T:A#AGGAJ$4$4S$9!$<= II? F wwqz'')#2.H 8$?A)QWWQZQWWQZ #aggVWj/,ZZ A & C @299S>QTggimnA AIIc1%)EEAc$i-!A#;&(QJEFA!AaC[H,QJEFA z FA c$i- uQqS/C&&4b#>C&&2Bzmain.. s E2"V*BvJEs
r )defaultz [CUTOFF] page z
min_date=z < z, stopz
[END] page z click faili u 列表完成: u 条, 抓详情...r% r' z<[^>]+>r r i, zQ]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar|ofd|wps))"[^>]*>([^<]*)r )namer5 F) ensure_ascii) site_name
source_urlr5 r% re rL summarycategory attachmentsrH z detail r$ g333333?z
pushing z items...) batch_labelc 3 h K | ]* }|j d xs dj r'd , yw)rL r r N)getr )rv rs r rw zmain.. s) O!quuY/?/E2.L.L.NAOs (22z DONE. pushed=u 空正文=)(r MAX_PAGESrd SITE_NAMEplaywright.sync_apiro chromiumlaunch new_contextUAnew_pager_ r- r` setranger@ addr/ r, minCUTOFFr= enumeraterm r r r r r+ Ir* CATEGORYjsondumpstimesleepr sumclose) max_pagesro pbrowserctxr: all_itemsseenr; r1 new_countr? min_dateresultsrh d_titled_daterf r% re r} attsr2 emptys r mainr s ) I ykYK
@M3 ;a**##T9hjx8y#z!!R!@||~ (x ? d# u9q=) ,B!$'Et?34@I
f:% F$ $Q
GB4r#e* WYKxIGWX`d eEErRHHv -(JxjF86RZ^_I~!$Q/M"Q${;4H%%d+- ,0 s9~ ..@AN y) EAr$0r&z$B !GVT*r'{E 3!3H7;bffZd3GffVS'288:4C@GD%y{ BD BF BF G QAKK)9)9);AGGAJ OPQ NN &bjF H"GKtzz$U CQS
A|q !A#aI'78E JJsO% (
3w<. 2$?i8OwOO
G ~[@M
w; ; ;s EOH7OO__main__) __doc__osr r r r r0 r urllib.parser pathinsertdirnameabspath__file__ crawler_libr r. r- r r r r r r r r6 r= r@ rm r __name__ru r r r s 1 0 0 277??277??8#<= > ( % - - 0 v6 FGT@F zF r