$hyj+ H d Z ddlZddlZddlZddlZddlZddlZddlm Z ej j dej j ej j e ddlmZ dZedz ZdZdZd Zd
Zd Zd Zd
Zd Zd Zd Z e dk( r e yy)u
惠安县人民政府 - 通知公告 (tzgg) 爬虫
列表: https://www.huian.gov.cn/zwgk/xwzx/gggs/tzgg/
avalon.js 前端渲染 (terton.dynamicAndStaticData), 静态 HTML 含 JS 数组(每页15条)
翻页: 点页码按钮 (ms-controller=list_pagebar), recordCount=1421 约95页
详情: /zwgk/xwzx/gggs/tzgg/YYYYMM/tYYYYMMDD_ID.htm
标题: 页面 或 h1
正文: div.TRS_UEDITOR 或类似容器
用法: python3 crawl_huian_gsgg.py [--pages=N] [--limit=N]
N)urljoin)push_to_searchdbzhttps://www.huian.gov.cnz/zwgk/xwzx/gggs/tzgg/u" 惠安县人民政府-通知公告u 通知公告_ zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c d\ } }t j dd D ]` }|j d r t |j d d } 2|j d sDt |j d d }b | |fS )N)r r z--pages==z--limit=)sysargv
startswithintsplit)pageslimitas %/root/gov_crawler/crawl_huian_gsgg.py
parse_argsr st LE5
XXab\ ) <<
# Q(E
\\*
% Q(E )
%< c t j | xs d } t j dd| } t j dd| } t j dd| } | j S )N z [\u200b\u200e\u200f\ufeff\xa0]\s+ u ^[•·\-—]\s*)html_modunescaperesubstrip)ts r clean_titler & sZ !'r"A
0"a8A
vsA A
"B*A 779r c | j d }|D ]K }|d }|j d r
t |dd z }n!|j d st t | }||d< M |S )u6 从渲染后页面提取 doctitle/docpuburl/docreltimeu\ () => {
const out = [];
// avalon 渲染后列表: 通常 ul li a
const items = document.querySelectorAll('ul li a[href*=".htm"], .list_base a[href*=".htm"]');
const seen = new Set();
for (const a of items) {
const href = a.getAttribute('href') || '';
if (!href.includes('.htm') || seen.has(href)) continue;
seen.add(href);
let date = '';
const li = a.closest('li');
if (li) {
const m = li.textContent.match(/(\d{4})[-\/](\d{1,2})[-\/](\d{1,2})/);
if (m) date = m[1] + '-' + m[2].padStart(2,'0') + '-' + m[3].padStart(2,'0');
}
const title = (a.textContent || '').trim();
if (title.length >= 4) out.push({href: href, title: title, date: date});
}
return out;
}href./ Nhttp)evaluater LIST_URLr )pageitemsitr s r
extract_itemsr) . ss MM
E* &z ??4 d12h&D (8T*D6
Lr c & | j d| S )u 点击页码按钮u) (pg) => {
// avalon pagebar 页码链接
const links = document.querySelectorAll('.page_base a, .pagebar a, .pgStyle a, a[href="javascript:void(0)"], a[href="#"]');
for (const a of links) {
const t = (a.textContent || '').trim();
if (t === String(pg)) { a.click(); return true; }
}
// 下一页
const next = document.querySelector('.page_base .next a, .pagebar .next a, a[title="下一页"], .nextpage');
if (next) { next.click(); return true; }
return false;
})r$ )r& pgs r
click_pager, O s ==
r c | j dd | j d | j }d}t j d| }|rt |j
d }|s2t j d| }|rt |j
d }|sDt j d | }|r,t |j
d j d
d }d}t j d | }|r"|j
d j d
d }|s_t j d| }|rGd|j
d t |j
d t |j
d fz }d}dD ] }t j dt j | z dz | }|s6|j }|j d| dz } d}
| } | t | k rE|| | dz dk( r |
dz
}
| dz
} n || | dz dk( r|
dz }
| dz
} |
d k( rn| dz
} | t | k rE|| | dz
} t j dd| } t j d d| } t j d d| } t j d fd | } t j d fd!| } t j d"d| } t j d#d| } | j }t | d$k\ s n |rt | d$k rd}|||fS # t $ r }
t! d% d&|
d'( Y d
}
~
y)d
}
~
ww xY w)*u# 详情页: h1/标题 + 正文容器commit0u
wait_untiltimeouti r z+<meta name="ArticleTitle" content="([^"]*)"r z<h1[^>]*>([^<]+)</h1>z<title>([^<]+)_r z&]*\bz\b[^"\'>]*["\']?> zz
z
]*>zzhref="([^"]*)"c d| j d j d s t | j d z S | j d z S )Nz href="%s"r )r# #
javascriptgroupr r murls r z fetch_detail.. sq kefelelmneoezez |W fXWSRSRYRYZ[R\E] 7i ^_ ^e ^e fg ^h 7i r z
src="([^"]*)"c d| j d j d s t | j d z S | j d z S )Nzsrc="%s"r )r# zdata:rB rC rE s r rH z fetch_detail.. sq Zcdcjcjklcmcxcx zY dZ73PQPWPWXYPZC[ 6k `a `g `g hi `j 6k r z\sstyle="[^"]*"zM]*>||]*>||]*>||]*>| z [ERR] detail : Tflush)r r r )gotowait_for_timeoutr9 r searchr rD r
r r escapestartfindlenr Exceptionprint)r& rG htmltitlerF pub_datebodycidirR depthjrawes ` r fetch_detailra _ s3 > #(E : d#||~ IIDd K
+E 2D9A#AGGAJ/ 3T:A#AGGAJ$4$4S$9!$<= II? F wwqz'')#2.H 8$?A)QWWQZQWWQZ #aggVWj/,ZZ A C @299S>QTggimnA AIIc1%)EEAc$i-!A#;&(QJEFA!AaC[H,QJEFA z FA c$i- uQqS/C&&4b#>C&&)2s3C&&2B s4y2~Dh$$
!#b,D9s% H5L" 9C
L" L" " M+MMc v
t \ } }| st } t dt d| d ddlm} g }t
} | 5 }|j j dddg }|j t d
}|j }|j d d
|j t dd |j d t! d| dz D ]v } | dkD rld}
t! d D ]5 } t# || }
|
r n$|j% d |j d 7 |
st d| dd n|j d t) | }
d}|
D ]r }|d }|j+ d r
t |d d z }n!|j+ d st- t. | }||v rL|j1 | |j3 | |dz
}t t d!| d"t5 |
d#| d$t5 | d |j% d% }|s| dkD rt d&| d n,t5 | d'k\ s^t d(t5 | d n |j7 d d d t d)t5 | d*d |rt5 | |kD r|d | } | 5 }|j j dddg }|j t d
}|j }|j d d+ g }t9 | D ]j \ }}t; ||d \ }}}|xs |d, }|xs |j= d-d. }|rt? j@ d/d0| n|}t? j@ d1d0| jC d d2 }g }|rjt? jD d3|t> jF D ]B }|j3 |jI d jC |jI d d4 D |j3 t |d |d ||||tJ |rtM jN |d5 nd.d6 |dz d7z dk( r t d8|dz d9t5 | d tQ jR d: m |j7 d d d t d;t5 d<d tU |t = tW d> |D } t d?t5 | d@| d y # t& $ r/} t d| d| d |j d Y d } ~ ,d } ~ ww xY w# 1 sw Y xY w# 1 sw Y xY w)ANz
[Huian] SITE=z max_pages=TrL r )sync_playwrightz---disable-blink-features=AutomationControlledz --no-sandbox)headlessargszzh-CN)
user_agentlocalez**/*c r | j j dv r| j S | j S N)imagemediafont
stylesheetrequest
resource_typeabort continue_routes r rH zmain.. C %--:U:U ZB ;B HM HW HW HY r r. r/ r0 i r Fr5 z.window.scrollTo(0, document.body.scrollHeight)i z [WARN] page z click err i z [STOP] page z no click targeti r r! r" r# z Page rK u 条(新u ) 累计u () => {
const n = document.querySelector('.page_base .next a, .pagebar .next a, a[title="下一页"]');
if (!n) return false;
return !(n.className.includes('disabled') || n.style.display === 'none');
}z
[END] page i z [CUTOFF] reached u 列表完成: u 条, 抓详情...c r | j j dv r| j S | j S ri rn rs s r rH zmain.. ru r rX dater z<[^>]+>r r i, zQ]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar|ofd|wps))"[^>]*>([^<]*))namerG ) ensure_ascii) site_name
source_urlrG rX rY r9 summarycategory attachmentsr4 z detail /g333333?z
pushing z items to searchdb...) batch_labelc 3 h K | ]* }|j d xs dj r'd , yw)r9 r r N)getr ).0rs r zmain.. s) Ka155+;+Ar*H*H*JKs (22z DONE. pushed=u 空正文=),r MAX_PAGESrV SITE_NAMEplaywright.sync_apirc setchromiumlaunch new_contextUAnew_pagert rN r% rO ranger, r$ rU r) r r BASE_URLaddappendrT close enumeratera r r r r finditerIrD CATEGORYjsondumpstimesleepr sum) max_pagesr rc all_itemsseenpbrowserctxr& r+ clickedattemptr` r' new_countr( r has_nextresultsr\ d_titled_daterZ rX rY r| attsrF emptys r mainr s3 !|Iu M)K {
;4H3I 5D 7a**##T9hjx8y#z!!R!@||~
6 Y Z (x ? d#9q=) , BAv $Qx 4G4",T2"6" !
&VW --c2
4 N2$.>?tL%%d+!$'EI
&z??4(#d12h.D 0"8T2D4< $Q
GB4r#e* WYKxIGWX`d e }} & H
Q
bT*$79~%+C N+;**<
=TJ Y %' fu% a**##T9hjx8y#z!!R!@||~
6 Y Z y) EAr$0r&z$B !GVT*r'{E 3!3H7;bffZd3GffVS'288:4C@GD%y{ BD BF BF G QAKK)9)9);AGGAJ OPQ NN &bjF H"GKtzz$U CQS
A|q !A#aI'78E JJsO% &
36
Js7|n$9
:$GW)4 K7K KE OCL>UG
r s 1 0 0 277??277??8#<= > ( % - - 0 v B
@FeJP zF r