vj d Z ddlZddlZddlZddlZddlZddlmZmZ ej j dej j ej j e ddl
mZ ej ed z
j! d ZdZd d
iZd Zd Zd
Zd Zd ZddZedk( rU ej6 Z e j; de d e j? Z ee jB Z" e#de" d yy)u' crawl_yq_sthjj_gszx.py - 阳泉市生态环境局-公示中心(双公示)
站点: https://sthjj.yq.gov.cn/zmhd/gszx/
CMS: TRS WCM (UTF-8)
列表: li > i + span(YYYY-MM-DD) + a[title=标题](href=./YYYYMM/tYYYYMMDD_xxx.shtml)
分页: countPage=34 → index_N.shtml (首页=index.shtml, 第2页=index_1.shtml, 每页15条)
详情: 标题 + div.view TRS_UEDITOR 正文
修复(2026-08-08): ①TRS_UEDITOR 容器非贪婪(.*?)</div> 首遇</div>截断 → 嵌套div后的附件丢失
②改用平衡div匹配 ③附件href相对路径转绝对URL
N)datetime timedelta)push_to_searchdbiG )daysz%Y-%m-%du% 阳泉市生态环境局-公示中心z
User-Agentz6Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0zhttps://sthjj.yq.gov.cnz zmhd/gszxc | dk( rt dt d}nt dt d| dz
d} t j |dt d }d|_ t
j d |j t j }g }|D ]a \ }}}t
j d
d | j }t | d k r;t dt d| }|j |||d
c |S # g cY S xY w)N /z/index_z.shtml Ftimeoutheadersverifyutf-8zc<li><i></i><span>(\d{4}-\d{2}-\d{2})</span><a href="\./(\d{6}/t\d+_\d+\.shtml)"[^>]*title="([^"]*)"\s+ )urlpub_datetitle)BASE LIST_PATHrequestsgetHEADERSencodingrefindalltextDOTALLsubstriplenappend)page_idxr ritemsresultspdhrefr s (/root/gov_crawler/crawl_yq_sthjj_gszx.py
fetch_listr- s 1}a {!$a {'(1* V< LLgwu M
JJn
E G ED%vr5)//1 u:> a {!D6*sUCD E N s $C+ +C1c d t j || }|sy| j d|j }|dk( ry|dz }d}t j d| |d D ]J }|j d j
d r|dz
})|dz }|dk( s4| |||j z c S | |d S ) u 平衡 div 匹配: 从匹配到的 <div...> 的 '>' 之后, 到对应 </div>, 返回内部 HTML。
修复非贪婪 (.*?)</div> 在正文含嵌套 div(表格/附件区)时截断的问题r >r z<div[\s>]|</div>Nr z<div)r searchfindstartfinditergroup
startswith)html open_tag_remgtidepthmms r, extract_balanced_divr> / s +t$A
3 "B Rx
QA
Ekk-tABx8 .
88A; ! !& ) QJE QJEzAa"((*n--
. 8O c V t d D ]( } t j | dt d }d|_ n yd}t j d|j t
j }|r[t j d d|j d
j }t j d | d j }d}t |j d
}|j r|j }|j s6t |j d }|j r|j }t j dd|t
j t
j z }| j dd
d t j dfd| }t j dd | }t j dfd| }t j dd | }t j dd| }||j fS # d }Y xY w)N r
Fr
r )r r r z <title>\s*([^<]+?)\s*r r u [-–—|_]r z1]*class="[^"]*view TRS_UEDITOR[^"]*"[^>]*>z-]*class="[^"]*article-zoom[^"]*"[^>]*>z)flagsr zhref="\./([^"]+)"c 4 d d| j d dS )Nhref="r r "r5 r=
detail_dirs r, z fetch_detail..^ s zl!BHHUVK=XY6Z r? z!href="(?!https?://|//|#)(/[^"]+)"c 8 dt | j d dS )NrD r rE r r5 r= s r, rI z fetch_detail.._ s" tfUWU]U]^_U`TaabFc r? zsrc="\./([^"]+)"c 4 d d| j d dS )Nsrc="r r rE rF rG s r, rI z fetch_detail..` s uZL"((ST+ VW5X r? z src="(?!https?://|//)(/[^"]+)"c 8 dt | j d dS )NrN r rE rK rL s r, rI z fetch_detail..a s" U4&QSQYQYZ[Q\P]]^C_ r? z (\s*()?\s*
)+$)ranger r r r r r1 r r! r" r5 r# splitr> Irsplit)r _attemptr' r r9 contentcrH s @r, fetch_detailrW C s !H S'75QA AJ y E
2AFFBIIFAvr1771:.446 /288:G QVV%YZAwwy'') ==? )Y Z 779ggiGff+R BDD@PQGC#A&Jff)+Z\cdGff9;celmGff(*XZabGff68_ahiGff5r7CG '--/ !!5 As $H!!H(r c x g }t }t | D ] }t |dz }|st d|dz d nt d|dz dt | d |D ] }|d r
|d t
k r|d |v r |j
|d t |d \ }}|s|d }|j sZ|j ||d |d |d t |d d
t d t | d |D cg c] }|d j s| } }| r t | d
t | S c c}w )Nr z Page z: 0 items (stop)z: z itemsr r r r )r r
source_urlr site_namerU summaryz Total: rU
yq_sthjj_gszx) setrP r- printr$ CUTOFFaddrW r# r% SITE_NAMEr )
max_pagesrecordsseenpgr( itr rU r' valids
r, runrh e s[ G 5DI 26" GBqD6!12 3
1vRE
|623 B*~"Z.6"9%yD HHRY )"U) 4NE77 ==? NN r%y5 zN &7r
,
Ic'l^6
*+ 811Y<#5#5#7Q 8E 8 0 u:
9s ?D7D7__main__z--pages)typedefault)rb zDone: z records)r )$__doc__sysosr r argparser r pathinsertdirnameabspath__file__ crawler_libr nowstrftimer_ ra r r r r- r> rW rh __name__ArgumentParserap add_argumentint
parse_argsargspagescntr^ r? r, r s ' & & ( 277??277??8#<= > (
(,,.9%0
0 : :: F 3 Q
R .( "D > z "BOOICO3
==?D
#C F3%x
! r?