vj d Z ddlZddlZddlZddlZddlZddlmZmZ ddlm Z ej j dej j ej j e ddlmZ ej" ed z
j% d Zd d
iZd Zd Zd
ZdZdddddddddZd Zd Zd Z d$dZ e d k( r ej> Z e jC d d e"ejG e jC d e$d e jK Z& e e&jN e&jP ! Z) e*d"e) d# yy)%u crawl_rizhao_sthjj.py - 日照市生态环境局 (2栏目: 行政许可/环境管理)
站点: http://sthjj.rizhao.gov.cn/col/col179934/index.html (行政许可)
http://sthjj.rizhao.gov.cn/col/col353302/index.html (环境管理)
CMS: 大汉版通 jcms1 + jpage 动态分页
列表: table > tr > td > a.bt_link[title=标题](href=/art/Y/M/D/art_{col}_ID.html) + b.dateqi(YYYY-MM-DD)
分页API: POST /module/web/jpage/dataproxy.jsp
startrecord/endrecord/perpage/unitid/webid/path/webname/col/columnid/sourceContentType/permissiontype
(返回XML: ...]]>)
栏目(--col):
179934 = 行政许可 (totalRecord=466)
353302 = 环境管理 (totalRecord=626)
详情: 标题 + div#zoom 正文
修复(2026-08-08): ①zoom 容器非贪婪(.*?)</div> 在表格嵌套div处截断 → 清单/名录类正文表格全丢
②改用平衡div匹配 ③href/src 相对路径转绝对(含 /module/download/downfile.jsp 附件)
N)datetime timedelta)quote)push_to_searchdbiG )daysz%Y-%m-%dz
User-Agentz6Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0zhttp://sthjj.rizhao.gov.cn157758780 u% 日照市生态环境局-行政许可179934i )namecolumnidtotalu% 日照市生态环境局-环境管理353302ir )r r c . t | d }|dz
t z dz }|t z }t dz }||t t t t dz t d d|ddd } t
j ||dt d
}d |_ t j d |j t j }g } |D ]W }
t j d
|
t j } | s\t j d|
t j } | sS| j d | j d | j d }}
} nF| j d | j d xs | j d | j d }}
} t j dd|
j# }t j dd| j# }t% | dk r'| j' d r| nt | z }| j) |||d Z | S # g cY S xY w)Nr
z /module/web/jpage/dataproxy.jsp/u 日照市生态环境局r ) startrecord endrecordperpageunitidwebidpathwebnamecolr
sourceContentTypepermissiontype) r
F)datatimeoutheadersverifyutf-8z&<record><!\[CDATA\[(.*?)\]\]></record>zS<a href='([^']+)'[^>]*title='([^']*)'[^>]*>(.*?)</a><b[^>]*>(\d{4}-\d{2}-\d{2})</b>z?<a href='([^']+)'[^>]*>(.*?)</a><b[^>]*>(\d{4}-\d{2}-\d{2})</b> z<[^>]+> \s+http)urlpub_datetitle)COLUMNSPERPAGEBASEUNITIDWEBIDr requestspostHEADERSencodingrefindalltextDOTALLsearchgroupsubstriplen
startswithappend)r page_nocolidstartendr) r rrecordsresultsrecmhref
title_textpdr+ s '/root/gov_crawler/crawl_rizhao_sthjj.py
fetch_listrM % s CL $E
q[G #a 'E
G
C
2
2C 375s
u-G'He!q D MM#D'7SX Y
jjBAFFBIIVGG E IIlnqsus|s| } \^aceclclmA#$771:qwwqz1771:b*D#$771:qwwqz/GQWWQZQRb*Dz2z288:vr5)//1 u:> oof-d4$;sUCD E N' s %H Hc d t j || }|sy| j d|j }|dk( ry|dz }d}t j d| |d D ]J }|j d j
d r|dz
})|dz }|dk( s4| |||j z c S | |d S ) u 平衡 div 匹配: 从匹配到的 <div...> 的 '>' 之后, 到对应 </div>, 返回内部 HTML。
修复非贪婪 (.*?)</div> 在正文含嵌套 div(表格/附件区)时截断的问题r& >r z<div[\s>]|</div>Nr z<div)r5 r9 findrB finditerr: r> )html open_tag_rerH gtidepthmms rL extract_balanced_divrY H s +t$A
3 "B Rx
QA
Ekk-tABx8 .
88A; ! !& ) QJE QJEzAa"((*n--
. 8O c t d D ]( } t j | dt d }d|_ n yd}t j d|j t
j }|r[t j d d|j d
j }t j d | d j }d}t |j d
}|j r|j }|j s6t |j d }|j r|j }t j dd|t
j t
j z }t j dd | }t j dd | }t j dd| }||j fS # d }Y xY w)Nr# )r F)r r r! r" )r& r& r& z <title>\s*([^<]+?)\s*r' r u [-–—|_]r zid="zoom"[^>]*>z(]*class="[^"]*wz_zoom[^"]*"[^>]*>z)flagsz!href="(?!https?://|//|#)(/[^"]+)"c 8 dt | j d dS )Nzhref="r "r. r: rX s rL z fetch_detail..u s" tfUWU]U]^_U`TaabFc rZ z src="(?!https?://|//)(/[^"]+)"c 8 dt | j d dS )Nzsrc="r r_ r` ra s rL rb z fetch_detail..v s" U4&QSQYQYZ[Q\P]]^C_ rZ z (\s*()?\s*
)+$)ranger1 getr3 r4 r5 r9 r7 r8 r; r: r< splitrY I)r) _attemptrD r+ rH contentcs rL fetch_detailrk \ sz !H S'75QA AJ y E
2AFFBIIFAvr1771:.446 /288:G QVV%78Awwy'') ==? )T U 779ggiGff+R BDD@PQGff9;celmGff68_ahiGff5r7CG '--/ !!- As $GG r c g }t }t | D ] }t | |dz }|st d|dz d nt d|dz dt | d |D ] }|d r
|d t
k r|d |v r |j
|d t |d \ }}|s|d }|j sZ|j ||d |d |d t | d |d
d t d t | d |D cg c] } | d
j s| }
} |
rt |
d| t |
S c c} w )Nr z Page z: 0 items (stop)z: z itemsr* r) r+ r r& )r+ r)
source_urlr* site_nameri summaryz Total: ri
rizhao_sthjj_) setrd rM printr= CUTOFFaddrk r< r? r, r ) r max_pagesrE seenpgitemsitr+ ri rD valids rL runr{ z sn G 5DI 3Q' GBqD6!12 3
1vRE
|623 B*~"Z.6"9%yD HHRY )"U) 4NE77 ==? NN r%y5 zN $S\& 1gRT
,
Ic'l^6
*+ 811Y<#5#5#7Q 8E 8 -u 56 u:
9s E E__main__z--col)defaultchoicesz--pages)typer} )ru zDone: z records)r )+__doc__sysosr1 r5 argparser r urllib.parser r insertdirnameabspath__file__ crawler_libr nowstrftimers r3 r. r0 r/ r- r, rM rY rk r{ __name__ArgumentParserap add_argumentlistkeysint
parse_argsargsr pagescntrr rZ rL r s= ' & & ( 277??277??8#<= > (
(,,.9%0
0 : :: F Q
R#
?H_bc>H_bc
!F( "< > z "BOOGXtGLLN7KOLOOICO3
==?D
dhh$**
-C F3%x
!
rZ