&j d Z ddlZddlZddlZddlZddlZddlZddlZddlm Z ej dd Z dZ dZ
dZed z Zd
Zd Zed d
ZddddZd Zd Zd Zd Zedk( r ddlZ ej4 e yy)u3 crawl_zz_hb.py v2 - 郑州市生态环境局政务公开 (合并 zz_collect + 详情提取)
自建CMS (JEECMS风格): /?a=dir&d=24&page=N 分页, 529页/10条
详情: /D44Y/4927267.jhtml (静态HTML, 无需playwright, 2026-08-21实测服务器直连200)
正文: div[class*=content], 标题
N)urljoin SEARCH_DBz/root/search.dbu% 郑州市生态环境局-政务公开zcrawl_zz_hb.pyz https://public.zhengzhou.gov.cnz /?a=dir&d=24zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z
2023-08-18z text/html,application/xhtml+xml)z
User-AgentAccept)newskiperrc | dk( rt n
t d| } t j |t dd }|j dk7 ry |j
S # t $ r+}t d| d| t j Y d }~y d }~ww xY w)
N z&page= Fheaderstimeoutverify u 列表页u 错误: file)
LIST_URLrequestsgetHEADERS status_codetext Exceptionprintsysstderr)pageurlres /root/gov_crawler/crawl_zz_hb.pyget_listr" sy ai(zv%>C LLgr% H ==C vv
D6!-CJJ?s ,A A B!B Bc g }t }t j d| D ]= }|j d }||v r|j | |j |dd ? |S )Nz;href="(https://public\.zhengzhou\.gov\.cn/D\w+/\d+\.jhtml)"r
)r title)setrefinditergroupaddappend)htmlpage_urlitemsseenmr s r! extract_list_itemsr1 # sd E 5D
[[WY]
^ 0ggaj $;
S2./ 0 L c dddd}t j d| t j }|r"|j d j |d< dD ] }t j || t j }|s+|j d }t j
dd|t j }t j
d d|t j }t
t j
d
d| j }|d k\ s|j |d < n t j d
| }|r|j d |d< |S )Nr$ )r% contentdatez<title[^>]*>(.*?)r
r% )z2]*class="content-txt[^"]*"[^>]*>(.*?)z3]*class="[^"]*content[^"]*"[^>]*>(.*?)z#]*id="zoom"[^>]*>(.*?)z,]*class="TRS_Editor"[^>]*>(.*?)z.]*class="news_content"[^>]*>(.*?)z]*>.*?)flagsz]*>.*?<[^>]+>2 r4 z(\d{4}-\d{2}-\d{2})r5 )r' searchDOTALLr) stripsublen)r, r resultr0 patterninnertext_lens r! extract_detailrB 0 s b"
5F
.bii@A''!***,wG
IIgtRYY / GGAJEFF7U"))TEFF5r5 RE266*b%8>>@AH2~$)KKMy!
($/Av Mr2 c d} t t j D ]y \ }}|dk( r?|dz t t j k r t t j |dz } J|j d s\t |j
dd d } { t dd t j t d
}|j }t j }t d }|s t d y d}t j d | }|rt! d
|D }t d| d| dd t# dt% | | dz D ] } | dk( r|n
t | }
|
st& dxx dz
cc< *t) |
t* } | st d| d nt d| dt | dd | D ]] } | d }
|j- d|
f |j/ rt& dxx dz
cc< > t1 j2 |
t4 dd }|j6 d k7 rt& dxx dz
cc< }t9 |j: |
}|d st& dxx dz
cc< |d xs d |
d |d d!} |j- d"t> t* |
|d |d# xs d$|d# xs d$t j@ d%d&| d d d'd(d)|d&d&d*d+d,|v rdnd)tB f |jD d)kD rt& d-xx dz
cc< nt& dxx dz
cc< |jG ` t jJ d |jM t d0t& d- d1t& d d2t& d d3t j |z
d4d5 d y # t< $ r }t& dxx dz
cc< Y d }~d }~ww xY w# t< $ r:}t d.| t jH / t& dxx dz
cc< Y d }~5d }~ww xY w)6N z--pagesr
z--pages==u. === 郑州政务公开 列表收集+详情 ===T)flush )r u 首页获取失败z\?a=dir&d=24&page=(\d+)c 3 2 K | ] }t | y w)N)int).0xs r! zmain..` s ,Q#a&,s u 总页数: u , 抓前 u 页r u 页u 无条目，停止z: u 条r z'SELECT id FROM gov_raw WHERE page_url=?r r Fr r r% r4 z z
aZ
INSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, date_rank, summary, status,
category, visits, content, tags, industry, attachments, group_name, has_table, script_name)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
r5 z
1900-01-01r7 r$ publishedu 政务公开r z[]u 河南za!ec#((m 3CHHQUO,I
\\*
%AGGCOA ./I 0
:4H ??2r *D
++-C
B
QKE
"#K
-u5A,!,, K }Ii[
=TJaY 4q89 1 ux~ %LA L "42 E$ 23 4
dV2c%j\.d; & "Du+C KKAC6 J||~f
"
LLgr%P==C'%LA%L $QVVS1 w<e ! )nT,se2c'l^8(TG
" xc'lCKr} s
2 1 1 1 RYY{ -. 3 , n $v 'H
I1Q'
6L~^ z G F r2