'j d Z ddlZddlZddlZddlZddlZddlZddlZddlm Z ej dd Z dZ dZ
dZed z Zd
Zd Zd Zd
dddddddZeedZddddZd Zd Zd Zd Zedk( r ddlZ ej8 e yy) u Crawler for 白银区人民政府 - 公示公告 (baiyinqu.gov.cn xwzx/GSGG)
JPAAS API: /api-gateway/jpaas-publish-server/front/page/build/unit
列表: tagId=分页 pageId=9139131b15e244b48c146e567381178d, 20条/页, paramJson pageNo/pageSize
详情: /xwzx/GSGG/art/YYYY/art_xxx.html 静态HTML, 正文 div[class*=content] / .article
2026-08-21 重建（原脚本残缺 crawl() 未定义）
N)urljoin SEARCH_DBz/root/search.dbu 白银区人民政府zcrawl_baiyinqu.pyzhttps://www.baiyinqu.gov.cnz7/api-gateway/jpaas-publish-server/front/page/build/unitz0https://www.baiyinqu.gov.cn/xwzx/GSGG/index.htmlzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z
2023-08-18 bulidstatic 1953af59ba95408c9ebfffd5af1eea96 e7f893abbbd14083b0cdc69bfdf92239columnu 分页null 9139131b15e244b48c146e567381178d) parseTypewebIdtplSetIdpageTypetagIdeditTypepageId)z
User-AgentReferer)newskiperrc t t }t j | dd |d< t j
t |t dd }|j }t |j d t r$|j di j dd }|S d }|S # t $ r+}t d
| d | t j Y d }~y d }~ww xY w)
N )pageNopageSize paramJson r F)paramsheaderstimeoutverifydatahtml u 列表API错误(页z): file)
dict
API_PARAMSjsondumpsrequestsgetAPI_URLHEADERS
isinstance Exceptionprintsysstderr)pager rdr" es #/root/gov_crawler/crawl_baiyinqu.py
fetch_listr8 " s
*
F **"%EFF; LL'Z_ `
FFH4>quuV}d4SquuVR $$VR0 Z\
%dV3qc2Ds A4B% !B% % C.!CCc L g }t }t j d| D ]~ }|j d |j d |j d j }}}t t | }||v rY|j | |j |||d |S )NzZ\s*(\d{4}-\d{2}-\d{2})\s*\s*]*href="([^"]*art[^"]*\.html)"[^>]*title="([^"]*)" )urltitledate) setrefinditergroupstripr BASE_URLaddappend)r" itemsseenmr? hrefr> r= s r7 extract_list_itemsrL / s E 5D
[[vx|
} A GGAJ
AGGAJ4D4D4FEdh% $;
S5$?@
A L c dddd}t j d| t j }|r"|j d j |d< dD ] }t j || t j }|s+|j d }t j
dd|t j }t j
d d|t j }t
t j
d
d| j }|d k\ s|j |d < n t j d
| }|r|j d |d< |S )Nr# )r> contentr? z]*>(.*?)r: r> )z7]*class="[^"]*content-txt[^"]*"[^>]*>(.*?)z3]*class="[^"]*article[^"]*"[^>]*>(.*?)z0]*class="[^"]*zoom[^"]*"[^>]*>(.*?)z#]*id="zoom"[^>]*>(.*?)z3]*class="[^"]*content[^"]*"[^>]*>(.*?)z6]*class="[^"]*TRS_Editor[^"]*"[^>]*>(.*?)z8]*class="[^"]*news-content[^"]*"[^>]*>(.*?)z]*>.*?)flagsz]*>.*?<[^>]+>2 rO z(20\d{2}-\d{2}-\d{2})r? )rA searchDOTALLrC rD sublen)r" resultrJ patterninnertext_lens r7 extract_detailr[ = s b"
5F
.bii@A''!***,wQ
IIgtRYY / GGAJEFF7U"))TEFF5r5 RE266*b%8>>@AH2~$)KKMy! " *D1Av MrM c 2 d} t t j D ]y \ }}|dk( r?|dz t t j k r t t j |dz } J|j d s\t |j
dd d } { t dd t j t d
}|j }t j }t d| dz D ] }t | }|rd |vrt d | d
nt | }|st d | d nt d | dt | dd |D ]a } | d }
|j! d|
f |j# rt$ dxx dz
cc< > t' j( |
t* dd } | j, dk7 rt$ dxx dz
cc< }t/ | j0 } | d xs | d }|st$ dxx dz
cc< | d xs d|
d | d }| d xs | d xs d } |j! d t4 t6 |
|||t9 j: d!d"| d d d#d$d%|d"d"d&d'd(|v rdnd%t< f |j> d%kD rt$ d)xx dz
cc< nt$ dxx dz
cc< |jA d t jD d |jG t d,t$ d) d-t$ d d.t$ d d/t j |z
d0d1 d y # t2 $ r }
t$ dxx dz
cc< Y d }
~
d }
~
ww xY w# t2 $ r:}
t d*|
t jB + t$ dxx dz
cc< Y d }
~
9d }
~
ww xY w)2Nr< z--pagesr: z--pages==u === 白银区公示公告 ===T)flush )r artu 页u 无内容，停止u 无条目，停止z: u 条r= z'SELECT id FROM gov_raw WHERE page_url=?r r F)r r r r r> rO z z
r? z
1900-01-01aZ
INSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, date_rank, summary, status,
category, visits, content, tags, industry, attachments, group_name, has_table, script_name)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
rQ r# publishedu 公示公告r z[]u 甘肃z rO r? s r7 mainr Z sa I#((# 01 >a!ec#((m 3CHHQUO,I
\\*
%AGGCOA ./I 0
)$7 ??2r *D
++-C
BaQ ' 3$ uD( E$ 23 4 "4( E$ 23 4
dV2c%j\.d; ( "Du+C KKAC6 J||~f
"
LLgwuU==C'%LA%L $QVV, L1DMEe ! )nM,se2eWH(MGv;>$v,>,D
" xeT4FF:r73DS9 GRT 8w#6aA{
<r s 2 1 1 1 RYY{ -. # ! (
N
N =v
/2
0
11Q'
:C~L z G F rM