j d Z ddlZddlZddlZddlZddlZddlmZ ddlm Z dZ
dZ ddiZ ej d d
Zd Zd Zd
Zd Zd Zd Zd ZddZd Zd Zedk( r e yy)u
爬虫：乌当区人民政府 - 通知公告
站点：https://www.gzwd.gov.cn/xwzx/tzgg/
CMS：TRS CMS
分页：index_N.html (0-indexed, N=0~232)，每页15条，共233页3486条
N)
BeautifulSoup)urljoinz"https://www.gzwd.gov.cn/xwzx/tzgg/u 乌当区-通知公告z
User-Agentz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36 SEARCH_DBz/root/search.db c t j t d } | j d | j d | j d | S )N )timeoutzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALzPRAGMA busy_timeout=30000)sqlite3connectDB_PATHexecuteconns crawl_wudang.py_get_dbr s> ??7B /DLL*+LL,-LL,- K c | j d | j d | j d | j y )NaS
CREATE TABLE IF NOT EXISTS gov_raw (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
content TEXT,
source_url TEXT NOT NULL UNIQUE,
publish_date TEXT,
site_name TEXT,
created_at TEXT DEFAULT (datetime('now','localtime'))
)
zACREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)zDCREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date))r
commitr s r ensure_tabler ! s8 LL
LLTULLWXKKMr c L | j d|f }|j d uS )Nz(SELECT 1 FROM gov_raw WHERE source_url=?)r
fetchone)r
source_urlcurs r
row_existsr 2 s&
,,AJ=
QC <<> %%r c Z | j d||||t f | j dkD ryy)NzeINSERT OR IGNORE INTO gov_raw(title, content, source_url, publish_date, site_name) VALUES (?,?,?,?,?)r TF)r
SITE_NAME
total_changes)r titlecontentr publish_dates r
insert_rowr! 6 s5 LLo \9= A r c t j d| }|r$|j d }|dd d|dd d|dd S t j d| }|r0|j d dd |j d dd }}| d| d S y
) u8 从URL路径提取精确日期: t20260615 → 2026-06-15z/t(\d{8})_\d+\.html N - z/(\d{6})/t\d+_\d+\.htmlz-01 )researchgroup)urlmdsymps r extract_date_from_urlr1 ? s (#.A
WWQZRa&2a7)1R!WI..
,c2A
2A
1Q2AbT~
r c . dD ] }| j | }|s|j d D ]p }|j dd }|rJ|j d s9|j d st t
| n|}|j
d| d a|j r |j d d
D ]S }|d }|s |j d r |j d
r/|j d rAt t
| |d < U t | c S y)u" 从详情页提取正文 - TRS CMS)z.contentz#contentz.article-contentz .TRS_Editorz .con_textz#zoomimgsrcr( zdata:httpz
aThrefr8 #
javascript)
select_onefind_allget
startswithr BASE_URL replace_with decomposestr)soupseldivr3 r4 img_urla_taghs r extract_contentrI M s ` ooc" ||E*
$ggeR(s~~g 6MMO
$ c 5
9&MQ\\& 1!,,s:KTUT`T`amTn$+Ha$8E&M
9 s8O r c L t | D ]V } |j | t d }d|_ |j dk( r%t |j d }t | }|c S X y # t j $ r*}||dz
k rt j d|z Y d}~d}~ww xY w)
u 获取详情页内容< headersr utf-8 html.parserr# Nr( ) ranger= HEADERSencoding status_coder textrI requestsRequestExceptiontimesleep)r, sessionretriesattemptrrC r es r
get_detailr` ` s >
) ) C" =A AJ}}#$QVV];)$/ $
) (( )1$
1<( )s AA&&B#9 B B#c g } |j | t d }d|_ |j dk7 r|S t |j d }|j dd D ] }|d }t j d
| s |j d d xs |j d
}|rt | dk rV|j d r|nt t | }t | } |j! ||| f |S # t $ r |cY S w xY w)u9 解析列表页，返回 [(title, detail_url, date), ...]rK rL rN rO rP r6 Tr7 r8 z/\d{6}/t\d+_\d+\.htmlr r( strip r5 )r= rS rT rU Exceptionr rV r< r) r* get_textlenr> r r? r1 append)
r, r[ resultsr^ rC r6 r8 r
detail_urldate_strs
r parse_list_pagerl o s G KKWbK 9
==C N
/D
]]3T]
* 6yyy148 gr"r s / $ o "))K!2
3 "& &
:-o^ z G r