j! d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
m Z ddlZ e
e j dkD r2e j d j r ee j d ndZe ed ee z dZdZd d
d Z ej. d d
ZdZd Zd Zd Z d Z d Z ddZ d Z d Z!e"dk( r e! yy)u
爬虫：贵阳市生态环境局 - 项目环评审批公示
站点：https://sthjj.gz.gov.cn/hjgl/jsxm/hpspgg/
CMS：TRS CMS 标准分页模式
N)
BeautifulSoup)urljoinurlparse z[AutoPg] max_pages=z)https://sthjj.gz.gov.cn/hjgl/jsxm/hpspgg/u 贵阳市-项目环评审批z[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36zhttps://sthjj.gz.gov.cn/)z
User-AgentReferer SEARCH_DBz/root/search.db c t j t d } | j d | j d | j d | S )N timeoutzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALzPRAGMA busy_timeout=30000)sqlite3connectDB_PATHexecuteconns crawl_guiyang.py_get_dbr " s> ??7B /DLL*+LL,-LL,- K c | j d | j d | j d | j y )NaS
CREATE TABLE IF NOT EXISTS gov_raw (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
content TEXT,
source_url TEXT NOT NULL UNIQUE,
publish_date TEXT,
site_name TEXT,
created_at TEXT DEFAULT (datetime('now','localtime'))
)
zACREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)zDCREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date))r commitr s r ensure_tabler ) s8 LL
LLTULLWXKKMr c L | j d|f }|j d uS )Nz(SELECT 1 FROM gov_raw WHERE source_url=?)r fetchone)r
source_urlcurs r
row_existsr : s&
,,AJ=
QC <<> %%r c Z | j d||||t f | j dkD ryy)NzeINSERT OR IGNORE INTO gov_raw(title, content, source_url, publish_date, site_name) VALUES (?,?,?,?,?)r TF)r SITE_NAME
total_changes)r titlecontentr publish_dates r
insert_rowr% > s5 LLo \9= A r c | j d }|r|j d D ]p }|j dd }|rJ|j d s9|j d st t
| n|}|j
d| d a|j r t | S | j d }|r t | S y)
u 从详情页提取正文z@#content, .content, .article-content, .info-content, .TRS_Editorimgsrc zdata:httpz
body)
select_onefind_allget
startswithr BASE_URL replace_with decomposestrfind)soup content_divr' r( img_urlr+ s r extract_contentr8 G s //"deK ''. C''%$C3>>'28;v8N'(C0TW :gYb!9:
; 99V D 4y
r c z t | D ] } |j | t d }d|_ |j dk( rt |j d }t | }d}|j }g d} | D ]} }
t j |
| } | s | j d j d d
j d d
j d d j d
d
j dd
} n ||fc S y# t j $ r+} ||dz
k rt j d|z Y d} ~ )d} ~ ww xY w)u 获取详情页内容< headersr
utf-8 html.parserr) )u+ (\d{4}[-/\.年]\d{1,2}[-/\.月]\d{1,2}日?)u6 发布时间[：:]\s*(\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2})u6 发布日期[：:]\s*(\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2})z(\d{4}-\d{2}-\d{2})r u 年-u 月u 日/. N)r) r) )ranger. HEADERSencoding status_coder textr8 researchgroupreplacerequestsRequestExceptiontimesleep)
urlsessionretriesattemptrr5 r# date_strrH
date_patternspmes
r
get_detailr[ \ sD > ) ) C" =A AJ}}#$QVV];)$/ vv!
' A !T*A#$771:#5#5eC#@#H#Hs#S#[#[\abd#e#m#mnqru#v#~#~ @C DG $H ((' $ )6 (( )1$
1<( )s A>C<A'C<<D: D55D:c t j t t d } d| _ | j
dk7 rt
d| j
yt j d| j }|rAt |j d }|t z dz
t z }t
d | d
| d |S t | j d }|j d
}d}|D ]O }|j dd }t j d| } | s.t | j d }
|
|kD sN|
}Q t
d| |S # t $ r} t
d| Y d} ~ yd} ~ ww xY w)u 获取总页数r r; r= r> u 无法访问首页: HTTP r u 记录总数[：:]\s*(\d+)r u 首页检测到总记录: u 条, 共u 页r? za[href*="index_"]hrefr) zindex_(\d+)\.htmu 首页检测到最大页: u 获取总页数失败: N)rM r. r0 rE rF rG printrI rJ rH intrK PAGE_SIZEr select Exception) rU rY totalpagesr5
page_linksmax_pagear] m2rX rZ s r get_total_pagesri { sL LL7B ?
==C -amm_= > II3QVV <
OEY&*y8E .ugXeWCH IL QVV]3[[!45
!A55$D.5B $x< H
! *8*56
's+,s2 AE A"E 4AE
E -E E"
E E"c J t } t | t }|dk( r t d | j yt d| d t j }|j j t d}d}t dt |t xs | dz D ]( }|dk( rt }n d|dz
d} |j |d
}d |_ |j d k7 r t d
| d|j d ` t% |j& d } d}
| j) dd D ]d } | d } | j dd xs | j+ d }
| j- d s| j- d sd| v sVd| v s"| j- d s| j- d r}|
rt/ |
d k rt1 j2 d |
rt5 || }|
dz
}
t7 | | r|dz
}t9 || \ }}|r t% |d j+ d nd}t/ | d k rt d!|
dd" d# |dz
} t; | |
||| r3|dz
}| j= t d$| d%|
dd& d'|xs d( d) `|dz
}g ||z }t d
| d*| d+|
d,| d-| d. t? j@ d + | j t d/| d0| d1||z d. y# t" $ r }t d
| d| Y d}~ud}~ww xY w)2u 全量采集r u 无法确定总页数，退出Nu
共 u 页，开始采集...r z/https://sthjj.gz.gov.cn/hjgl/jsxm/hpspgg/index_z.htmr: r r= r> u 第u 页 HTTP u ，跳过u 页请求失败: r? rg T)r] r] r" r) )stripz.htmlz/contentindex_z index.htmz
index.html z^\d[\d\.\s\-]*$ u ⚠ 内容过短: r u
... 跳过u ✓ [z] ( z... (u 无日期)rA u 页: 本页u 条, 累计新u
条/跳过u 条u
✅ 采集完成！新增u 条，跳过u 条，总处理)!r r ri r^ closerM Sessionr< updaterE rD min_MAX_PGr0 r. rF rG rb r rH r- get_textendswithlenrI matchr r r[ r% r rO rP )r total_pagesrR total_new
total_skippagerQ rU rZ r5 links_foundrg r] r"
detail_urlr# rV content_text
page_totals r crawlr s_ 9D!#Ka
./
F;- 5
67 G OO7#IJa['*@[AAEF @ 19CCDF84PC C ,A AJ}}#dV9Q]]O9EF $ QVV]3 s . & AV9DEE'2&@!**4**@E MM&)T]]7-CzUYGY4 4==#= |A\CJN xx*E2 d+J 1 K$
+a
!+:w ? GX Za=-@IIPTIUfhL< 2%,U3BZL
CDa
$w
HEQ
{"U3BZLh>U+=VVWXYa
M& P +
dV1[M[M PY{Zdeodppstu
1
A@D JJL ' {,zl/ZcdnZnYoor
sto E$ 04 5 s AK== L"L L"__main__) )#__doc__rM rI r ossysrO bs4r urllib.parser r _SYSrx argvisdigitr_ ru r^ r3 r0 r rE getenvr r` r r r r% r8 r[ ri r __name__ r r r s
* "499~ 1diil6J6J6L#diil
RV
#g,
./ 7 * o) "))K!2
3 "&* > @Vur z G r