j d Z ddlZddlZddlZddlZddlZddlmZ ddlm Z dZ
ddl Z e j Zej dedd ej# \ ZZej( dkD r ej( ne
Zd Zd
Zd d d
Z ej2 dd Zd Zd Z d Z d Z d Z ddZ d Z!d Z"e#dk( r e" yy)u1
爬虫：广州市生态环境局 - 建设项目环评审批公告
站点：https://sthjj.gz.gov.cn/hjgl/jsxm/hpspgg/
CMS：NFCMS (广东省政府网站群系统)
分页：index_N.html (N=1~100)，每页16条
结构:
N)
BeautifulSoup)urljoind z--pagesu 限制页数)typedefaulthelpz)https://sthjj.gz.gov.cn/hjgl/jsxm/hpspgg/u 广州市-环评审批公告z[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36zhttps://sthjj.gz.gov.cn/)z
User-AgentReferer SEARCH_DBz/root/search.dbc t j t d } | j d | j d | j d | S )N )timeoutzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALzPRAGMA busy_timeout=30000)sqlite3connectDB_PATHexecuteconns crawl_gz_hpspgg.py_get_dbr # s> ??7B /DLL*+LL,-LL,- K c | j d | j d | j d | j y )NaS
CREATE TABLE IF NOT EXISTS gov_raw (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
content TEXT,
source_url TEXT NOT NULL UNIQUE,
publish_date TEXT,
site_name TEXT,
created_at TEXT DEFAULT (datetime('now','localtime'))
)
zACREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)zDCREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date))r commitr s r ensure_tabler * s8 LL
LLTULLWXKKMr c L | j d|f }|j d uS )Nz(SELECT 1 FROM gov_raw WHERE source_url=?)r fetchone)r
source_urlcurs r
row_existsr ; s&
,,AJ=
QC <<> %%r c Z | j d||||t f | j dkD ryy)NzeINSERT OR IGNORE INTO gov_raw(title, content, source_url, publish_date, site_name) VALUES (?,?,?,?,?)r TF)r SITE_NAME
total_changes)r titlecontentr publish_dates r
insert_rowr% ? s5 LLo \9= A r c 2 | j d }|r|j d D ]{ }|j dd }|rU|j d sD|j d st t
| n|}|j
dj | l|j } |j dd
D ]S }|d }|s |j d r |j d r/|j d
rAt t
| |d < U t | S y)u) 从详情页提取正文 - NFCMS广州站z #logPanelimgsrc zdata:httpz
aT)hrefr, #
javascript)
select_onefind_allget
startswithr BASE_URL replace_withformat decomposestr)soupdivr' r( img_urla_taghs r extract_contentr= H s //+
&C
<<& C''%$C3>>'28;v8N'(C0TW !1!8!8!AB
\\#D\1 5Ef
A f-all36GPQP\P\]iPj '! 4f
5 3x
r c t | D ] } |j | t d }d|_ |j dk( rt |j d }t | }d}dD ]W }t j ||j } | s&| j d j d d
j d d
} n ||fc S y# t j $ r*}
||dz
k rt j d |z Y d
}
~
d
}
~
ww xY w)u 获取详情页内容< headersr
utf-8 html.parserr) )z(\d{4}-\d{2}-\d{2})u6 发布时间[：:]\s*(\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2})u6 发布日期[：:]\s*(\d{4}[-/\.]\d{1,2}[-/\.]\d{1,2}) /-. N)r) r) )ranger1 HEADERSencoding status_coder textr= researchgroupreplacerequestsRequestExceptiontimesleep) urlsessionretriesattemptrr8 r# date_strpmes r
get_detailr` ] s > ) ) C" =A AJ}}#$QVV];)$/ A
!QVV,A#$771:#5#5c3#?#G#GS#Q ((! $ )0 (( )1$
1<( )s A8C 7CD C>>Dc g } |j | t d }d|_ |j dk7 r|S t |j d }|j d D ] }|j d }t | dk r#|d j d
}|r|j d sK|d }d |vrU|j d
} | rt | dk sd| v r||j d r|n t | | }
t | dkD r|d j d
nd} | rt j d| sd} |j! | |
| f |S # t $ r |cY S w xY w)u9 解析列表页，返回 [(title, detail_url, date), ...]r? r@ rB rC rD zdiv.conts-listspanrI r r+ r, z
content/post_Tstrip u 穗好办r* rE r) z\d{4}-\d{2}-\d{2})r1 rK rL rM Exceptionr rN selectr0 lenfindget_textr2 r rO matchappend) rW rX resultsr[ r8 r9 spansr+ r, r"
detail_urlr\ s r parse_list_pagerp y se G KKWbK 9
==C N
/D {{+, 6 V$ u:> !HMM# f
y $ &
&E
Q+*> !__V 4T'#t:L
47JN58$$4$0 BHH%98 DHz845'6* N7 s /E E E c
t } t | t j }|j j t d}d}t dt dz D ][ }|dk( rt }ndj | }t || }|s t dj | n|D ] \ }}} t | | r|dz
}t || \ }
} | s| r| } t |
xs dd j! d } t# | d k r#t d
j |d d |dz
}t% | ||
|| r9|dz
}| j' t d j ||d d
| xs d |dz
} t dj |t t# | || t) j* d ^ | j- t dj ||||z y )Nr rE z6https://sthjj.gz.gov.cn/hjgl/jsxm/hpspgg/index_{}.htmlu" 第{}页无文章，采集结束r) rD Trc u ⚠ 内容过短: {}... 跳过r u ✓ [{}] {} ({})( u 无日期u0 第{}/{}页: {}条, 累计新{}条/跳过{}条u8
✅ 采集完成！新增{}条，跳过{}条，共{}条)r r rS SessionrA updaterK rJ MAX_PAGESr3 r5 rp printr r` r rj rh r% r rU rV close)
r rX total_new
total_skippagerW articlesr" ro r\ r# detail_date content_texts
r crawlr s 9D G OO7#IJaQ ' " 19CJQQRVWC"30 6==dC D +3 'E:x$
+a
#-j'#B G[ 8 & ( B
FOOVZO[L< 2%8??cr
KLa
$w
KHQ
*11)U3BZIcXcdea
) , @GGiY\]eYfhqs}~
1
E"H JJL
E
L
LYXbdmnxdx
yzr __main__) )$__doc__rS rO r osrU bs4r urllib.parser MAX_PAGES_DEFAULTargparse_APArgumentParser
_AP_PARSER add_argumentintparse_known_args_AP_ARGS_pagesrv r3 r rK getenvr r r r r% r= r` rp r __name__ r r r s
S
!
Q^ L))+ ! &nnq 0HNN6G 6 * o) "))K!2
3"&*8#J/{b z G r