Hj#8 & d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl Z ddl
m Z ddl m
Z
ddl ZdZe dZ ej" dd Zd
Zd Zd Zd
ZddiZd Zd dZd ZefdZd dZ d Z d Z d Z d Z d Z!d Z"d Z#e$d k( r e# yy) u$
crawl_yueyang_tzgg.py - 岳阳市人民政府 公告公示 爬虫
站点: yueyang.gov.cn/gggs/szbm/
编码: GB2312
列表: ul.news-list > li > a + span (日期), li.line 跳过
分页: default_0.htm ~ default_4.htm (5页, 15条/页)
详情: div.content#zoom, 标题 h1.content-title
附件: text
用法:
python3 crawl_yueyang_tzgg.py # 增量爬取 (7天内)
python3 crawl_yueyang_tzgg.py --full # 全量爬取 (最多5页)
python3 crawl_yueyang_tzgg.py --help # 帮助
N)datetime timedelta
BeautifulSoup)urljoinzhttps://www.yueyang.gov.cnz
/gggs/szbm SEARCH_DBz/root/search.dbu 岳阳市-公告公示u 湖南 z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c 2 t dt d| d y )N[z] T)flush)print SITE_NAME)msgs crawl_yueyang_tzgg.pylogr 2 s Ai[3%
- c
t d|dz D ]o } t j | t d }d|_ |j
dk( r|j c S |j
dk( r yt d|j
d | q y# t j $ r8}t d
| d | d | ||k rt j | Y d}~d}~ww xY w)
u" 抓取页面，返回 decoded text )headerstimeoutgb2312 i NzHTTP z for u 请求失败 (尝试 /z): ) rangerequestsgetHEADERSencoding status_codetextr RequestExceptiontimesleep)urlretriesdelayattemptrespes r fetchr, 6 s GaK(
" "<<WbAD $DM 3&yy !!S(eD , ,-U3%89
" (( " 'y'#aSA B
5! "s# =BB%BC.C Cc v | sy| j d s| j d r| S t t dz | S )u 将相对URL转为绝对URL zhttp://zhttps://r )
startswithr CHANNEL_URLhrefs r make_abs_urlr3 I s6 y!T__Z%@ ;$d ++r c | sy t j | j d }|t j t | z
k\ S # t
$ r Y yw xY w)u$ 检查日期是否在指定天数内Tz%Y-%m-%d)days)r strptimestripnowr
ValueError)date_strr5 dts r
date_in_ranger< R sS
x~~ /
<X\\^iT&:::: s AA
AAc ddl m} |t | d }|j d D ]P }|j dd }|s|j d r*|r t j j || n||d< R |j d D ]P }|j d d }|s|j d
r*|r t j j || n||d < R t | S ) uA 保留 HTML 表格结构，仅将相对链接/图片转绝对 URLr r html.parserar2 r. )http
javascript#imgsrc)r@ z//zdata:) bs4r strfind_allr r/ urllibparser )tablebase_urlr tblr? r2 rC rD s r html_table_to_htmlrM ] s !
E
M
2C
\\#
SuuVR (CD@H ,,Xt@H --h>>-G1A1A$1GD#D)H X > ? @ ;F4::k 6M2Mr c , | j ddd }|s| j dd }|s dD ] }| j | }|s n |sy|j d D ], }t | }|j t d | d d
. t
| S ) uj 提取详情页正文 (div.content#zoom)，保留 HTML 结构，
同时将表格转为 markdown 格式divcontentzoom)class_id)rh )zdiv.TRS_Editorzdiv.Custom_UnionStylez div.contentr. rJ
r> )find
select_onerG rM replace_withr rF )r] content_divselectorrJ md_tables r extract_content_htmlrp y s ))E))?K ii&i1 R H //(3K %%g. L%e,
=2hZr):MJKL { r c p t | d }g }|j dd }|s|}|j dd D ]y }|j d rd |j d xs g v r-|j dg }t |t
r|j
}d |v rd|j d
}|sx|j d d j }|j d
xs |j d xs d j }|r|st | } |j d }
d } |
rM|
j d } t j d| } | r#| j d }
|
j dd } t j dd| j }|j || | f | |S )u
解析列表页，返回 [(title, url, date_str), ...]
结构: ul.news-list > li > a + span (日期), li.line 跳过
r> ulz news-listrg liF) recursiveclassliner? r2 r. titleTrO spanz (\d{4}[-/]\d{1,2}[-/]\d{1,2})r r -\s+ )r rj rG r
isinstancerF splitr7 rV r3 rR rU groupreplacesubrZ )htmlr] itemsrr rt li_classr` r2 rx ra ry r:
date_matchraws r parse_list_pager s
} -D E
4 ,B
kk$%k0 %2
66'?v"&&/*?R @ 66'2& h $ ~~'H X yy$**,7#Gu~~D~'AGRNNPD % wwv }}4}0H#CXNJ &&q) ;;sC0 vsE*002
eXx01K%2N Lr c | s||ddfS t | d }|}|j dd }|s|j d }|r9|j d }|r%t j dd | j }|}g d
}|D ]@ \ } }
t j | | } | s | j d j d d
} n t | } | sd} t |t }
||| |
fS )uK
解析详情页，返回 (title, date, content, attachments_json)
r. r> h1z
content-titlers TrO r{ r| ))uP ]*name=["\']?[Pp]ub[Đđ]ate["\']?[^>]*content=["\']?(\d{4}-\d{2}-\d{2})N)zO]*name=["\']?dcterms.issued["\']?[^>]*content=["\']?(\d{4}-\d{2}-\d{2})N)u2 发布日期[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})N)u2 发布时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})N)u, 日期[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})N)u, 时间[：:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})N)u4 (\d{4}[-/]\d{1,2}[-/]\d{1,2})\s*\d{1,2}[:：]\d{1,2}Nr r rz ) r rj rV rR r r7 rU r r rp rb BASE_URL)r
list_title list_dater] rx r tr:
date_patternspattern_mre r^ s r parse_detail_pager s 9b",, } -D
E
4 0B
YYt_ KKdK # FF63*002E H
M $
IIgt $ wwqz))#s3H #4(G &dH5K (G[ 00r c t j t d } | j d | j d | S )u 获取数据库连接r )r zPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000)sqlite3connectDB_PATHexecute)conns r get_db_connectionr s1 ??7B /DLL*+LL,- Kr c | j d|d |d |j dd |d |j dd |j dd |j d d |j d
d |j d d f | j d j d
d
kD S # t j $ r Y yt
$ r}t
d| Y d}~yd}~ww xY w)u
保存一条记录到 gov_raw 表
record = {
"page_url", "title", "publish_date", "site_name",
"group_name", "summary", "content", "attachments", "source_url"
}
zINSERT OR IGNORE INTO gov_raw
(page_url, title, publish_date, site_name, group_name, summary, content, attachments, source_url)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)page_urlrx publish_dater. site_name
group_namesummaryre r^
source_urlzSELECT changes()r Fu DB 错误: N)r r fetchoner IntegrityError Exceptionr )r recordr+ s r
save_to_dbr
s
5 z"w
>2.{#
<,
9b)
9b)
="-
<,
|| ./88:1=AA ! ! k! s B(B+ +C C CC c t dt j v } dt j v sdt j v rt t y t d| rdnd d | st dt
d g }| rt nd
}t | D ] }|d k( r
t d }n t d
| d}t d|d
z d| d| t | }|st d| d St | }t dt | d | so|rm|D cg c] }t |d t
s| }}t dt | d |j | t d |D } | r t d n)|j | t j d t d t | d |s t d y t# }
|
j% } d } d }
d }t' | D ]8 \ }\ }}}| j) d |f | j+ r|
d
z
}
4t | }|st d |d d! |
d
z
}
Xt- ||| \ }}}}|st d"|d d! |
d
z
}
t/ |d# j1 d$% }|d d& j3 d'd( j5 }|||t6 t8 |||t d) }t; |
| r9| d
z
} | d*z d k( r1|
j= t d+| dt | d, n|d
z
}t j d- ; |
j= |
j? t d'd. t d/t6 t d0t | d1 t d2| d1 t d3|
d4 t d5| d1 t d. y c c}w )6Nz--fullz--helpz-hu 启动爬取 (mode: u 全量u 增量)u 增量窗口: u 天r r z /default.htmz /default_z.htmu 抓取列表页 r z: u
列表页 u 获取失败，跳过u 解析到 u
条记录 u 增量过滤后: u 条 (7天内)c 3 D K | ] }t |d t yw)r N)r< INCREMENTAL_DAYS).0its r zmain..Q s UmBqE3C DDUs u* 已触及7天前的数据，停止翻页g ?u
共获取 u
条待处理u 无新数据，退出z)SELECT id FROM gov_raw WHERE page_url = ?u" [跳过] 详情页无法获取: ( u [跳过] 正文为空: r> TrO r ri r| ) r rx r r r r re r^ r r
u
进度: u
已入库g333333?z-=============================================u 爬取完成: u
总计: u 条u
新增: u
跳过: u 条 (已存在/无内容)u
错误: ) sysargvr __doc__r r MAX_PAGESr r0 r, r lenr< extendanyr$ r% r cursor enumerater r r r rV r r7 r GROUPr commitclose) full_mode all_items total_pagespager r r r filteredhas_oldr cinsertedskippederrorsidxrx r: detail_html detail_title detail_datere r^ text_onlyr r s r mainr - s CHH$I388tsxx /
g
9x( C1EF n - .d 34I ))aKk" 19% l3H% ivT:H taxj+ b
CDX *XJ&<= > % l3u:,j 12U%*UrmBqECS.TUHU %c(m_NC D X &UuUUG@B U #
39 < *S^$M23 #$ D
A HG
F,5i,@ 0(
(eXx ={K ::< qLG Ho 4U3BZLA B qLG ;L ;
7 k7K +L",=+>? @ qLG "'=9BBBN DS/))$4::< !! '" &%
dF # MH!|q
j
!C N+;:FG aKF
3a0d KKMJJL "XJ. $%*S^$D)**XJd#$*WI 789*VHD!"8*q Vs N5 N5__main__) r )r. )%r rR r r$ r[ rW r r r r rE r urllib.parser rH r r0 getenvr r r r r r r r, r3 r< rM rb rp r r r r r __name__ r r r s
( (
*%
"))K!2
3 $
) .&, "2
N 84r/1h Fxv zF r