%&jf d Z ddlZddlZddlZddlZddlZddlmZ ddlm Z dZ
dZ ddiZ ej d d
Zd Zd Zd
Zd Zd Zd Zd Zd ZddZd Zd Zedk( r e yy)u
爬虫：昆都仑区人民政府 - 通知公告
站点：https://www.kdl.gov.cn/xwdt/tzgg/
CMS：TRS CMS
分页：index_N.html (N=0~30)，每页10条，共31页~310条
N)
BeautifulSoup)urljoinz!https://www.kdl.gov.cn/xwdt/tzgg/u 昆都仑区-通知公告z
User-Agentz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36 SEARCH_DBz/root/search.db c t j t d } | j d | j d | j d | S )N )timeoutzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALzPRAGMA busy_timeout=30000)sqlite3connectDB_PATHexecuteconns /root/gov_crawler/crawl_kdl.py_get_dbr s> ??7B /DLL*+LL,-LL,- K c | j d | j d | j d | j y )NzCREATE TABLE IF NOT EXISTS gov_raw (id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, content TEXT, source_url TEXT NOT NULL UNIQUE, publish_date TEXT, site_name TEXT, created_at TEXT DEFAULT (datetime('now','localtime')))zACREATE INDEX IF NOT EXISTS idx_gov_raw_site ON gov_raw(site_name)zDCREATE INDEX IF NOT EXISTS idx_gov_raw_date ON gov_raw(publish_date))r
commitr s r ensure_tabler ! s8 LL { |LLTULLWXKKMr c L | j d|f }|j d uS )Nz(SELECT 1 FROM gov_raw WHERE source_url=?)r
fetchone)r
source_urlcurs r
row_existsr ' s&
,,AJ=
QC <<> %%r c Z | j d||||t f | j dkD ryy)NzeINSERT OR IGNORE INTO gov_raw(title, content, source_url, publish_date, site_name) VALUES (?,?,?,?,?)r TF)r
SITE_NAME
total_changes)r titlecontentr publish_dates r
insert_rowr! + s4 LLx*lIFH A r c | t j d| }|r$|j d }|dd d|dd d|dd S y) uA 从URL文件名提取日期: t20260624_930176.html → 2026-06-24z/t(\d{8})_\d+\.html N - )researchgroup)urlmds r extract_date_from_urlr/ 2 sO
(#.A
GGAJBQ%!Aa&1Qq6(++
r c @ | j d }|s| j d }|r|j d D ]p }|j dd }|rJ|j d s9|j d st t
| n|}|j
d| d a|j r |j d
d D ]S }|d
}|s |j d r |j d r/|j d rAt t
| |d
< U t | S y)u 从详情页提取正文z#contentz7.content, .article-content, .TRS_Editor, #zoom, .detailimgsrcr( zdata:httpz
aT)hrefr5 #
javascript)
select_onefind_allget
startswithr BASE_URL replace_with decomposestr)soupdivr1 r2 img_urlr4 hs r extract_contentrD : s
//*
%C oo WX
<<& C''%$C3>>'28;v8N'(C0TW :gYb!9:
c - 1A& A f-all36GPQP\P\]iPj #Ha 0& 1 3x
r c dD ]B }| j | }|s|j d }dt | cxk r dk s>|c S D y)uH 从详情页提取真实标题（页面.title，比列表页更干净）)z.titleh1z.h1z.article-titlez .bt_titlez
.detail-titleTstrip r( )r8 get_textlen)r@ seleltxts r
extract_titlerP N sR V
__S
!
++D+)CCH##
$ r c f t | D ]c } |j | t d }d|_ |j dk( r2t |j d }t | }t | }||fc S e y # t j $ r*}||dz
k rt j d|z Y d}~d}~ww xY w)
u$ 获取详情页内容和真实标题< headersr utf-8rJ html.parserr# N)r( r( )
ranger: HEADERSencoding status_coder textrD rP requestsRequestExceptiontimesleep) r, sessionretriesattemptrr@ r proper_titlees r
get_detailrg X s > )
) C" =A AJ}}#$QVV];)$/ ,T 2 ,, $ ) (( )1$
1<( )s A A33B0 B++B0c g } |j | t d }d|_ |j dk7 r|S t |j d }|j d D ] }|j d D ] }|j d }|r|j d s(|d }d
|vr2|j d } | rt | d
k rUt j dd| j } |j d r|nt t | }
t# |
} |j% | |
| f |S # t $ r |cY S w xY w)u 解析列表页rR rS rU rJ rV ullir4 r5 t20TrG rI z ^\d{2}-\d{2}r( r3 )r: rY rZ r[ Exceptionr r\ r9 findrK rL r) subrH r; r r< r/ append) r, ra resultsrd r@ ri rj r4 r5 r
detail_urldate_strs r parse_list_pagers h s? G KKWbK 9
==C N
/DmmD! :++d# :B AAEE&MV9DD JJTJ*ECJNFF?B6<<>E!% !8ghPT>UJ,Z8H NNE:x8 9! ::& N1 s /D2 2 E ?E c r t } t | t j }|j j t d}d}t t D ]6 }|dk( rt n t d| d}t || }|st d|dz d n|D ] \ }}} t | | r|dz
}t || \ }
} | r| n|} t |
xs dd j d
}
t! |
d k rt d | d d
d |dz
}tt# | | |
|| r.|dz
}| j% t d| d| d d d| d |dz
} t d|dz dt dt! | d| d| d t' j( d 9 | j+ t d| d| d y )Nr index_z.htmlu 第r# u 页无文章，结束r( rV TrG u ⚠ 内容过短: r u
... 跳过u ✓ [z] ( z ()/u 页: u 条, 累计新u
条/跳过u 条u
✅ 完成！新增u 条，跳过)r r r] SessionrT updaterY rX TOTAL_PAGESr< rs printr rg r rK rL r! r r_ r` close)r ra total_new
total_skippager, articlesr rq rr r re final_titlects r crawlr s 9D G OO7#IJk" !)hH:VD6)G"30 E$q&!67 8 +3 'E:x$
+a
$.z7$C !G\*6,EK w}"m<EEDEQB2w|,["-=,>jIJa
$ Wj(KQ
{"["-=,>b
!LMa
# & d1fXQ{m5X ~i[Xbcmbnnqrs
1
7: JJL !)L C
HIr __main__) )__doc__r] r) r
osr_ bs4r urllib.parser r< r rY getenvr r| r r r r! r/ rD rP rg rs r __name__ r r r s . ' o "))K!2
3 &( D&JP z G r