j4 d Z ddlZddlZddlZddlZddlZddlZddlmZ ddlZddl m
Z
dZ e dZ dZ
dZd Zd
Zd d d
Z ej$ Zej( j+ e h dZd Zd Zd Zd Zd Zd Z d Z d Z d Z d Z d Z!d dZ"e#d k( rbdZ$ e%ejL dkD rAejL dd D ]/ Z'e'jQ d s e)e'jU d d Z$1 e"e$ yy# e+$ r Y Dw xY w)!u
赵县人民政府 - 公告公示 爬虫
CMS: WebIMS v8.0.2 (石家庄市人民政府平台)
列表: index.html 首页15条（AJAX分页不可用）
详情: UUID路径, biaoti标题, conN正文
N)datetime)
BeautifulSoupzhttp://www.zhaoxian.gov.cnz8/columns/0f1b1d84-6a48-4c2f-8419-7fe7e4b9fb51/index.htmlz$0f1b1d84-6a48-4c2f-8419-7fe7e4b9fb51u 赵县人民政府u 县区z/mnt/data/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8)z
User-AgentAccept>) pdddldth1h2h3h4h5h6hrlioltdthtruldivnavpreformmainasidetabletbodytfoottheadcenterdialogfigurefooterheaderhgroupaddressarticlecaptiondetailssectionfieldset
blockquote
figcaptionc | yddl }ddlm g fd | dj }|j dd| }|j dd | }|j S )
ul 块级边界出换行、行内节点直接拼接、
出换行（≈ 浏览器看到的换行结构）。N r )NavigableStringc F | j D ] }t | rj t | *t |dd dk( rj d Lt |dd t
v r+j d | j d | y )Nnamebr
)children
isinstanceappendstrgetattr _BLOCK_TAGS)nodechr1 outwalks (/root/gov_crawler/crawl_zhaoxian_gsgg.pyr? zbody_text..walk: s --
B"o.
3r7#VT*d2
4 VT*k9
4 R
4 R
z[ \t\r\f\v]*\n[ \t\r\f\v]*r5 z\n{3,}
)rebs4r1 joinsubstrip)el_retr1 r> r? s @@@r@ body_textrK 2 sb z#
C H
A -tQ7A 61%A 779rA c V | syt j dd| j } | S )u! 清理文本：去除多余空白r0 z\s+ )rC rF rG texts r@
clean_textrP M s(
66&#t $ * * ,D KrA c t j d| }|r7|j d d|j d d|j d S t j j d S )u# 从文本中提取日期 YYYY-MM-DDz(\d{4})-(\d{2})-(\d{2}) - z%Y-%m-%d)rC searchgroupr nowstrftime)rO ms r@ extract_publish_dater[ U s\
,d3A''!* Qqwwqzl!AGGAJ<88 <<> " ": ..rA c t t j d| }|r|j d S t | }| dS )u$ 从文本中提取完整日期时间z!(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})rR z 00:00)rC rV rW r[ )rO rZ ds r@ extract_datetimer^ ] s:
6=Awwqz T"AS<rA c * | syt | d } |ddg D ] }|j g }|j ddg D ]( }|j d }|s|j | * |s|j d g}d j | S )
u% 清洗正文HTML，保留段落结构r0 html.parserscriptstyler r T)rG rB )r decomposefind_allget_textr8 rE ) html_contentsouptagtextselemrJ s r@ clean_content_htmlrk f s } 5DXw'(
E
sEl+ MMM % LLO T *+ ;;u rA c f t j | j d j S )Nutf-8)hashlibmd5encode hexdigestrN s r@ ro ro y s# ;;t{{7+ , 6 6 88rA c \ g }t | d }|j dt j d }|s
t d |S |j dt j d D ]> }|j dt j d
}|s-|j
d d }|r|j d
s t |z }|j dt j d }|rt |j nd }|j dt j d } d }
| rt | j }
nA|j dt j d } | rt | j }
|s'|s+|j |||
f A |S )u2 解析列表页，返回 [(title, url, date), ...]r` r pagingCont_idu [WARN] 未找到列表容器z
border-bottomrb az\.html)hrefrx r0 httpzline-height:\s*35pxspanu 发布时间)stringz
color:#666666)
r findrC compileprintrd get
startswithBASE_URLrP re r[ r8 ) htmlitemsrg
paging_divitem_diva_tagurl title_divtitle date_spandate_strdate_divs r@
parse_listr ~ sp E } -D5RZZ%?@J
./ ''RZZ@P5Q'R 1
c
9(=
> ii# s~~f-S.C JJuBJJ7M,NJO 4=
9--/02 JJvbjj .IJJ +I,>,>,@AH zz%rzz:J/KzLH/0A0A0CD S LL%h/ 0/12 LrA c t | d }d}d}d}d}|j dd }|rN|j dt j d }|r&t t | }|j
d d }|j dd
} | r| j }
t j d |
} | r| j d }|s)t j d
|
} | r| j d }t j d|
} | r | j d j }|j dd } | rt t | }||||fS )u? 解析详情页，返回 (title, publish_date, source, content)r` r0 r biaotirt rz zfont-size:\s*26pxrv r5 titNu6 发布时间[：:]\s*(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})rR u( 发布时间[：:]\s*(\d{4}-\d{2}-\d{2})u 来源[：:]\s*([^\s<]+)conN)
r r| rC r} rK rP replacere rV rW rG rk r9 )
r r rg r publish_datesourcecontent
biaoti_div
title_spantitn_div titn_textrZ conn_divs
r@ parse_detailr s@ } -D EL
FG 5X.J __V2::>R3S_T
j 45EMM$+E yy6y*H %%' IIOQZ [ 771:L EyQA wwqz II19 = WWQZ%%'F yy6y*H$S]3 , //rA c B | s t d yd}| D ]e } t j dt g|d ddd }|j dk( r|dz
}n*d |j
v rnt d
|j
d d g |S # t $ r}t d
| Y d }~d }~ww xY w)u! 批量写入 gov_raw + gov_searchu 无新增数据r sqlite3sqlT inputcapture_outputrO timeoutrR UNIQUE constraint [DB] N [DB] 错误: )r~
subprocessrunDB_PATH
returncodestderr Exception)
items_dataaddeditemresultes r@ insert_to_dbr s
!"
E ) ) ^^G$5k #$F
A%
&&--7GFMM$3$7#89:)" L ) OA3' ( ( )s A"A== B BB c | d }| d j dd }| j dd j dd }| j dd }| j dd j dd }t }t }t j
j
d }|d
d } t | }
d | d
| d
| d
| d
| d
| d
| d
| d} | S )u 构建INSERT SQLr r 'z''r r0 r r z%Y-%m-%d %H:%M:%SNr zINSERT OR IGNORE INTO gov_raw
(page_url, title, content, publish_date, source_url, site_name, summary, group_name, industry)
VALUES
('z', 'u ', '政府公告');
)r r SITE_NAMEGROUPr rX rY ro ) r page_urlr r r r site_namerW rX summary content_hashsql_raws r@ build_insert_sqlr s E{H M ! !#t ,Ehhy"%--c48G88NB/L
XXh
#
+
+C
6FI E
,,.
!
!"5
6CDSkGw 0 THEN last_insert_rowid() ELSE 0 END;
r zINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary)
SELECT rowid, title, site_name, summary FROM gov_raw WHERE rowid = r r r r r u 入库成功: u 无新增u === 完成 ===)r~ LIST_URLsessionr encodingr r rO lensetr r r r stdoutrG splittimesleepr r r8 int
ValueErrorr ) max_pagesrr r
known_urlsr new_itemsr r r detail_titlepub_dater r content_len item_datadb_addedr combined_sqlr> r fts_sqls r@ r r s ,. HXJ
KK"K -
qvv E N3u:,
'(
$% J
!g h dB
!CI==CVCVCXV]] 0 0 2 8 8 >?^a^cJ
KJ(
)* I % % sH *
5":,c*+ C ,A AJ 3?qvvs2K/ h L H '--/* ,[M Sb@Q?RS T " $
,I6 %#
3K%N
Js9~&d
+, $ -D#
- #E{ . $ (&#'dB
$$) ----/C&$' $4E !" qy'DDI7 K# # & 0")+/dB
!A
*&--?
ds(;'< =>E$ -L xj-.
m
_
,QC01&
Jqc" # JJqM d * &$%E&&
-s+,,
-s "L BL3 M AN0M= A N L0L++L03 M ?M M: #M55M:= N N
N N N0N++N0__main__z--pages==)r )rR ),__doc__rC sysjsonr rn r r requestsrD r r r COLUMN_IDr r r HEADERSSessionr headersupdater; rK rP r[ r^ rk ro r r r r r r __name__pagesr argvargr r r r rA r@ r sM
(ZO P 2
DO
(
w * 6/ &9
$P&0T6.
w t z
E
388}q88AB< C~~j) #q 1 2E 5 " s C<<DD