uj/ d Z ddlZddlZddlZddlZddlZddlmZ ddlm Z ddl
m
Z
dZ dZ dZ
d d
iZd Zd Zd
Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Ze dk( r e yy)u`
南康区-公告公示 爬虫
URL: http://www.nkjx.gov.cn/nkqxxgk/nk1404/list.shtml
CMS: UCAP CMS
列表: ul.newsList > li > a[href] + span.time
分页: list.shtml → list_2.shtml → list_3.shtml ... (198页, 20条/页)
详情: div#zoomcon > UCAPCONTENT > p/table
日期:
附件: a[href] 含 .pdf/.doc/.xls/.zip
N)
BeautifulSoup)urljoin)datetimez0http://www.nkjx.gov.cn/nkqxxgk/nk1404/list.shtmlu 南康区-公告公示u 江西z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 g ?z/root/search.dbc @ t d| t j y )Nz
[DEBUG] )file)printsysstderr)msgs $/root/gov_crawler/crawl_nkjx_tzgg.pydebugr s Jse
3::. c t j d| }|rK|j d dt |j d ddt |j d dS y)Nu' (\d{4})[-/年](\d{1,2})[-/月](\d{1,2}) - 02d )researchgroupint)textms r
parse_dater ! sV
> ' +E!G9E!= > JJqMM s A A!! B *)BB c B | dk( rt }nd| d}t | }|sg S g }|j dd }|s|j dd }|s
t d g S |j d d
D ] }|j d }|r|j d
s(t
t |d
}|j dd xs |j j }|j dd } | r| j j nd}
|j |||
d |S )Nr z+http://www.nkjx.gov.cn/nkqxxgk/nk1404/list_z.shtmlulnewsListclass_pageListu 未找到列表 ulliF recursiveahreftitler spanr* )r< r, date)
BASE_URLr0 findr find_allr$ r r stripappend) page_numr, soupitemsr2 r7 r: r; r< date_spandate_strs r
fetch_list_pagerI 3 s 1};H:VL C=D E
4
+B
YYtJY
/
$& kk$%k0 F GGCLf
x6+gr"4affllnGGF6G2 -69>>'')B
uT8DE F Lr c | j d }|D ]F }|j xs d}t j d| }|s,t |j d c S y)uG 从 createPageHTML('page_div',10, 1,'list','shtml',198) 提取总页数scriptr zBcreatePageHTML\([^,]+,\s*(\d+),\s*\d+,\s*[^,]+,\s*[^,]+,\s*(\d+)\)r r )rA stringr r r r )rE scriptssr r s r
get_total_pagesrO R sX mmH%G
#xx~2 II[]a b qwwqz? " #
r c
, g }g }| j D ] }|j s?t | j }|r#t d |D s|j | O|j j
}|dk( rDt d |j d D }|r|j D ] }|j s?t | j }|r#t d |D s|j | N|j dk( r|j t | x|j dv r'|j d
}|s|j | |j d
}|s|j | v|j d
} | st d | D r|j | |dk( r |j t | |d k( rL|j d
d }
|j dd } |
s|j d| dt d|
d %|dv r)|j d
} | s?|j | R|dk( rY|dk( r)|j d
} | st|j | |j d
} | s|j | | j dd D ]j } | d }
| j j }t j d|
j
sGt d|
}|j d| d| d l t }g }|D ]. }|d d }||vs
|j! | |j | 0 d j# | }||fS ) uO 解析 UCAPCONTENT 内的内容，提取文本段落、表格、图片、附件c 3 $ K | ] }|d v
ywu
N .0cs r
z%parse_ucap_content..d s C_ 4 C pc 3 8 K | ] }|j d v yw))tabledivr2 olN)namerT s r
rW z%parse_ucap_content..l s lqAFF&B Bl Fr8 c 3 $ K | ] }|d v
ywrR rS rT s r
rW z%parse_ucap_content..q s (I!o)=(IrX r[ )rY r\ TrB c 3 $ K | ] }|d v
yw)u
NrS rT s r
rW z%parse_ucap_content.. s "J1(<#<"JrX imgsrcr altz)r2 r] brr\ r: )r; r; z3\.(pdf|doc|docx|xls|xlsx|zip|rar|ceb|ofd|ppt|pptx)$[Nd
)childrenr^ strrB allrC loweranyrA get_textr$ r r r r setaddjoin)con_txtparts attachmentschildr tag has_blocksubttxtrd re r: r; full_urlseen unique_partsrY keycontents r
parse_ucap_contentr \ s$ EK!! 6"zzu:##%DC Cd C C T" jj #:lENNejNLkllI >> ,C 88 HNN , S(Iq(I%I !LLO xx7* SX. \1 LLtL 4 !LLO LLtL 4 !LLO ," nn4n0s"Jc"J JLL%
G^ LLU $
E\))E2&C))E2&C r#b1I3)O(PPQ RS
L
..t.,C S!
D[
E\..t.,C S!..t.,C S!m6"r
c
- 8yvv||~
99KTZZ\ Z 7>H 4&8*A 6 7 8 5DL
#g d? HHSM " # kk,'G K r c g }g }| j d }|r|j j nd}|s'| j dd }|r|j d }|s;| j dd d
i }|r$|j d r|d j }|sR| j d
}|r?|j j }t j dd| j }d}| j dd } | r*| j dd }
|
rt |
j }|s6| j dd di } | r | j d rt | d }| j dd } | rt| j d }
|
r1t |
\ }}|j | |j | n0t | \ }}|j | |j | dj d |D }||||fS )N ucaptitler h1z
article-titler4 Tra metar^ ArticleTitle)attrsr r< u \s*\|\s*南康区信息公开.*$ddscrqr\ z
display-blockPubDatezoomcon)id ucapcontentrj c 3 & K | ] }|s| y w)NrS )rU rY s r
rW z"extract_content.. s 0a!0s ) r@ r rB rp r$ r rz r r rC extendrs )rE ru rv
ucap_titler< r r r{ pub_dater r\ meta_datezoomucapr attachs r
extract_contentr s EK ;'J'1JOO ! ! #rE
YYtOY
4
KKdK+E yy'?y@ DHHY'O))+E IIg FFLLNEFF>EJPPRE H
4 'B ggeOg4 !#((+H IIfVY,?I@ y1!)I"67H 99Uy9 )D yy ' 0 6OGV LL ! v & 1 6OGV LL ! v &kk0U00G '8[ 00r c dd l } | j t }|j }|j d |j d |j |j
y )Nr aK CREATE TABLE IF NOT EXISTS gov_raw (
id INTEGER PRIMARY KEY AUTOINCREMENT,
page_url TEXT,
title TEXT,
content TEXT,
publish_date TEXT,
site_name TEXT,
summary TEXT,
attachments TEXT,
date_rank TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)zSCREATE INDEX IF NOT EXISTS idx_gov_raw_url
ON gov_raw(page_url, site_name))sqlite3connectDB_PATHcursorexecutecommitclose)r connrV s r
init_dbr sT ??7 #D
AII
II + ,KKMJJLr c dd l }|j t }|j }d}| D ]a }|d r|d d d nd} |j d|d |d |d |d t
||d rd
j
|d nd|d xs d f |d z
}c |j |j |S # t $ r }t d
|d d| Y d }~d }~ww xY w)Nr r r! r zINSERT OR REPLACE INTO gov_raw
(page_url, title, content, publish_date, site_name, summary, attachments, date_rank)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)r, r< r rv
z
0000-00-00r u 入库失败: z - ) r r r r r SITE_NAMErs r) r r r )rF r r rV importeditemsummaryr/ s r
save_to_dbr s ??7 #D
AH :+/ ?$y/$3' :
II 3 U W
YZ 26}2E $}-.2Z 0L 5
MH :$ KKMJJL O : $T%[MQC8 9 9 :s AB44 C=CCc
dd l } | j d }|j dt dd |j dd d
|j }t t
d t d
t t }|s t
d y t | }t
d| |j dk( r|nt |j | }t
d| d g }g }t d|dz D ]u }t
d| d| d t | } | st
d| d 3|j t! | t
dt! | d t# | D ] \ }
} t
dt! | dz d| d d d d t | d } | s,t
d |j | d | d d!| d" g d# gt% | \ }
}}}|
xs | d }|xs | d" }|j || d |||d# |r3t! |j' d$ }t
d%| d&|rd'nd( d)| nt
d*|rd'nd( t) j* t, x t! | }t/ d+ |D }t/ d, |D }t/ d- |D t1 |d z }t
d.t d/ t
d0| d t
|rd1| d2||z d3z d4d5nd6 t
d7|d4 t
d8| |r$t
d9t/ | t! | z d4 |j2 st5 | }t
d:| d y t
d; y ) )r< r, r r rv rj z seg=z
| attach=YESnoz | date=u ⚠️ 空正文 | attach=c 3 , K | ] }|d s d yw)r r NrS rU is r
rW zmain..\ s 9!AiLA9s
c 3 8 K | ] }t |d yw)rv N)lenr s r
rW zmain..] s @s1]+,@r_ c 3 b K | ]' }|d s t |d j d ) yw)r rj N)r splitr s r
rW zmain..^ s* Ua) #a l((01Us
/"/z
===== u 爬取完成 =====u 共爬取: u 有正文: u 条 (ri z.1fz%)u 有正文: 0 条u 平均段落数: u 附件数: u 页均条数: u 入库: u 入库: 已跳过)argparseArgumentParser add_argumentr
parse_argsr r r r0 r? rO pagesminr" rI rC r enumerater r r* r+ DELAYsummaxskip_dbr )r parserargsrE total_pagespages_to_fetch all_itemspage_items_countpagerF idxr detail_soupr< r r rv final_title
final_date seg_counttotal with_body total_attachavg_segr s r
mainr s
$
$1P
$
QF
Q=UV
L~N D I F9+]
+, H D
&'!$'K K }
%&$(JJ!O[TZZ9UN K't
,-Ia !+, +
4&.!1:; % F4&
. / E
+
c%j\./"5) ! IC CIq()DM#2,>+?sC D"4;/K 8: !'] ; ! $V #% " 4CK4P 1E7Hk 04=K!1T&\J $E{" & *
f 5 6 :{ePT5UU]^h]ijk6 uQU6VWX JJu C! + Z
NE9y99I@i@@LU9UUX[\eghXiiG HYK 2
34 Kwd
#$ K {&5 )r s = $ D
/
>
O b71r*6V#p zF r