正文 |
日期
用法:
python3 crawl_hhgj.py # 全量（两个栏目）
python3 crawl_hhgj.py 1 # 增量（只爬首页）
N)
BeautifulSoup)urljoin)datetime timedelta SEARCH_DB GOV_DB_PATHz/root/search.dbzhttps://www.hhgj.gov.cnu 个旧市人民政府u 环境保护u 个旧市环境保护z /ztlm/ggqsydwxxgkpt/hjbh1.htmz/ztlm/ggqsydwxxgkpt/hjbh1
new_to_olddot)namesourcelist_urlpage_dirn_pages directiondate_fmtu 环保信息u 个旧市环保信息z"/zfxxgk/fdzdgknr/zdlyxxgk/hbxx.htmz /zfxxgk/fdzdgknr/zdlyxxgk/hbxx6
old_to_newdashz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c t d| y )Nz[hhgj] )print)msgs /root/gov_crawler/crawl_hhgj.pylogr > s GC5/ c b | j dd j dd j S )Nu -个旧市人民政府 u
-个旧市)replacestrip)titles r clean_titler! A s* ==12 6 > >|R P V V XXr c |dk( rPt j d| }|r7|j d d|j d d|j d S yt j d| }|r|j d S y) z Extract date from list item textr z(\d{4})\.(\d{2})\.(\d{2}) - (\d{4}-\d{2}-\d{2})r )researchgroup)textfmtms r extract_date_from_textr. D sx
e| II2D 9 ggaj\1771:,a
|< <
II,d 3 771:
r c |dk( r| d }nt | d d| d} t j |t d }d|_ |j
}g }t j d |t j D ] }|j d }t j d
| } | s.| j d }
t j d|t j } | r4t j dd| j d j nd} t j dd| j }
t j dd|
j }
t |
}
t! | | d }|
j# d r-t ddj% |
j' d dd }nh|
j# d rt d|
dd }nH|
j# d rt d|
dd }n(|
j# d r t |
}n t) ||
}|j+ ||
|f |S # t $ r }t d | d
| g cY d }~S d }~ww xY w)z9Fetch one list page, return [(url, title, date_str), ...]r# r r /z.htm headerstimeoututf-8 ERROR fetching : Nz
]*>(.*?)z.href=\"([^\"]*(?:info|content)/\d+/\d+\.htm)\"z]*>(.*?)z<[^>]+>r z\d{4}\.\d{2}\.\d{2}z\d{4}-\d{2}-\d{2}r z ../../../r& z../../ z../)BASE_URLrequestsgetHEADERSencodingr+ Exceptionr r( finditerDOTALLr* r) subr r! r.
startswithjoinsplitr append)colpage_numurlresphtmleitemsr- liahref
text_matchr+ r date_strfull_urls r fetch_list_pagerS P s/ 1}*o
3z?+1XJd;||C"=
yy
E
[[/ryy
A 2
WWQZ IIG L wwqzYY 3RC
FPrvvj"j&6&6q&9:@@BVX -r48>>@+R7==?E" *$J@ ??; '"1SXXdjjoab.A%B$CDH
__X
&"1T!"XJ/H
__U
#"1T!"XJ/H
__S
!"D6*H sD)H
hx01= 2@ LO uBqc *+ s /H3 3 I<IIIc t j | t d }d|_ |j }t |d }d }d
d d idfd
d d
idfd
dt j d idffD ]) }|j |d |d }|s t | } n d }|j dddi } | rM| j d r<| d j }
t j d|
} | r| j d }|s)t j d| } | r| j d }|st
d| d|fS ||fS # t
$ r}t
d| d| Y d}~yd}~ww xY w)z/Return (content_html, date_str) or (None, None)r1 r2 r5 r6 r7 N)NNz html.parserr divclasscontentzxxgk-conid vsb_contentr r# metar PubDate)attrsr' u( 发布日期[：:]\s*(\d{4}-\d{2}-\d{2})z WARNING: no content for )r: r; r< r= r+ r> r r r( compilefindstrr matchr* r) ) rH rI rJ rK soup content_htmlselectorrU rQ rZ rawr- s r fetch_detailre s ||C"=
yy
} -D L )$d+ *%t, rzz.12D9
ii Xa[1 s8L H 99VFI#69 7D #9o##% HH+S 1 wwqzH IIA4 H wwqzH ( ./X~ !!K uBqc *+s /D; ; E EE c
r dd l }| syt dt |j t }|j }d}| D ]* \ }}}}} |j dt ||||| f |dz
}, |j |j |S # t $ r}
t d| d|
Y d }
~
nd }
~
ww xY w)Nr z Opening DB: zINSERT OR REPLACE INTO gov_raw
(site_name, page_url, title, publish_date, content, category)
VALUES (?, ?, ?, ?, ?, ?)r# z DB error for r7 )
sqlite3r DB_PATHconnectcursorexecute SITE_NAMEr> commitclose) rL rg conncinsertedrH r rQ rb r
rK s r
save_to_dbrr s ,wi ! ??7 #D
AH6;
.2UHlF .
II0 C,G
MH
. KKMJJL O . /#b, - - .s
!B B6B11B6c t j t d z
j d }| d }t dd t d| t d| |r| d d
k( rd g}n2| d g}n+| d d
k( rt d | d d z }nt | d d
d }g }|D ] }t
| | }|st d| d "|D
cg c] \ }} }
|
s
|
|k\ s|| |
f } } }}
| r8|j | t d| dt | dt | d | d dk( rA|s?|r=t d |D r+t d |D } | |k rt d| d| d nP| d d
k( s|r|st d |D st d |D }
|
|k st d| d|
d n t d t | g }t |d D ][ \ }\ }}}t d | dt | d |d d d! t | \ }}|xs |}|j ||||xs d"| d# f ] |S c c}
} }w )$z@Crawl a single column, return [(url, title, date, html, source)]iG )daysz%Y-%m-%dr
z2==================================================zColumn: z3-year cutoff: r r
r# r r z Page z
: no articlesr7 r0 z articlesr c 3 . K | ]
\ }}}|s
| y wN .0_ds r z crawl_column.. ?gaAQ ? c 3 . K | ]
\ }}}|s
| y wrx ry rz s r r~ z crawl_column.. $D71a!Q$Dr z : oldest=z < cutoff, stoppingc 3 . K | ]
\ }}}|s
| y wrx ry rz s r r~ z crawl_column.. r r c 3 . K | ]
\ }}}|s
| y wrx ry rz s r r~ z crawl_column.. r r z : newest=zTotal articles: [z] N2 z...r r
)r nowr strftimer rangerS extendlenallminmax enumeratere rE )rF incrementalthree_years_agor pages all_articlespnarticlesutr} filteredoldest_on_pagenewest_on_pageresultsirH r rQ rb detail_date
final_dates r crawl_columnr s ||~ w(??II*UO v;D"VH
(4&//*+, { | +CE ^$E [ \ )aY !+, c)na,L "3+ '"]+ , .6TT'!QqO?SQ1ITT ) '"RH
aH
iH I { | +KC ? ? ?!$$Dh$D!D!O3'"Y~.>>QRS { | +KC ? ? ?!$$Dh$D!D!O3'"Y~.>>QRS58
3|,-./ G%.|Q%? T! !C as!C %&bs C 89$0$5! k ,H
UJ 0BCMRS T N? Us I
I
I
c
t t j dkD xr t j d dk( } g }t D ] }t || }|j | ! t
| }t d| dt | dt t d |rht |D cg c]
}|d s |d c} }t d|d d
|d t d |D }t d
dj | d d l }|j t } t D ] }| j d|d f | j | j! t d y c c}w # t" $ r}
t d|
Y d }
~
y d }
~
ww xY w)Nr# 1z
Total saved: r0 z (z columns)r% z Date range: r z ~ rv c 3 & K | ] }|d yw) Nry )r{ rs r r~ zmain.. s 0qad0s z Sources: z, a'
INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
SELECT rowid, title, site_name,
CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
FROM gov_raw WHERE category = ?
r
zFTS index rebuiltzFTS rebuild note: )r sysargvCOLUMNSr r rr r sortedsetrC rg ri rh rk rm rn r> ) r all_resultsrF r savedr datessourcesrg ro rK s r mainr sx chh-!#: s(:KK $ sK07#$
{ #E/%#k"2!32c'l^9MNk:QqT!:; l58*Cb { 340K00 i '*+ ,-&w' #C LL
h-!
# #
% ;& & $%%&s%
E&+E& A%E+ + F 4FF __main__)r )F)__doc__osr( r r: bs4r urllib.parser r r environr; rh r9 rl r r< r r! r. rS re rr r r __name__ry r r r s
(
**..bjjnn]DU&V
W % # ) >>/! ) CC4! . 3 Y
2h+"Z.?B!&H zF r