uj d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZmZ dZ dZ
e j dd Z ej ed z
j d Zd
Zdd Ze fd Zd
Zedk( r e yy) N)datetime timedeltazhttp://www.wschina.gov.cnz /kfswsxwz/c00511/pc/list.html SEARCH_DBz/root/search.dbiG )daysz%Y-%m-%du 尉氏县-环境保护c t | D ]o } t j dddd| gdd }|j dk( r+|j r |j j d d
c S t
j d q y
# Y xY w)Ncurlz-sLz
--max-time15T )capture_outputtimeoutr zutf-8replace)errors )range
subprocessrun
returncodestdoutdecodetimesleep)urlretriesirs "/root/gov_crawler/crawl_wschina.pyfetchr
s{
7^ |T3 GX\fhiA||q QXXxxwyAA
1
s AA??Bc dx}}t j d| }|r |j d j }t j d| }|r |j d j }d}t j d| t j }|r |j d j }|r fd}t j
d|| }|||fS )Nr z/]*content="([^"]*)" z*]*content="([^"]*)"zC]*id="zoomcon"[^>]*>(.*?)
c | j d }t j d| }|r|j d }|j d s|j d r|S |j d s&j d dz |j d z nj d |z }|j
d|z dz d|z dz }|S ) Nr z
src="([^"]*)"r zhttp://zhttps:///zsrc="")groupresearch
startswithrstriplstripr
)mtagssrcabbase_urls r _fix_imgz extract_detail.._fix_img s ''!*C *C0Aggaj>>),z0J JKN>>Z]K^hooc*S03::c?Bemetetuxey|ekk'C-#"5w|c7IJJ z
]*>)r% r& r$ stripDOTALLsub)htmlr/ titlepub_dater* contentr0 s ` r extract_detailr9 s EH
DdKA!''!*""$%
?FAQWWQZ%%'(G
XZ^`b`i`ijAAGGAJ$$&' && ': (G ##r1 c t t t z } | s t d y t j
d| t j }|s t d y t j |j d }t dj t | t j t }d}|D ] }|d d d t k rt t j |d d
z }t | }|sCt! | \ }}} | sd |j# d d
z dz } t j$ dd
| d d j' }
d} |j) | t+ |d |d | |d d d ||t, |
f |dz
} |j1 |j3 t dj | y # t. $ r)} t dj |d | Y d } ~ 9d } ~ ww xY w)NFailedz var articleList = (\[.*?\]);zNo articleListr z Total: {}r pubDate
urlspczsummaryr z
z<[^>]+> zINSERT OR REPLACE INTO gov_raw (id, title, content, publish_date, source_url, page_url, site_name, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)idr6 z Error {}: {}zDone! {} records)r BASELIST_URLprintr% r& r3 jsonloadsr$ formatlensqlite3connectDB_PATHCUTOFFr9 getr4 r2 executeint SITE_NAME Exceptioncommitclose)
r5 r* articlesdbtotala
detail_url detail_html_r8 r@ sqles
r mainr^ * s !D x &
14CA e$%vzz!''!*%H +
S]
+, !B
E
7 Y< v %xDJJqy 1$ 77
J' H&{3
1g%!%% 2*>">"G&&R1$37==? \ 7 JJsS4\1W:w) Sb@QS]_iktv} ~ QJE
7 IIK
#
#E
*+ 7 "))!D'15 6 6 7s 8G G: G55G:__main__) )ossysr% rF r rJ r r r rC rD getenvrL nowstrftimerM rQ r r9 r^ __name__ r1 r rh s / / / (" *
"))K!2
3
(,,.9%0
0 : :: F $ #' $.,2 zF r1