正文
/
用法:
python3 crawl_zibo_epb.py # 全量（近3年）
python3 crawl_zibo_epb.py 1 # 增量（仅最新页）
N)
BeautifulSoup)urljoin)datetime timedelta)ThreadPoolExecutor as_completed SEARCH_DB GOV_DB_PATHz/root/search.dbzhttps://epb.zibo.gov.cnz>/gongkai/channel_c_5f9fa491ab327f36e4c13066_n_1605682663.2363/u 淄博市生态环境局u 淄博市生态环境局-受理
z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 c t d| y )Nz [zibo_epb] )print)msgs #/root/gov_crawler/crawl_zibo_epb.pylogr . s Ku
c 6 t t dt dt t j } t d D ] } t
j | t d }d|_ |j }t j d | }|r t |j! d c S t d |dz d
t j d
|d
z z y# t $ r9}t d|dz d | t j d
|d
z z Y d }~d }~ww xY w)Nz%?open=fdzdgknr&nr_page=1&nr_per_page=&_t= headerstimeoututf-8z ERROR fetching first page (try ): z(fdzdgknr_total\s*=\s*parseInt\("(\d+)"\)z total not found (try ), retryingr )BASE_URL LIST_PATHPER_PAGEinttimerangerequestsgetHEADERSencodingtext Exceptionr sleepresearchgroup)urlattemptresphtmlems r fetch_total_countr6 2 s
Jyk!FxjPTUXY]YbYbYdUeTf
gC8
$ <<WbAD #DM99D
IIA4 H qwwqz? " %gai[ <=
1w{?#
$
1'!)CsC D JJq7Q; ' s /C D /DDc t t d| dt dt t j }t d D ] } t
j |t d }d|_ |j }t |d
}|j d }|r n4t d| d|d
z d t j d |d z z g S |j d }g } t! | D ] \ }
} | j# d } | r| j d s+| d }
| j d xs | j% d xs d}|j' }|
t) | k r||
j% d nd}t+ t |
}| j- |||f | S # t $ r=}t d| d |d
z d | t j d |d z z Y d }~d }~ww xY w)Nz?open=fdzdgknr&nr_page=z
&nr_per_page=r r r r r z
ERROR page z (try r r r html.parserztd.zfxxgk-list-title Page z empty rows (try r ztd.zfxxgk-list-timeahreftitleT)strip )r r! r" r# r$ r% r& r' r( r) r* r+ r r, r select enumeratefindget_textr= lenr append)pager0 r1 r2 r3 r4 souprowsdatesitemsitdr: r; r< date_strfull_urls r fetch_list_pagerN E s
Jyk!8mH:UYZ]^b^g^g^iZjYk
lC8 <<WbAD #DM99D
T=1{{12 gdV,WQYK{ CD
1w{?# KK- .E E4 22 GGCLf
yg>!**4*"8>B
45E
N58$$4$08T*
hx01 2 L3 -vVGAI;c!= > JJq7Q; ' s /F G$'2G G$c > t j | t d }d|_ |j }t |d }|j dd d
i }|r$|j d r|d j nd
}d
}|j dd di }|rK|j d r:t j d|d j } | r| j d }|j dd }
|
s|j dd }
|
rdD ]( } |
j | D ] } | j * |
j t j d D ]( } | j# g d }
|
s|
j * |
j dd }|r|d }
t% |
}nd
}|s,d D ]' |j dfd }
|
s t% |
} n |st
d | d ||fS |||fS # t
$ r}t
d| d| Y d }~yd }~ww xY w) Nr r r z ERROR : NNNr8 metaname ArticleTitle)attrscontentr> PubDatez(\d{4}-\d{2}-\d{2})r divc T | xr% dt | t r| ndj | v S Ndetails-content
isinstancestrjoincs r
z fetch_detail..y s- A4y:KU_`acfUgPQmpmumuvwmx:y r )class_r[ )id) z.sk-breadcrumbztable.details-infozh1.details-titlez.share-details-rowz.sharez.details-erweimaz
.related-itemzli.details-info-fontsizezli.details-change-fontsizeu 打印)string)r: lispanbuttonc T | xr% dt | t r| ndj | v S rZ r] ra s r rc z fetch_detail.. sE ! CHHYcmnoqtcu^_{~ |D |D EF |G IH r )z zfxxgk-textnr_rightrV articlec V | xr% t | t r| ndj | v S )Nr\ r] )rb clss r rc z fetch_detail.. s+ A>"?@D !
!))% 9H) I
$R.K;' D C ))E2s)tK "; /
( ./Xu$$ 5 ((W hse2aS !" s /G8 8 H HH c 2 i }t t 5 }| D ci c] }|j t | | }}t | D ] }|| } |j ||< d d d |S c c}w # t $ r}d||< Y d }~Ad }~ww xY w# 1 sw Y |S xY w)N) max_workersrQ )r MAX_WORKERSsubmitr} r resultr+ )urlsresultsexr0 fut_mapfutr4 s r fetch_detail_batchr s G 4 2@DE299\3/4EE ( 2C#,C
2 "zz| 22 N F
2 1
2
2 NsE B A,B A1 B ,B 1 B :B?B B B Bc
X dd l }| sy|j t d }|j }d}| D ]- \ }}}} |j dt
||||t f |dz
}/ |j |j |S # t $ r} t d| d| Y d } ~ qd } ~ ww xY w)Nr < r zINSERT OR REPLACE INTO gov_raw
(site_name, page_url, title, publish_date, content, category)
VALUES (?, ?, ?, ?, ?, ?)r z DB error for rP ) sqlite3connectDB_PATHcursorexecute SITE_NAMESOURCEr+ r commitclose)
rI r connrb insertedr0 r< rL r| r4 s
r
save_to_dbr s ??7B? /D
AH.3
.*UHl .
II0 C,G
MH
. KKMJJL O . /#b, - - .s %B B)B$$B)c ddl }| s
t S |j t d } dj dt | z }|j
d| d| j }|D ch c] }|d c}|j S c c}w # t $ r t cY |j S w xY w# |j w xY w) u2 返回已入库的 URL 集合 (增量停止条件)r Nr r ,?z0SELECT page_url FROM gov_raw WHERE page_url IN ())
r setr r r` rC r fetchallr r+ )r r r phrG rs r check_existingr s u ??7B? /D
XXcCIo
&||>rd! Dd
(*
##!#
$ u
s6 AB . B
:B
B B8%B; 7B88B; ;C
c t t j dkD xr t j d dk( } t j t d z
j
d }t d| t }|dk( r t d y |t z dz
t z }t d | d
| d g }| r|}nt |t }t d|dz D ] }t | }|st d | d
n|D
cg c] \ }} }
|
s
|
|k\ s|| |
f } } }}
|j | | rxt | D cg c] \ }} } |
c} } }
| D
cg c] \ }} }
||
vs || |
f }} }}
|st d | d nft d| dt | dt | |r| st d | d n*|dz dk( st d| d| dt | d t dt | |sy g }t dz }t dt | | D ] }||||z }|D cg c] \ }} } |
}}} t d|dz d|t | z dt | d t# | }|D ]A \ }}}|j% |d \ }}}|xs |}|xs |} |j' ||| |xs d f C t) | } t d d t d t | d | d! |r9t+ |D cg c]
} | d" s | d" c} } t d#| d d$| d% dd l} | j/ t0 d&' }!|!j3 d(t4 f |!j7 |!j9 t d) y c c}
} }w c c} }w c c}
} }w c c} }w c c} w # t: $ r}"t d*|" Y d }"~"y d }"~"ww xY w)+Nr 1iG )daysz%Y-%m-%dz3-year cutoff: r z+ERROR: could not detect total article countzTotal: z articles, z pageszPage z: no data, stoppingz : all already in DB, stoppingr9 rP z new, cumulative z : all before cutoff, stoppingr /z so farzTotal filtered: z Details [-z]...rQ r>
z2==================================================z
Done. Total: z (saved: r r z Date range: z ~ rk r r a
INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
SELECT rowid, title, site_name,
CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
FROM gov_raw WHERE category = ?
zFTS index rebuiltzFTS rebuild note: ) rC sysargvr nowr strftimer r6 r" min MAX_PAGESr% rN extendr r r r' rD r sortedr r r r r r r r+ )# incrementalthree_years_ago total_count total_pages all_articles max_pagesrE rI utdfiltered_existingnew_onesr
batch_size batch_startbatch
batch_urls
detail_mapr0 r< rL r| detail_date detail_title final_title
final_datesavedr rH r r r4 s# r mainr s chh-!#: s(:K ||~ w(??II*UO//*+,#%Ka 9: )A-(:K'+ k+ f=>L Y/ aQ ' J % %v 01 2 -2QQ'!QaA ?@ '$r#h- 0A#lBSATU V %v :; < "9> '$q 2c,.?-@H I'J*
3|,-./ G q J QL 1:> O [z)AB',-GAq!a-
- k+a- +c%j*@)A3|CTBUUY Z['
3
$) O C6@nnSJ\6] 3L+|&/%K$0J NNCj,:L"M N O O
w E"XJ-G ~YugQ78g61!67 l58*Cb { 34
&w3
Y
g R 'BO( . 7 & $%%&sO N
N
N
;
N
N
$N
3
N%
N+
N+3A N0 0 O9O O__main__)#__doc__osr- r r$ r& bs4r urllib.parser r r concurrent.futuresr r environr' r r r! r r r" r( r r r r6 rN r} r r r r __name__ r r r s
( ?
**..bjjnn]DU&V
W $ L & *
3
& F0)f
.$L&^ zF r