正文
/
用法:
python3 crawl_zibo_gxqhbj.py # 全量（近3年）
python3 crawl_zibo_gxqhbj.py 1 # 增量（仅最新页）
N)
BeautifulSoup)urljoin)datetime timedelta)ThreadPoolExecutor as_completed SEARCH_DB GOV_DB_PATHz/root/search.dbz http://www.china-zibo.gov.cnz6/gongkai/site_gxqhbj/channel_6076a2721887a377796bb7cf/u! 淄博高新技术产业开发区u 高新区环保局-环评信息
z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 c t d| y )Nz[zibo_gxqhbj] )print)msgs &/root/gov_crawler/crawl_zibo_gxqhbj.pylogr 1 s N3%
! c 6 t t dt dt t j } t d D ] } t
j | t d }d|_ |j }t j d | }|r t |j! d c S t d
|dz d t j d
|d
z z y# t $ r9}t d|dz d | t j d
|d
z z Y d }~d }~ww xY w)z/Fetch first page to detect total article count.z%?open=fdzdgknr&nr_page=1&nr_per_page=&_t= headerstimeoututf-8z ERROR fetching first page (try ): Nz(fdzdgknr_total\s*=\s*parseInt\("(\d+)"\)z total not found (try ), retryingr )BASE_URL LIST_PATHPER_PAGEinttimerangerequestsgetHEADERSencodingtext Exceptionr sleepresearchgroup)urlattemptresphtmlems r fetch_total_countr6 5 s
Jyk!FxjPTUXY]YbYbYdUeTf
gC8
$ <<WbAD #DM99D
IIA4 H qwwqz? " %gai[ <=
1w{?#
$
1'!)CsC D JJq7Q; ' s /C D /DDc t t d| dt dt t j }t d D ] } t
j |t d }d|_ |j }t |d }|j d }|r n4t d| d|d
z d t j d |d z z g S |j d }g } t! | D ] \ }
} | j# d } | r| j d s+| d }
| j d xs | j% d xs d}|j' }|
t) | k r||
j% d nd}t+ t |
}| j- |||f | S # t $ r=}t d| d |d
z d | t j d |d z z Y d
}~d
}~ww xY w)z$Return [(url, title, date_str), ...]z?open=fdzdgknr&nr_page=z
&nr_per_page=r r r r r z
ERROR page z (try r r r N html.parserztd.zfxxgk-list-title Page z empty rows (try r ztd.zfxxgk-list-timeahreftitleT)strip )r r! r" r# r$ r% r& r' r( r) r* r+ r r, r select enumeratefindget_textr= lenr append)pager0 r1 r2 r3 r4 souprowsdatesitemsitdr: r; r< date_strfull_urls r fetch_list_pagerN I s
Jyk!8mH:UYZ]^b^g^g^iZjYk
lC8 <<WbAD #DM99D
T=1{{12 gdV,WQYK{ CD
1w{?# KK- .E E4 22 GGCLf
yg>!**4*"8>B
45E
N58$$4$08T*
hx01 2 L7 -vVGAI;c!= > JJq7Q; ' s /F G$'2G G$c > t j | t d }d|_ |j }t |d }|j d d
d i }|r$|j d
r|d
j nd}d}|j d d
di }|rK|j d
r:t j d|d
j } | r| j d }|j dd }
|
s|j dd }
|
rdD ]( } |
j | D ] } | j * |
j t j d D ]( } | j# g d }
|
s|
j * |
j dd }|r|d }
t% |
}nd}|s,d D ]' |j dfd }
|
s t% |
} n |st
d | d||fS |||fS # t
$ r}t
d| d| Y d}~yd}~ww xY w) z=Return (content_html, date_str, title) or (None, None, None).r r r z ERROR : NNNNr8 metaname ArticleTitle)attrscontentr> PubDatez(\d{4}-\d{2}-\d{2})r divc T | xr% dt | t r| ndj | v S Ndetails-content
isinstancestrjoincs r
z fetch_detail.. s- A4y:KU_`acfUgPQmpmumuvwmx:y r )class_r[ )id) z.sk-breadcrumbztable.details-infozh1.details-titlez.share-details-rowz.sharez.details-erweimaz
.related-itemzli.details-info-fontsizezli.details-change-fontsizeu 打印)string)r: lispanbuttonc T | xr% dt | t r| ndj | v S rZ r] ra s r rc z fetch_detail.. sE ! CHHYcmnoqtcu^_{~ |D |D EF |G IH r )z zfxxgk-textnr_rightrV articlec V | xr% t | t r| ndj | v S )Nr\ r] )rb clss r rc z fetch_detail.. s+ A>"?@D !
!))% 9H) I
$R.K;' D C ))E2s)tK "; / ( ./Xu$$ 5 ((m hse2aS !" s /G8 8 H HH c 2 i }t t 5 }| D ci c] }|j t | | }}t | D ] }|| } |j ||< d d d |S c c}w # t $ r}d||< Y d }~Ad }~ww xY w# 1 sw Y |S xY w)N) max_workersrQ )r MAX_WORKERSsubmitr} r resultr+ )urlsresultsexr0 fut_mapfutr4 s r fetch_detail_batchr s G 4 2@DE299\3/4EE ( 2C#,C
2 "zz| 22 N F
2 1
2
2 NsE B A,B A1 B ,B 1 B :B?B B B Bc
T ddl }| sy|j t }|j }d}| D ]- \ }}}} |j dt
||||t f |dz
}/ |j |j |S # t $ r} t d| d| Y d} ~ qd} ~ ww xY w)z3items: list of (url, title, date_str, content_html)r NzINSERT OR REPLACE INTO gov_raw
(site_name, page_url, title, publish_date, content, category)
VALUES (?, ?, ?, ?, ?, ?)r z DB error for rP ) sqlite3connectDB_PATHcursorexecute SITE_NAMESOURCEr+ r commitclose)
rI r connrb insertedr0 r< rL r| r4 s
r
save_to_dbr s ??7 #D
AH.3
.*UHl .
II0 C,G
MH
. KKMJJL O . /#b, - - .s %B B' B""B'c ddl }| s
t S |j t } dj dt | z }|j
d| d| j }|D ch c] }|d c}|j S c c}w # t $ r t cY |j S w xY w# |j w xY w)u2 返回已入库的 URL 集合 (增量停止条件)r N,?z0SELECT page_url FROM gov_raw WHERE page_url IN ())
r setr r r` rC r fetchallr r+ )r r r phrG rs r check_existingr s u ??7 #D
XXcCIo
&||>rd! Dd
(*
##!#
$ u
s6 AB , B 8B B B6#B9 5B66B9 9C c t t j dkD xr t j d dk( } t j t d z
j
d }t d| t }|dk( r t d y |t z dz
t z }t d | d
| d t d g }| r|}nt |t }t d|dz D ] }t | }|st d
| d n|D
cg c] \ }} }
|
s
|
|k\ s|| |
f } } }}
|j | | rxt | D cg c] \ }} } |
c} } }
| D
cg c] \ }} }
||
vs || |
f }} }}
|st d
| d nit d| dt | dt | |r| st d
| d| d n*|dz dk( st d| d| dt | d |s t d y t dt | g }t dz }t dt | | D ] }||||z }|D cg c] \ }} } |
}}} t d|dz d |t | z dt | d t# | }|D ]A \ }}}|j% |d \ }}}|xs |}|xs |} |j' ||| |xs d f C t) | } t d d! t d"t | d#| d$ |r9t+ |D cg c]
} | d% s | d% c} } t d&| d d'| d( dd l} | j/ t0 }!|!j3 d)t4 f |!j7 |!j9 t d* y c c}
} }w c c} }w c c}
} }w c c} }w c c} w # t: $ r}"t d+|" Y d }"~"y d }"~"ww xY w),Nr 1iG )daysz%Y-%m-%dz3-year cutoff: r z+ERROR: could not detect total article countzTotal: z articles, z pages (z/page)zPage z: no data, stoppingz : all already in DB, stoppingr9 rP z new, cumulative z: all before cutoff (z ), stoppingr /z articles so farz No articles after date filterzTotal filtered: z Details [-z]...rQ r>
z2==================================================z
Done. Total: z articles (saved: r r z Date range: z ~ rk a
INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
SELECT rowid, title, site_name,
CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
FROM gov_raw WHERE category = ?
zFTS index rebuiltzFTS rebuild note: ) rC sysargvr nowr strftimer r6 r" min MAX_PAGESr% rN extendr r r r' rD r sortedr r r r r r r r+ )# incrementalthree_years_ago total_count total_pages all_articles max_pagesrE rI utdfiltered_existingnew_onesr
batch_size batch_startbatch
batch_urls
detail_mapr0 r< rL r| detail_date detail_title final_title
final_datesavedr rH r r r4 s# r mainr s* chh-!#: s(:K ||~ w(??II*UO//*+, $%Ka 9: )A-(:K'+ k+ hxjOPL Y/ aQ ' S % %v 01 2 .3QQ'!QaA ?@ '$r#h- 0A#lBSATU V %v 2?2C;O P "9> '$q 2c,.?-@@PQ R5S8 +,
3|,-./ G q J QL 1:> O [z)AB',-GAq!a-
- k+a- +c%j*@)A3|CTBUUY Z['
3
$) O C6@nnSJ\6] 3L+|&/%K$0J NNCj,:L"M N O O
w E"XJ-G ~%7wa@Ag61!67 l58*Cb { 34
&w'
Y
} R
'BO4 . 7 & $%%&sO N$
N$
N$
N+
N1
+N1
N8
N>"N>AO O$ O O$__main__)#__doc__osr- r r$ r& bs4r urllib.parser r r concurrent.futuresr r environr' r r r! r r r" r( r r r r6 rN r} r r r r __name__ r r r s
( ?
**..bjjnn]DU&V
W * D / *
3 "
(#L<)~
0$Z&z zF r