正文
/
/
用法:
python3 crawl_chuanshan.py # 全量
python3 crawl_chuanshan.py 1 # 增量（仅爬第一页）
N)
BeautifulSoup)urljoin)datetime timedelta)ThreadPoolExecutor as_completed SEARCH_DB GOV_DB_PATHz/root/search.dbz https://www.chuanshan.gov.cnz4/gxq/zwgk/list/db49e9ea4397a019d43a5057ef8cbe9e.htmlu 船山区人民政府u 高新区动态z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 d c t d| y )Nz [chuanshan] )print)msgs $/root/gov_crawler/crawl_chuanshan.pylogr * s L
c X | dkD rt t d| n
t t } t j |t d }d|_ |j }g }t j d
|t j D ] }|j d j }|j d }|j d sEt t | } d
}
t j dt j" | d| } t j dt j" | d|t j } | r| j d }
|j% | ||
f t j d| d u}
||
fS # t $ r }t d| d| g dfcY d }~S d }~ww xY w)z$Return [(url, title, date_str), ...] z?page= headerstimeoututf-8z
ERROR page : FNz=
]*> z/gxq/zwgk/show/ zre:\s*z'["\']?>\s*(\d{4}-\d{2}-\d{2})z]*>.*?href="z1"[^>]*>.*?\s*(\d{4}-\d{2}-\d{2})u page=(\d+)">下一页)BASE_URLLIST_URLrequestsgetHEADERSencodingtext Exceptionr refinditerDOTALLgroupstrip
startswithr searchescapeappend)pageurlresphtmleitemsmtitlehreffull_urldate_strdli_matchhas_nexts r fetch_list_pager< . s 15XJxjtf
-(H:?VC||C"=
yy
E
[[H bii 2
"wwqz 01 8T* II $ 00]^`d e 99!"))D/!22i j "))
~~a(H
hx01/22 yy5tz fetch_detail..p s+ A,q2CZXY[^M_ehememnoep2q r )class_z WARNING: no content for )r r r! r" r# r$ r r findr) r% matchr( rH ) r/ r0 r1 r2 soup
meta_titler5 r8 meta_dater4
content_el content_htmls r fetch_detailrT W sE ||C"=
yy
} -D 6&.)ABJ-7JNN9#D99D>c 2 i }t t 5 }| D ci c] }|j t | | }}t | D ] }|| } |j ||< d d d |S c c}w # t $ r}d||< Y d }~Ad }~ww xY w# 1 sw Y |S xY w)N) max_workersr> )r MAX_WORKERSsubmitrT r resultr$ )urlsresultsexr/ fut_mapfutr2 s r fetch_detail_batchr_ z s G 4 2@DE299\3/4EE ( 2C#,C
2 "zz| 22 N F
2 1
2
2 NsE B A,B A1 B ,B 1 B :B?B B B Bc
T ddl }| sy|j t }|j }d}| D ]- \ }}}} |j dt
||||t f |dz
}/ |j |j |S # t $ r} t d| d| Y d} ~ qd} ~ ww xY w)z3items: list of (url, title, date_str, content_html)r NzINSERT OR REPLACE INTO gov_raw
(site_name, page_url, title, publish_date, content, category)
VALUES (?, ?, ?, ?, ?, ?)r z DB error for r ) sqlite3connectDB_PATHcursorexecute SITE_NAMESOURCEr$ r commitclose)
r3 ra connrJ insertedr/ r5 r8 rS r2 s
r
save_to_dbrl s ??7 #D
AH.3
.*UHl .
II0 C,G
MH
. KKMJJL O . /#b, - - .s %B B' B""B'c F t t j dkD xr t j d dk( } t j t d z
j
d }t d| g }t dt dz D ] }t | \ }}|st d| d n|D cg c] \ }}}|s
||k\ s|||f } }}}|j | | rt d t | d
nM|r| st d| d n8|d z d
k( rt d| dt | d |rt d| d n t dt | g }
t dz } t d
t | | D ] } || | | z }
|
D cg c] \ }}}|
}}}t d| dz d| t |
z dt | d t | }|
D ]A \ }}}|j |d \ }}}|xs |}|xs |}|
j ||||xs df C t! |
}t dd t d t |
d | d |
r9t# |
D cg c]
}|d s |d c} }t d |d
d!|d" d
d l} | j' t( } | j+ d#t, f | j/ | j1 t d$ y c c}}}w c c}}w c c}w # t2 $ r} t d%| Y d } ~ y d } ~ ww xY w)&Nr 1iG )daysz%Y-%m-%dz3-year cutoff: zPage z: no data, stoppingzIncremental: page 1, z articlesz : all before cutoff, stopping
r z Page r z articles so farz: no next page, stoppingzTotal articles: z Details [-/z]...r> r
z2==================================================z
Done. Total: z articles (saved: )r z Date range: z ~ a
INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
SELECT rowid, title, site_name,
CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
FROM gov_raw WHERE category = ?
zFTS index rebuiltzFTS rebuild note: )lensysargvr nowr strftimer range MAX_PAGESr< extendrW r_ r r- rl sortedra rb rc re rg rh ri r$ ) incrementalthree_years_ago all_articlesr. articlesr; utr9 filteredr[
batch_size batch_startbatch_
batch_urls
detail_mapr/ r5 r8 rS detail_date detail_title final_title
final_datesavedrdatesra rj r2 s r mainr sO chh-!#: s(:K ||~ w(??II*UO//*+, LaQ ' ,T 2( %v 01 2 -5TT'!QqO?SQ1ITTH% 'H
i@ A H %v :; < "9> '$r#l"3!44DE F %v 56 7 -0
3|,-./ G q J QL 1:> O [z)AB',-GAq!a-
- k+a- +c%j*@)A3|CTBUUY Z['
3
$) O C6@nnSJ\6] 3L+|&/%K$0J NNCj,:L"M N O O
w E"XJ-G ~%7wa@Ag61!67 l58*Cb { 34
&w'
Y
o U2 . 7 & $%%&s= 2 K-
>K-
K-
K4
K:+K:AK? ? L LL __main__) __doc__osr% rx r bs4r urllib.parser r r concurrent.futuresr r environr rc r r rf rg r! rW r} r r< rT r_ rl r __name__ r r r s
( ?
**..bjjnn]DU&V
W ) A # 3 &R )F
0F&R zF r