正文 |
日期
用法:
python3 crawl_guanxian.py # 全量
python3 crawl_guanxian.py 1 # 增量（只爬首页）
N)
BeautifulSoup)urljoin)datetime timedelta)ThreadPoolExecutor as_completed SEARCH_DB GOV_DB_PATHz/root/search.dbzhttp://www.guanxian.gov.cnz/channel_x_0.0_6351/u 冠县人民政府u 冠县通知公告z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 i c t d| y )Nz [guanxian] )print)msgs #/root/gov_crawler/crawl_guanxian.pylogr . s Ku
c " | j S )N)strip)titles r clean_titler 1 s ;;=r c ^ | dkD r t d| nt } t j |t d }d|_ |j
}g }t j d
|t j D ] }|j d }|j d }t j d d
| j } t j dd| j } d
}
t j d| } | r| j d }
t j dd
| j } t | } |j! d r|}
nt# t$ | }
|j' |
| |
f |t) | d kD fS # t $ r }t d| d| g dfcY d }~S d }~ww xY w)z$Return [(url, title, date_str), ...] z?page= headerstimeoututf-8z
ERROR page : FNz3
]*>.*?r z<[^>]+> z\s+ (\d{4}-\d{2}-\d{2})z\d{4}-\d{2}-\d{2}http)LIST_URLrequestsgetHEADERSencodingtext Exceptionr refinditerDOTALLgroupsubr searchr
startswithr BASE_URLappendlen)pageurlresphtmleitemsmhrefa_htmlr' date_strdr full_urls r fetch_list_pager? 4 s '+axXJfTF
#XC||C"=
yy
E
[[> bii 2 wwqzvvj"f-335vvfc4(..0 II,d 3 wwqzH +R6<<>E" ??6 "H x.H
hx01326 #e*q. C mD6A3 '(5ys /F F,
F'!F,'F,c 0 t j | t d }d|_ |j }t |d }d }|j d
d }|s|j d
d
}|r t | }d }|j dddi }|rK|j d r:t j d|d j } | r| j d }|st
d| d|fS ||fS # t
$ r}t
d| d| Y d}~yd}~ww xY w)z/Return (content_html, date_str) or (None, None)r r r z ERROR r NNNz html.parserr divc | xr
d| v xr d| v S )Nmainovh )cs r z fetch_detail..l s A,L&A+,L%1* r )class_rD metanamePubDate)attrscontentr r z WARNING: no content for )r# r$ r% r& r' r( r r findstrr) matchr r, )
r4 r5 r6 r7 soup content_htmlrB r< rJ r9 s
r fetch_detailrT ^ s ||C"=
yy
} -D L
))E"L)
MC iifi-
3x H 99VFI#69 7D # HH+T)_-B-B-D E wwqzH ( ./X~ !!5 hse2aS !"s /C1 1 D:DDc T i }t t 5 }| D ci c] }|j t | | }}t | D ] }|| } |j ||< ddd |S c c}w # t $ r }t d| d| d||< Y d}~Rd}~ww xY w# 1 sw Y |S xY w)z'Fetch multiple detail pages in parallel) max_workersz ERROR in thread for r rA N)r MAX_WORKERSsubmitrT r resultr( r )urlsresultsexr4 fut_mapfutr7 s r fetch_detail_batchr_ s G 4 ,@DE299\3/4EE ( ,C#,C
, "zz| ,, N F
,,SEA378 +
,
, NsE B A,B A1 B ,B 1 B:BB BB B'c
T dd l }| sy|j t }|j }d}| D ]- \ }}}} |j dt
||||t f |dz
}/ |j |j |S # t $ r} t d| d| Y d } ~ qd } ~ ww xY w)Nr zINSERT OR REPLACE INTO gov_raw
(site_name, page_url, title, publish_date, content, category)
VALUES (?, ?, ?, ?, ?, ?)r z DB error for r ) sqlite3connectDB_PATHcursorexecute SITE_NAMESOURCEr( r commitclose)
r8 ra connrG insertedr4 r r< rS r7 s
r
save_to_dbrl s ??7 #D
AH.3
.*UHl .
II0 C,G
MH
. KKMJJL O . /#b, - - .s %B B' B""B'c | t t j dkD xr t j d dk( } t j t d z
j
d }t d| g }t dt dz D ] }t | \ }}|st d| d n|D cg c] \ }}}|s
||k\ s|||f } }}}|j | | rt d t | d
ni|rA| s?t |D
cg c] \ }
}
}|s
|
c}}
xs d g } | r| |k rt d| d n&|d
z dk( st d| dt | d t dt | g } t dz }
t dt | |
D ] }||||
z }|D
cg c] \ }}
}
|
}}}
t d|dz d|t | z dt | d t | }|D ]: \ }}}|j |d \ }}|xs |}| j! ||||xs d f < t# | }t dd t dt | d | d | r9t% | D cg c]
}|d s |d c} }t d |d d |d! dd l}|j) t* } | j- d"t. f | j1 | j3 t d# y c c}}}w c c}}
w c c}
}w c c}w # t4 $ r} t d$| Y d } ~ y d } ~ ww xY w)%Nr 1iG )daysz%Y-%m-%dz3-year cutoff: zPage z: no data, stoppingzIncremental: page 1, z articlesr z : all before cutoff, stopping2 r z Page r z articles so farzTotal articles: z Fetching details [-/z]...rA
z2==================================================z
Done. Total: z articles (saved: ) z Date range: z ~ a
INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
SELECT rowid, title, site_name,
CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
FROM gov_raw WHERE category = ?
zFTS index rebuiltzFTS rebuild note: )r2 sysargvr nowr strftimer range MAX_PAGESr? extendmaxrW r_ r$ r1 rl sortedra rb rc re rg rh ri r( ) incrementalthree_years_ago all_articlesr3 articleshas_datautr= filtered_oldest_on_pager[
batch_size batch_startbatch
batch_urls
detail_mapr4 r r< rS detail_date
final_datesavedrdatesra rj r7 s r rD rD sh chh-!#: s(:K ||~ w(??II*UO//*+, LaQ ' G ,T 2( %v 01 2 .6TT'!QqO?SQ1ITTH% 'H
i@ A H 8!A1aq!!A!IbT JN .?"BeD6!>?@ "9> '$r#l"3!44DE F-G0
3|,-./ G q J QL 1:>
I [z)AB',-GAq!a-
- ";q=/;s5z3I2J!CP\L]K^^b cd'
3
$) I C(2sL(I %L+$0J NNC
L4FBG H I
I
w E "VH
-G ~%7wa@Ag61!67 l58*Cb { 34
&w'
Y
q U "B ." 7 & $%%&sI 2 L
>L
L
L L -
L5
L L&AL L;#L66L;__main__) __doc__osr) rx r# bs4r urllib.parser r r concurrent.futuresr r environr$ rc r0 r" rf rg r% rW r} r r r? rT r_ rl rD __name__rF r r r s
( ?
**..bjjnn]DU&V
W ( , , 3 (!T "D ,H&V zF r