m }j<dZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z ejjdejjddZd Zed zZd Zd Zd dddddddZdZdZddiZdZdZdZdZdZdZedk(reyy) u crawl_wjq.py — 第六师五家渠市人民政府·通知公告 ============================================= JPaaS CMS，API JSON 分页 列表 API：/api-gateway/jpaas-publish-server/front/page/build/unit paramJson={"pageNo":N,"pageSize":12} 控制分页 返回 JSON { data: { html: "..." } } 详情：
正文 日期 用法: python3 crawl_wjq.py # 全量 python3 crawl_wjq.py 1 # 增量（只爬首页） N) BeautifulSoup)urljoin)datetime timedelta SEARCH_DB GOV_DB_PATHz/root/search.dbzhttps://www.wjq.gov.cnz7/api-gateway/jpaas-publish-server/front/page/build/unitu!第六师五家渠市人民政府u五家渠市通知公告 bulidstaticgXiRWWCDbJQMHYm3xJLgFGLcGu6Emt0FbNhQGT3B8ecolumnu当前栏目的列表nullWb3cNQs5RoDi7h7NnZA55) parseTypewebIdtplSetIdpageTypetagIdeditTypepageId z User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c td|y)Nz[wjq] )print)msgs /root/gov_crawler/crawl_wjq.pylogr:s F3%.c"|jSNstrip)titles r clean_titler#=s ;;=rctt}tj|tdd|d< t j t|td}|j}|j d r!|j d ij d sgdfS|d d }t|d }g}|jdD]}|jd} | r| j ds(| dj} | jdsd| j!dz} | jd} d} | r&| j ddxs| j#d} | jd} | r| j#dnd}t%t&| }|j)|t+| |f|t-|dkDfS#t$r}td|d|gdfcYd }~Sd }~wwxYw)z&>v&F5y < D } -D EmmD!? GGCLf y s#T**D FF3K EE'2&@!**4**@Evvf~044==t=,"8T* h E 2H=>'?* #e*q.  A  $TF"QC 015ys1G G) G$G)$G)c tj|td}d|_|j}t|d}d }|jd d }|r t|}d }|jd ddi}|rM|jdr<|dj} tjd| } | r| jd}|st d|d|fS||fS#t $r}t d|d|Yd}~yd}~wwxYw)z/Return (content_html, date_str) or (None, None)r))r+r,zutf-8z ERROR fetching r-N)NNr1r7div zw_content)class_metanamePubDate)attrscontentz(\d{4}-\d{2}-\d{2})z WARNING: no content for )r>r?rAencodingtextrBrrrDstrr!rematchgroup) urlrLr0rMrN content_htmlrTrPrWrawms r fetch_detailrgks||C"= yy } -DL ))E,) /C 3x H 99VFI#69 7D #9o##% HH+S 1 wwqzH  ( ./X~  !!3  uBqc *+s/C D&C<<Dc ddl}|s tdytdt|jt}|j }d}|D]-\}}}} |j dt ||||tf|dz }/|j|jtd |d t|d |S#t$r} td|d| Yd} ~ d} ~ wwxYw) zUpsert records into search.dbrNzNo items to savez Opening DB: zINSERT OR REPLACE INTO gov_raw (site_name, page_url, title, publish_date, content, category) VALUES (?, ?, ?, ?, ?, ?)r\z DB error for r-zSaved r5z records to DB) sqlite3rDB_PATHconnectcursorexecute SITE_NAMESOURCErBcommitcloserJ) rOriconncinsertedrcr"rPrdrMs r save_to_dbrus  ,wi ! ??7 #D AH.3.*UHl . II0    MH.& KKMJJL& !CJ<~67 O  . /#b, - - .s%B;; CCCc  ttjdkDxrtjddk(}tjt dz j d}td|g}tdtdzD]}t|\}}|std|dn|Dcgc]\}}}|s ||k\s|||f} }}}|j| |rtd t| d no| rGt| t|kr0td |D} | |krtd |D} | |krn&|d zdk(std|dt|d tdt|g} t|dD]k\} \}}}td| dt|d|dddt|\}}|xs|}|r| j!||||fW| j!|||dfmt#| }tddtdt| d|d| r9t%| Dcgc] }|ds |dc}}td|dd|d  ddl}|j)t*}|j-d!t.f|j1|j3td"tdycc}}}wcc}w#t4$r}td#|Yd}~5d}~wwxYw)$Nr\1iG)daysz%Y-%m-%dz3-year cutoff: zPage z: no data, stoppingzIncremental: page 1 only, z articlesc3.K|] \}}}|s |ywr.0_ds r zmain..s:Aq1: c3.K|] \}}}|s |ywrrzr{s rrzmain..s<71a!Q 200 THEN substr(content, 1, 200) ELSE content END FROM gov_raw WHERE category = ? zFTS index rebuiltzFTS rebuild note: )rJsysargvrnowrstrftimerrange MAX_PAGESrRextendminmax enumeratergrIrusortedrirkrjrmrorprqrB) incrementalthree_years_ago all_articlesrKarticleshas_datautr~filteredearliestnewestresultsircr"rPrd detail_date final_datesavedrdatesrirrrMs rmainrschh-!#: s(:K||~ w(??II*UO//*+,LaQ'N,T2( %v01 2 .6TT'!QqO?SQ1ITTH%  ,S]O9E F  H H 5:H::H/) /$xL0A/B)L M3N6 3|,-./G%.|Q%?9! !C as!C %&bs C 89$0$5! k ,H  NNC LA B NNC B7 89 w E"VH -G ~%7wa@Ag61!67 l58*Cb { 34 &w'  Y      6(wUP7 &   $%%&s72 K >K K = KK.AK"" L+K>>L__main__)__doc__osr`rr;r>bs4r urllib.parserrrenvironr?rjrHr@rnror:r=rrArr#rRrgrur__name__rzrrrs"  ( **..bjjnn]DU&V W $ N N / # $' $%    3 )!V"B#JIX zFr