正文 |
用法:
python3 crawl_tongbai.py # 全量
python3 crawl_tongbai.py 1 # 增量（只爬首页）
N)
BeautifulSoup)urljoin)datetime timedelta SEARCH_DB GOV_DB_PATHz/root/search.dbzhttps://www.tongbai.gov.cnz/apzrmzf/dtxx/tzgg/u 桐柏县人民政府u 桐柏县通知公告z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c t d| y )Nz
[tongbai] )print)msgs "/root/gov_crawler/crawl_tongbai.pylogr
) s Jse
c | j dd j dd j dd j S )z Remove site suffix from titleu# -通知公告-桐柏县人民政府 u
-通知公告u -桐柏县人民政府)replacestrip)titles r clean_titler - s9 ==> C K K
g&+EEG4r c t dt t j t t d } d| _ | j }t |d }g }|j dd }|s
t d |S |j d
}|s
t d |S |j d d
D ] }|j d }|r|j d s(|d j }d|v s|j t sU|j dd xs |j d } |j d }
|
r|
j d nd} |j || | f t dt | d |S )z7Fetch page 1 list, return [(url, title, date_str), ...]zFetching list: headerstimeoututf-8 html.parserdiv zfxxgk_zdgkc)class_z!ERROR: cannot find list containerulz ERROR: cannot find list
liF) recursiveahrefzmp.weixin.qq.comr r T)r bzFound z articles on page 1)r
LIST_URLrequestsgetHEADERSencodingtextr findfind_allr
startswithBASE_URLget_textappendlen) resphtmlsoupitemslist_divr r r" r# r r$ date_strs r
fetch_listr8 4 sX /($% <<'2 >DDM 99D } -D E yy~y6H /0 t B
*+ kk$%k0 . GGCLf
y %T__X-F gr"|d
j } | j dd } t j | d | }|s)t j d| }
|
r|
j# d }|sOt j d| }
|
r7|
j# d d|
j# d d|
j# d }|st
d| d|fS ||fS # t
$ r}t
d| d| Y d}~yd}~ww xY w# t $ r Y w xY w)z/Return (content_html, date_str) or (None, None)r r r z ERROR fetching : N)NNr r content)r idr metanamePubDate)attrs/-%Y-%m-%du" 时间[：:]\s*(\d{4}-\d{2}-\d{2}) z/(\d{4})/(\d{2})-(\d{2})/ z WARNING: no content for )r& r' r( r) r* Exceptionr
r r+ strr r r strptime
ValueErrorresearchgroup) urlr2 r3 er4 content_div content_htmlr7 r= rawms r fetch_detailrT [ s ||C"=
yy
} -D ))E) )BKL;' H 99VFI#69 7D #9o##%kk#s# c: .H
II;T B wwqzH II2C 8 ''!* Qqwwqzl!AGGAJ<@H ( ./X~ !!U uBqc *+. s) /E 0F F %E;;F FFc
ddl }| s t d yt dt |j t }|j }d}| D ]6 \ }}}} |j dt |t | ||t f |dz
}8 |j |j t d | d
t | d y# t $ r} t d| d| Y d} ~ d} ~ ww xY w) z Upsert records into search.dbr NzNo items to savez Opening DB: zINSERT OR REPLACE INTO gov_raw
(site_name, page_url, title, publish_date, content, category)
VALUES (?, ?, ?, ?, ?, ?)rD z DB error for r: zSaved rA z records to DB)
sqlite3r
DB_PATHconnectcursorexecute SITE_NAMEr SOURCErG commitcloser1 )
r5 rV conncinsertedrN r r7 rQ rO s
r
save_to_dbrb s ,wi ! ??7 #D
AH.3 .*UHl .
II0 &
MH .& KKMJJL&
!CJ<~67 . /#b, - - .s .C C' C""C'c t t j dkD xr t j d dk( } t j t d z
j
d }t d| t }|s t d y |D cg c] \ }}}|s
||k\ s|||f }}}}t dt | d t | | rt d
|}n|}t d t | d g }t |d D ]k \ } \ }
} } t d
| d t | d| d d d t |
\ }
}|xs | }|
r|j |
| ||
f W|j |
| |df m t | t dd t dt | d |r9t |D cg c]
}|d s |d c} }t d|d d|d t d y c c}}}w c c}w )NrD 1iG )daysrC z3-year cutoff: zNo articles found, exitingzAfter 3-year filter: rA z&Incremental mode: crawling page 1 onlyzFull mode: crawling z articles[z] 2 z...r
z2==================================================z
Done. Total: rE z Date range: r z ~ )r1 sysargvr nowr strftimer
r8 enumeraterT r0 rb sorted) incrementalthree_years_agoarticlesutdarticles_filtered
to_processresultsirN r r7 rQ detail_date
final_daterdatess r mainr~ s chh-!#: s(:K ||~ w(??II*UO//*+, |H () 3;YYwq!QaA DX!QYY
$5 6 7qX HI 45&
&
"3z?"39 => G%.z1%= 9! !C as!C
O$BuSbzl# 67$0$5! k ,H
NNC
LA B
NNC
B7 8 9 w "VH
-G ~Y/0g61!67 l58*Cb { 346(C Z> 7s G0 G0#G03
G7>G7__main__)__doc__osrK rj r& bs4r urllib.parser r r environr' rW r. r% r[ r\ r( r
r r8 rT rb r~ __name__ r r r s
(
**..bjjnn]DU&V
W ( + + # 3 4$N0"f"8J/d zF r