用法:
python3 crawl_ahjx.py # 全量
python3 crawl_ahjx.py 1 # 增量（仅最新页）
N)
BeautifulSoup)urljoin)datetime timedelta)ThreadPoolExecutor as_completed SEARCH_DBz/root/search.dbu 泾县人民政府u 建设项目环境影响评价zhttps://www.ahjx.gov.cnz,/XxgkContent/showList/336/82020/page_{}.htmlzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36ztoken_verified=true)z
User-AgentCookie c t d| y )Nz[ahjx] )print)msgs /root/gov_crawler/crawl_ahjx.pylogr - s GC5/ c * t j | } t j |t d }d|_ |j }g }t j d|t j D ]z }|j d }|j d j }|j d
} |j d r|nt t | }
|j# |
|| f | |S # t $ r }t d| d| g cY d}~S d}~ww xY w) z
]*>.*?\s*(\d{4}-\d{2}-\d{2}) http)LIST_URLformatrequestsgetHEADERSencodingtext Exceptionr refinditerDOTALLgroupstrip
startswithr BASE_URLappend) page_nourlresphtmleitemsmhreftitledate_strfull_urls r fetch_list_pager9 1 s
//'
"C||C"=
yy
E
[[ /
2 wwqz
"771: ??624$8O
hx01
2 L% A3 /0 s /C, , D5D
D
Dc t j | t d }d|_ |j }t |d }|j d d
d i }|r$|j d
r|d
j nd}d}|j d d
di }|rK|j d
r:t j d|d
j } | r| j d }|j d }
|
r t |
nd} | st
d| d||fS | ||fS # t
$ r}t
d| d| Y d}~yd}~ww xY w)z=Return (content_html, date_str, title) or (None, None, None).r r r z ERROR r NNNNz html.parsermetaname ArticleTitle)attrscontent PubDatez(\d{4}-\d{2}-\d{2})r zoom)idz WARNING: no content for )r r! r" r# r$ r% r r findr* r& matchr) str) r/ r0 r1 r2 soup
meta_titler6 r7 meta_dater4 content_div content_htmls r fetch_detailrM M sE ||C"=
yy
} -D 6&.)ABJ-7JNN9 200 THEN substr(content, 1, 200) ELSE content END
FROM gov_raw WHERE category = ?z
FTS note: ) rZ r[ r\ r^ r` ra rb r% r )rZ rc r2 s r rebuild_ftsrh sf
w'
2 N
j s AA A4 A//A4c 2 t t j dkD xr t j d dk( } t j t d z
j
d }t d| g }d} t | }|s|dk( r t d y nt d| d t | d
|D cg c] \ }}}|s
||k\ s|||f }}}}|j | | r t d n/|r't | t | k rt d| d n|dz
}t d
t | |s t d y g } t dz }
t dt | |
D ] } || | |
z } | D
cg c] \ }}
}
|
}}}
t d| dz d| t | z dt | d t | }| D ]A \ }}}|j |d \ }}}|xs |}|xs |}| j ||||xs df C t | }t d| dt | | rAt! | D cg c]
}|d s |d c} }t d|r|d nd d|r|d nd t# t d y c c}}}w c c}
}w c c}w ) Nr 1iG )daysz%Y-%m-%dz3-year cutoff: zNo articles foundzPage r z articleszIncremental: page 1 onlyz': some articles before cutoff, stoppingzTotal articles to process: zNo articles within 3 yearsr r z Details [-/z]...r; rA zSaved: r z Date range: zN/Az ~ zDone!)lensysargvr nowr strftimer r9 extendrP rangerX r! r- rf sortedrh ) incrementalthree_years_ago all_articlespager3 utdfilteredrT
batch_size batch_startbatch_
batch_urls
detail_mapr/ r6 r7 rL detail_date detail_title final_title
final_datesavedrdatess r mainr s chh-!#: s(:K ||~ w(??II*UO//*+, L D
%qy'( eD6CJ O [z)AB',-GAq!a-
- k+a- +c%j*@)A3|CTBUUY Z['
3
$) O C6@nnSJ\6] 3L+|&/%K$0J NNCj,:L"M N O O
w E'%#g, ()g61!67 lu58%8E"ITY;Z [\MLW R2 . 7s$ 8 JJ
J9
J9
JJ__main__) __doc__osr& rp r bs4r urllib.parser r r concurrent.futuresr r getenvr\ r_ r` r, r r" rP PAGE_SIZEr r9 rM rX rf rh r __name__ r r r s "
( ?
"))K!2
3 . $ZE F3 $ 8 )F
. "?D zF r