uj@ P d Z ddlZddlZddlZddlZddlmZmZmZ ddlZddl Z ddlZ e j e j j dZ
dZdZ ej dd Zd Z ej& ej( ed
z
j+ d Zd
diZd Zd Zd Zd Zd"dZ e dk( rodej< v Z e de
de rdnd de d ej Z! e e \ Z"Z# e d ej e!z
d d e d e" d!e# yy)#u
聊城市生态环境局-环保要闻 (sthjj.liaocheng.gov.cn)
==============================================
CMS: 政府门户 (layui 动态列表)
列表: /channel_t_189_13529/?page=N
详情: /channel_t_189_13529/doc_XXXXX.html
正文: div.content-text
JS 分页: ?page=N 参数可用
用法:
python3 crawl_liaocheng.py # 全量
python3 crawl_liaocheng.py --test # 测试 5 条
N)datetimetimezone timedeltau% 聊城市生态环境局-环保要闻z http://sthjj.liaocheng.gov.cnz2http://sthjj.liaocheng.gov.cn/channel_t_189_13529/ SEARCH_DBz/root/search.db iG )daysz%Y-%m-%dz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c t j | t dd }d|_ |j S # t
$ r}t
d| Y d }~y d }~ww xY w)N F)headerstimeoutverifyzutf-8u ⚠ fetch fail: )requestsgetHEADERSencodingtext Exceptionprint)urlres $/root/gov_crawler/crawl_liaocheng.pyfetchr sN LLgr% H
vv
%aS)*s /2 AAAc r g }t j d| t j D ] }|j d }t j d| }|st j d| }t j d| }|s\|j d j }t
|j dk\ r |j d j nd}|st
| dkD s|j ||j d r|nt |z |r|j d ndd
|S ) u) 提取列表条目 (url, title, pub_date)z (.*?)
z"href="([^"]+)"[^>]*title="([^"]*)"zhref="([^"]+)"[^>]*>([^<]+)z-(\d{4}-\d{2}-\d{2}) r http)titler pub_date) refinditerDOTALLgroupsearchstriplengroupsappend
startswithBASE_URL)htmlitemsmitemlinkdatehrefr s r
parse_listr3 ( s E
[[94
K wwqzyy>E99 ;TBDyyI4P ::a=&&(D-0 -?1-DDJJqM'')"EUa "#'??6#:4415
1
2 L c t | }|sddddS d}t j d|t j }|r |j d j }|sLt j d| }|r4t j dd|j d j }d}t j d|t j }|r|j d dd }|s)t j d
| }|r|j d }d}t j d |t j }|r|j d }|rt j d d|t j t j z
}t j dd|t j t j z
}t j dd| }|j }|||dS )u( 获取详情页 title, content, pub_dater )r contentr z3]*name="ArticleTitle"[^>]*content="([^"]*)"r z([^<]+)u
[-_—|].*z-<meta[^>]*name="PubDate"[^>]*content="([^"]+)N
z(\u65f6\u95f4\uff1a\s*(\d{4}-\d{2}-\d{2})z%class="content-text"[^>]*>(.*?)</div>z<script[^>]*>.*?</script>)flagsz<style[^>]*>.*?</style>z style="[^"]*")r r! r% Ir$ r& subr# )r r, r r. r r6 s r fetch_detailr; < s :D ;;
E
H$PRPTPTUA
" II' . FF="aggaj9??AE H
BD"$$OA771:cr? IIA4 H wwqzH G
:D"))LA''!*&&5r7"))TVTXTX.Y&&3R RTRVRVW&&*B8--/wH EEr4 c | s t d yddl}|j t }|j d |j d d\ }}| D ] } |j dt
|j
d xs d dd
|j
d d |j
d d |j
d
xs d dd d df |j dkD r|dz
}n|dz
} |j dt
f |j |j t d| d| d ||fS # t $ r}|dz
}Y d}~d}~ww xY w)u 批量写入 search.dbu ⎭ 无数据r r r NzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALzwINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, tags) VALUES (?,?,?,?,?,?,?)r r i r r6 r r7 u 环保要闻r zINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?u 💾 入库: 新增u , 跳过u , FTS已同步) r sqlite3connectr execute SITE_NAMEr
total_changesr commitclose)r- r> dbokfailitr s r to_dbrI c sT
+, #BJJ()JJ*+HB JJ) VVG_*DS1FF5"%FF9b)VVJ'-2s3.
!#a '. JJ \
IIKHHJ 3B4~dVKb
cd
t8O AID s BD'' D?0D::D?c g }t }t dt dz D ] }|dkD r t d| nt }t | }|s't | }|st
d| d nd}|D ]F }|d |v r |j |d |d r
|d t k r1|j | |dz
}H |d j dd } |d j dd }
t
d
| d t | d | d
|
d|
| r |dd }t
dt | d t
dt | d |syg } t |d D ] \ } }t |d }
|
d xs |d |d |
d |
d xs |j dd d}| j | t
d| dt | d |d d |d dd d t |
d d t j d! t! | \ }}||fS )"u 全量爬取r z?page=u ℹ 第u 页无数据，到达末页r r r ?u
📄 第u 页: u 条 (z ~ u ), 新增Nr u" 🧪 测试模式: 只处理前 u 条u
📊 列表汇总: u 条（近3年）r= r r6 r )r r r6 r z [/] ( z... (zB)g333333?)setrange MAX_PAGESLIST_URLr r3 r addCUTOFFr) r r' enumerater; timesleeprI )test all_items seen_urlspager r, r- newr/
first_date last_dateresultsidetailentryrF rG s r crawlre sg IIaQ ' v+/!8
&'Sz 4 OD6)_` a DE{i' MM$u+ &J D$4v$= T " 1HC 1X\\*c2
"IMM*c2
#D6#e* Xj\QTU^T__nornstu+v. bqM
PQTU^Q_P``ghi 5c)n5EEe
fg G Y*
4 d5k*G_5W
; i( z*Fdhhz2.F
u
A3aI'r%
*;)<AeGnSb>Q=RRWX[\bcl\mXnWooqrs
3
W~HB
t8Or4 __main__z--testu
📡 [rN u 测试模式u 全量u (最多u
页, 近3年))rZ u ⏱ 耗时: z.1fsu ✅ 新增: u ❌ 跳过: )F)$__doc__r! sysosrX r r r r urllib3disable_warnings
exceptionsInsecureRequestWarningrA r+ rT getenvr rS nowutcstrftimerV r r r3 r; rI re __name__argvrZ r t0rF rG r4 r <module>rw s\ 2 2 ++BB C 5
,
A BIIk#4 5
hll8<<(9%+@@
J
J:
V N O(%FN*X3j z sxx D N9+Rd(BP^'__mnwmx yO P Q B$ HB #IDIIKN3#7q
9: #B4'>tf
EF
r4