ujL d dl Z d dlZd dlZd dl Z d dlZd dlZ e j
dd ZdZdZe dZ ddiZ
d Z dd
Z d Z
d Zed
k( r e yy) N SEARCH_DBz/root/search.dbu 临猗县公示公告zhttps://www.sxly.gov.cnz/zwzx/gggs/index.shtmlz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c ` t j | t d }d|_ |j S )N )headerstimeoutzutf-8)requestsgetHEADERSencodingtext)urlresps /root/gov_crawler/crawl_sxly.py
fetch_htmlr s% <<Wb 9DDM 99 c V t | }g }t j dt j
}|j
| D ] }|j d j }|j d j }|j d } t j dd|j d j }
|j d
s
t | }| d | } |j |
|| d |S # t $ r}t d| g cY d}~S d}~ww xY w)
u 提取列表页条目u ❌ 请求失败: Nz\[([^\]]+)\]]*target="_blank"[^>]*>(.*?) <[^>]+> http-)titler
pub_date)
r ExceptionprintrecompileDOTALLfinditergroupstripsub
startswithBASE_URLappend) r
known_yearhtmleitemspatternmdate_mdlinkyearr r s r get_list_itemsr2 " s #
Ejj O
G
d
# J''!*""$wwqz !wwqzz2qwwqz288:v& Zv&DV1WI&
uTxHIJ L)
$QC() s D D(D# D(#D(c " t | d }t j
d|t j }|r |j d j nd}t j
d |t j }|r;t j d
d|j d j }|r|| d < t j
d | }|r?|j d j j d
d j nt }|xs t }t j
d| }|r|j d | d< || d< t | d< | S # t $ r(}t d| d| d< t | d< | cY d}~S d}~ww xY w)u 获取详情r
u ❌ 详情页请求失败: r contentsourceNz*]*>(.*?)\s*r z%]*class="green"[^>]*>(.*?)r r z%([^<]+),r u& 发布日期：(\d{4}-\d{1,2}-\d{1,2})r ) r r r SITE_NAMEr searchr! r# r$ r% split)itemr* r+ r. r4 detail_titler5 s r
get_detailr< = s[ $u+& ?ryyQA$%aggaj 2G :D"))LAvvj"aggaj9??A (DM :DAA9:QWWQZ
%
%c
*1
-
3
3
5 F
yF ;TBA 771:Z DO DN K;
/s34 Y"X s E F& F F Fc t t j dkD xr t j d dv } | rdnd}t dt d| d t j
}g }| rdnd}t
d|dz D ] }|dk( rt nt j d d
| d }t | }|s#|dkD rt d | d ndt d
nW|D cg c] }|d r |d dk\ s| } }t d | dt | dt | d |j | |s t d y t dt | d t | D ]; \ }
} t | |
dz dz dk( s t d|
dz dt | d = t j
|z
} t d t | d | d d dd l
}
|
j t }|j! d |j! d! d"\ }}|D ]w } |j! d#t |d$ |d$ |d% xs d&d d' |j# dd& |d% xs d&d d' |j# d(d& d)d*d&f
|j$ dkD r|dz
}n|dz
}y |j) |j! d+t f |j) |j+ t d,| d-| y c c}w # t& $ r}|dz
}Y d }~d }~ww xY w).Nr )1z
--incrementalu 增量u 全量u 🔍 z [] z.shtml_u
📄 第u 页: 无数据，结束u 📄 第1页: 无数据r z2023-06u 页: u 条 (近3年u 条)u ⏭ 无数据u
爬取详情页 (u 条)...
r z ... /
)endu
📊 共获取 u 条，耗时 z.0fszPRAGMA journal_mode=WALzPRAGMA synchronous=NORMAL)r r zINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r
r r i r4 activeu 公示公告zINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?u 💾 入库: 新增u , 跳过)lensysargvr r7 timerangeLIST_URLrstripr2 extend enumerater< sqlite3connectr executer
total_changesr commitclose) incrementalmodet0 all_items max_pagespager
r, itfilteredir: elapsedrQ dbokskipr+ s r mainrd a s" chh-!#M 7M(MK"8D E)BtfA
&'
BI bIaQ ' # !)hHOOH,E+FavV)T s#ax
4&(?@A - / !&\2bn:R[@[B\\
4&c%j\ c(m_DQR" #
!#i.!1
9: Y' ?44
ER<1 HQqSE3y>"23 >?
iikB G c)n -^GC=
JK #BJJ()JJ*+HB JJ8 r%y"U)[&B-rvvj"/E[&B-rvvi/D nb
!#a & IIKJJ M
IIKHHJ !"XdV
45k ]R AID s% K"+K"A4K'' K?0K::K?__main__)N)osr rI rK r getenvr r7 r' rM r
r r2 r< rd __name__ r r rj s} # " " BIIk#4 5 # $Z- . D
6!HH6V zF r