uj d dl Z d dlZd dlZd dl Z d dlZd dlZ e j
dd ZdZdZe dZ ddiZ
ej d ej Z
d
Zd Zd Zd
Zedk( r e yy) N SEARCH_DBz/root/search.dbu 湖口县行政许可zhttps://www.hukou.gov.cnz/sthjj/fdzdgknr/hjjg/z
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36ze]*href=\"([^\"]+)\"[^>]*title=\"([^\"]*)\"[^>]*>.*?\s*([^<]+)c ` t j | t d }d|_ |j S )N )headerstimeoutzutf-8)requestsgetHEADERSencodingtext)urlresps /root/gov_crawler/crawl_hukou.pyfetchr ! s% <<Wb 9DDM 99 c t | }g }t j | D ] }|j d j
}|j d j
}|j d j
}|j d s%|j d r t |z n
t d| }|j |||d |S # t $ r}t d| g cY d}~S d}~ww xY w) u 获取列表页u ❌ 请求失败: N http/)titler
pub_date)
r ExceptionprintLIST_PATfinditergroupstrip
startswithBASE_URLappend)r
htmleitemsmhrefr r s r get_list_itemsr( ' s Sz
E
t
$ Jwwqz !
"771:##%v&&*ooc&:8d?8*AdV@TD
uTxHI
J L
$QC() s C C6 C1+C61C6c t | d }t j
d|t j }|r |j d j nd}t j d d|t j
}t j
d | }|r|j d | d < t j
d
| }|rV|j d j dd j dd j dd j }|r|| d< || d< t | d< | S # t $ r(}t d| d| d< t | d< | cY d}~S d}~ww xY w)u 获取详情r
u ❌ 详情页请求失败: contentsourceNz&]*id="content"[^>]*>(.*?)r z]*>.*?)flagsz+PubDate"\s*content="(\d{4}-\d{1,2}-\d{1,2})r z(.*?)u - 湖口县人民政府网站u - 湖口县人民政府u - 湖口县r ) r r r SITE_NAMEresearchDOTALLr r subreplace)itemr# r$ r& r+ ts r
get_detailr6 9 sQ T%[! ;T299MA$%aggaj 2Gff12wbiiPG @$GA 771:Z )40A
GGAJ ? D L LMgik l t t vD FH
I
O
O
Q DM DO DN K3
/s34 Y"X s D) ) E2 EEEc t t j dkD xr t j d dv } | rdnd}t dt d| d t j
}g }t
t }|s t d y t d t | d
|d d d
|d d d |j | | st dd D ] }t d| d}t
| }|s n|D cg c] }|d r |d dk\ s| }}t d|dz dt | d
|d d d
|d d dt | d |j | t | t | k s n |s t d y t dt | d t | D ]; \ } }
t |
| dz dz dk( s t d | dz d t | d = t j
|z
} t d t | d!| d"d# dd l } | j t }
|
j d$ |
j d% d&\ }}|D ]w } |
j d't |d( |d( |d) xs d*d d+ |j! d d* |d) xs d*d d+ |j! d,d* d-d.d*f
|
j" dkD r|dz
}n|dz
}y |
j' |
j d/t f |
j' |
j) t d0| d1| y c c}w # t$ $ r}|dz
}Y d }~d }~ww xY w)2Nr )1z
--incrementalu 增量u 全量u 🔍 z []u ⏭ 首页无数据u 📄 第1页: u 条 (r ~r ) index_z.htmlz2023-06u
📄 第u 页: u
) → 近3年u 条u ⏭ 无数据u
爬取详情页 (u 条)...
z ... r
)endu
📊 共获取 u 条，耗时 z.0fszPRAGMA journal_mode=WALzPRAGMA synchronous=NORMAL)r r zINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r
r r* i r+ activeu 环境影响评价zINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?u 💾 入库: 新增u , 跳过)lensysargvr r. timer( LIST_URLextendrange enumerater6 sqlite3connectr executer
total_changesr commitclose) incrementalmodet0 all_itemsr% pgr
itfilteredir4 elapsedrL dbokskipr$ s r mainr^ Y s chh-!#M 7M(MK"8D E)BtfA
&'
BI
8 $E
%& SZLeBi
.C-DAeAhzFZE[[\
]^
U 1+ B JfRD.C"3'E%*`r"Z.BzNV_D_`H` Jr!tfE#e* U59Z;P:QQRSXYZS[\fSgRhhuvy {C wD vE EH I
J X &8}s5z)
!#i.!1
9: Y' ?44
ER<1 HQqSE3y>"23 >?
iikB G c)n -^GC=
JK #BJJ()JJ*+HB JJ8 r%y"U)[&B-rvvj"/E[&B-rvvi/D 2B
!#a & IIKJJ M
IIKHHJ !"XdV
45o aV AID s% .L&L&A4L++ M4L>>M__main__)osr/ rE rG r getenvr r. r! rH r
compiler1 r r r( r6 r^ __name__ r r re s # " " BIIk#4 5 # %Z, - D
2::lII $ @N6b zF r