ujE d Z ddlZddlZddlZddlZddlZddlZddlZddl Z dZ
dZ dZ e dz Z
dadZ ej d d
Z e j$ Zd d iZd
Zd Zd Zd Zd Zd Zd Ze dk( r e yy)u
五原县人民政府 — 通知公告 爬虫
site: www.wuyuan.gov.cn
栏目: /tzgg/ (内蒙古五原县，非江西婺源)
列表: JS API POST /rmt/tmpl/api/site/{siteId}/channelid/{channelId}
body: {"currpage":N,"pagesize":10,"params":"..."} → {"errcode":0,"data":[{docno,title,inputTime(ms),...}]}
1069条, 10条/页, 107页
详情: http://www.wuyuan.gov.cn/tzgg/{docno}.html
详情结构: meta ArticleTitle/PubDate, div#content.hl_data=hlText(正文, 标题首p+表格)
Nu 五原县-通知公告zwww.wuyuan.gov.cnzhttp://www.wuyuan.gov.cnz^/rmt/tmpl/api/site/f4f8152cf0cd47c0a69f16be6bce423f/channelid/14bd7ef54c7f4a6f987f96e9e45d4c97 i- SEARCH_DBz/root/search.db
User-Agentz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36c t j j | t } t j j |dt
}|j
j dd S # t $ r Y y w xY w)N)headers timeoutcontextutf-8replaceerrors) urllibrequestRequestHEADERSurlopenCTXreaddecode Exception)urlreqresps &/root/gov_crawler/crawl_wuyuan_tzgg.pyhttp_getr si
..
g
6C~~%%c2s%Cyy{!!')!<< s AA. . A:9A:c v t j | j d }t j j | |t d dd } t j j |dt }t j |j j dd S # t $ r Y y w xY w)
Nr r z application/json;charset=utf-8)r z Content-Type)datar r r r
r )
jsondumpsencoder r r r r r loadsr r r )r payloadr r r s r http_post_jsonr% # s ::g % %g .D
..
4 l+8:
C~~%%c2s%Czz$))+,,WY,GHH s AB, , B87B8c t t dz } | syt j d| }|r|j d S dS )u 从列表页提取 params 签名/tzgg/Nzparams:\s*"([^"]+)" )r BASE_URLresearchgroup)htmlms r get_api_paramsr/ / s= Hx' (D
($/A1771:$$ c
| dt d}t t | }|r|j d dk7 rg S |j d xs g }g }|D ] }|j d }|j d xs dj }|j d xs d}|r,t j d
t j |d z nd} |s|j ||| t d | d
d |S )u0 调 API 拿一页列表 → [{docno, title, ts}]
)currpagepagesizeparamserrcoder r docnotitler inputTimez%Y-%m-%di r' z.html)r7 r8 publish_dater )
API_PARAMSr% API_URLgetstriptimestrftime localtimeappendr) )
pager$ jr itemsitr7 r8 tsdates
r fetch_list_pagerI 7 s R:FGw(A i A% 55= BD E w &B--/
VVK
%AGIt}}Z T )BCr LL $"6%6
Lr0 c t | }|syd}t j d| }|r |j d j }d}t j d| }|r"|j d j dd }d}t j d|t j
}|s%t j d|t j
}|r|j d j }t j d d|t j
t j z
}t j d d|t j
t j z
}|j }|xs d||d S )
u+ 获取详情：标题、正文HTML、日期Nr z+]*>(.*?)\s*]*>(.*?)
z]*>.*?)flagsz]*>.*?)r8 contentrH )r r* r+ r, r> DOTALLsubI)r r- r8 m_trH m_drL m_cs r fetch_detailrS M s, C=D E
))BD
IC
! ""$
D
))=t
DC
yy|!!#CR(G
))MtUWU^U^
_C ii=tRYYO
))A,$$&&&5r7"))TVTXTX.Y&&3R RTRVRVW--/[bWd CCr0 c t j t } | j d | j d | j | j
y # t j $ r Y 6w xY w)Naa CREATE TABLE IF NOT EXISTS gov_raw (
id INTEGER PRIMARY KEY AUTOINCREMENT,
site_name TEXT, source_url TEXT, page_url TEXT,
title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0,
summary TEXT, status TEXT, category TEXT DEFAULT '',
visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT ''
)zCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
title, content, site_name,
content='gov_raw', content_rowid='id', tokenize='unicode61'
))sqlite3connectr executeOperationalErrorcommitclose)conns r init_dbr\ i se ??9 %DLL
KKMJJL # #
s A A43A4c dd l } | j }|j ddd |j dt dd |j }t at s t d
y t d t t d t t d
z dz }|j r|n |j }|j rdnd| d}t dt d| d t dt d| d t t j t }|j# }d}d} t% d|dz D ] }
t' |
} | st d|
dd| d %d} d}
| D ]d }|j) d |d t f j+ }|r |
dz
}
| dz
} :t- |d }|rM|j/ d d }|j/ d xs |d }|j/ d!|j/ d"d }nd }|d }|j/ d"d }|s |
dz
}
| dz
} |j) d#||||d t0 t d$f |j2 dkD rt|j4 } t7 j8 d%d&| }t7 j8 d'd&| j; d d( xs |}|j) d)||t |f | dz
} |dz
}g |j? t d|
dd| d*t | d+| d,|
d- tA jB d. |jE t d/| d0| y # t< $ r Y w xY w# t< $ r Y w xY w)1Nr z--full
store_trueu 全量107页)actionhelpz--pagesr( u 指定页数)typedefaultr` u ✗ 无法获取 API paramsu API params 获取成功 (u 字符) r2 u 全量u 增量(u 页)u 🔍 u — u 模式u
总条数: u , 本次爬取: u 页z [p2d/u ] ✗ API无数据z6SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?r rL r r8 rH r: z|INSERT OR IGNORE INTO gov_raw (title, content, publish_date, page_url, source_url, site_name, status) VALUES (?,?,?,?,?,?,?) publishedz<[^>]+> z\s+ zSINSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?,?,?,?)u ] ✓ u 条 (新增u 跳过)g333333?u
📊 完成！新增: u | 跳过: )#argparseArgumentParser add_argumentint
parse_argsr/ r; printlenr\ TOTAL_COUNTfullpages SITE_NAMErU rV r cursorrangerI rW fetchonerS r= DOMAINrowcount lastrowidr* rN r> r rY r? sleeprZ )rj parserargs total_pagespages_to_crawlmoder[ cur new_total
skip_totalrC rE page_new page_skipitemexistsdetailrL r8 rH row_idplains r mainr | s
$
$
&F
NK
Q^L D !J
+, %c*o%6h
?@ I ?r)K$(II[4::Nyy8/?t&DD E)E$v
./ M+ &6~6Fc
JK G ??9 %D
++-CIJa !+, : % Db >"22DE F . D[[Hei( hj
Q a
!$u+.F **Y3
7+2*FG W
xx3Q a
- GT4; ;W <r s = < < < $ %
u
u
BIIk#4 5 $c$$&o
%,D8&YIv zF r0