uj8 d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl
m Z m Z ddl
Z
ddlmZmZ ddlZ ej$ d dZdZd Zd
Zd ad Zd
ddddddddddddd
ZdZd Zd d d d Z d! Z d6d"Z d7d#Z d$ Z d7d%Z!d& Z"d' Z#d( Z$d8d*Z%e&d+k( rk ejN d,- Z(e(jS d.e*d)d/0 e(jS d1d2d34 e(jW Z, e%e,jZ e,j\ 5 yy)9uW
霍邱县人民政府 - 行政审批信息公开 爬虫
CMS: 六安政府网站群 (luan_govc)，信息公开目录系统
列表API: GET /site/label/8888 (返回HTML片段)
params: labelName=publicInfoList, siteId=6786971, pageSize=15,
pageIndex=N, organId=6600621, type=4, catId=7365565,
action=list, file=/c3/huoqiu/publicInfoList_newest
详情页: /public/6600621/{id}.html
标题: meta[ArticleTitle]
日期: meta[PubDate]
正文: div.gkwz_contnet.j-fontContent (段落/表格/附件链接)
日跑: --max-pages 1 (增量取最新页)
N)datetime)urljoin urlencode)
BeautifulSoupNavigableStringignorezhttps://www.huoqiu.gov.cnu 霍邱县人民政府u 行政审批u 安徽省z/mnt/data/search.dbz)https://www.huoqiu.gov.cn/site/label/8888publicInfoList678697115listtruez
yyyy-MM-dd50660062147365565 u 暂无相关信息z /c3/huoqiu/publicInfoList_newest)
labelNamesiteIdpageSizeactionisDate
dateFormatlengthorganIdtypecatIdcIdresultfile zuMozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-Languagec h t dt j j d d| y )N[z%H:%M:%S] )printr nowstrftime)msgs &/root/gov_crawler/crawl_huoqiu_tzgg.pylogr+ 9 s) Ahlln%%j1
2"SE
:; c t | D ] } |xs t j }t t }t
d|d< |j
| ||d }|j dk( rd|_ |j c S t d|j d|d z d
| d t j d y# t $ r7}t d
| d|d z d
| d t j d Y d}~d}~ww xY w)u 获取页面内容z=/public/column/6600621?type=4&catId=7365565&action=list&nav=3Referer )paramsheaderstimeout zutf-8z HTTP u ，重试 ( /) u 请求异常: N)rangerequestsSessiondictHEADERSBASE_URLget status_codeencodingtextr+ timesleep Exception) urlr0 sessionretriesisr1 respes r* get_pagerM = s
7^
-8++-A7mG$,:-j!kGI 55VWb5ID 3& '
yy '$**+;qse1WIQG H JJqM
"1#[1QwiqA B JJqMM s A)B6<7B66 C6?-C11C6c ~ t t }t | |d< t dt | }t || S )u 获取列表页 pageIndex?rG )r<
API_PARAMSstrAPI_URLr rM )page_numrG r0 rF s r*
get_list_pagerV P s=
*
F h-F;
IQy()
*C C ))r, c g }t | d }|j dd }|s
t d |S |j dd D ] }|j d d
}|r|j d s+|d j }|j
d st t | }d
}|j D ]0 }|j t | j } | s,|| z
}2 t j dd| j }|s|j d }|j dd }
|
r|
j d nd
} |s|s
|j ||| d ! |S )u' 解析列表返回 [{url, title, date}] html.parserul
xxgk_nav_list)class_u ⚠️ 未找到 xxgk_nav_listliF) recursiveatitlehrefhttpr z\s+ Tstripspandate)rF r_ pub_date)r findr+ find_allr? rd
startswithr r> childrennamerS resubget_textappend) htmlitemssouprY r\ r^ r` r_ childt date_spanrg s r*
parse_listrw X sX E } -D
4 0B
./ kk$%k0 N GGCG (f
y v&8T*D ZZ Ezz!J$$&QJE
vsE*002JJTJ*EGGF6G2 5>9%%D%1B T LL8L M1N4 Lr, c \ t | | }|syt |d }d}|j dddi }|r$|j d r|d j }|s%|j d
}|r|j d }d}|j ddd
i }|r$|j d r|d j }d} |j dddi }
|
r$|
j d r|
d j } |j
d } g } d}
d}| rn| j d D ] }|j dd }|j d xs d}t j d|t j sQ|j d st t | }| j ||d t | j d }g }d%fd | j D ]g }t# |dd }|dv r|dk( rd|j d xs g v r2|dk( rd |j d xs g v rO | }|j% | i d j' d |D }
t j( d d |
j }
|
s | r d!j' d" | D }d#| }
||| |
|| d$S )&u 获取详情页rQ NrX r metarl ArticleTitle)attrscontenth1Trc PubDate
ContentSourcez div.gkwz_contnet.j-fontContentFr^ r` u 附件z,\.(pdf|doc|docx|xls|xlsx|zip|rar|7z)(\?|#|$)ra )rF rB tablec t | t r t | j }|r|gS g S g }g }| j D ] }t |t rWt | j }|s0|j d D ]& }|j }|s|j
| ( k|j dk( r&|s}|j
dj | g }|j dk( r|r"|j
dj | g }g }|j d D ]R }|j ddg D cg c] } | j d }
} |j
d
j |
T |s?|j
dj | a|j d v rb|j d d
xs |j dd
} | s| j d st t | } |j
d| d |j dv r?
| } |r"|j
dj | g }|j |
| } t! | dkD r7|r"|j
dj | g }|j | k| so|j
| d |r |j
dj | |S c c} w )u+ 递归提取段落，
视为换行分隔
brrb r trtdthTrc z | )imgsrcr zdata-srcra u [图片](r6 )divpr4 r )
isinstancer rS rd rk splitrp rl joinri ro r? rj r r> extendlen)elementis_roottxt
paragraphscurrentrt linerowsr r cellsr sub_parasextract_paragraphss r* r z(fetch_detail..extract_paragraphs s '?3'l((* #u++JG )) -
5 e_5 e***,C$'IIdO 5D#'::z fetch_detail.. s "9q1"9s z\n{3,}r c 3 : K | ] }d |d d|d d yw)r$ rB z](rF r6 Nr )r r^ s r* r z fetch_detail.. s' L1Aai[1U8*A6Ls u 附件下载：
)r_ rg source content_text has_table attachments)F)rM r rh r? rd ro
select_oneri rm searchIrj r r> rp boolrk getattrr r rn )rF rG rq rs r_ mtr} rg mdr ms content_divr r r r^ r` rB partselemrl paraslinksr s @r* fetch_detailr ~ s C )D } -D
E
6&.!9 :B bffY 9
##%
YYt_
KKdK+E H
6&)!4 5B bffY i=&&( F
6&/!: ;B bffY I $$& //"BCKKLI %%c* @A55$D::D:)5XDyyH$PRPTPTUv."8T2D""4#>?
@ ))'23 9 v ((
D4.D**u}TXXg->-D"!Eu}DHHW,=,C!D&t,E LL
{{"9e"99 vvi >DDF K L LL*5'2 $ "
r, c t j t } | j }|j d |j d | j | |fS )Na CREATE TABLE IF NOT EXISTS gov_raw (
id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT NOT NULL,
page_url TEXT NOT NULL UNIQUE,
source_url TEXT,
publish_date TEXT,
content TEXT,
site_name TEXT,
group_name TEXT,
industry TEXT DEFAULT 'other',
attachments TEXT DEFAULT '',
has_table INTEGER DEFAULT 0,
summary TEXT DEFAULT ''
)zsCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
title, site_name, summary, tokenize=trigram
))sqlite3connectDB_PATHcursorexecutecommit)conncs r* init_dbr sS ??7 #D
AII
II
KKM 7Nr, c ^ |j dd xs d}|r|dd nd}t j |j dg d }|j d rd nd
}|j d |d |j d
d |j dd ||j dt |d |j dt
d|||f |j d
k( ry|j } |j d||j d
d |j dt |f | j y# t j $ rE | j t ||j d
d |j dt | Y hw xY w# t $ r}t d| Y d}~yd}~ww xY w)u INSERT OR IGNORE 入库r| r Nr3 r F) ensure_asciir r4 r zINSERT OR IGNORE INTO gov_raw
(page_url, title, publish_date, content, site_name, source_url,
group_name, industry, attachments, has_table, summary)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)rF r_ rg site_name
group_nameotherzMINSERT INTO gov_search (rowid, title, site_name, summary) VALUES (?, ?, ?, ?)Tu 入库异常: )r? jsondumpsr SITE_NAMEGROUProwcount lastrowidr OperationalErrorr
_fts_fallbackrE r+ ) r curdatar| summaryattachments_strr rowidrL s r* insert_articler # s $((9b)/R#*'$3- **TXXmR%@uU +.AA 8
K HHWb ! HHZ $ HH[) , K HH\5 ) :
<<1
c KKg " 5txx Y7WY`a
c
'' c KKM %'2!6i8XZa b c
qc "#sC CF F %:D0 F 0AFF FF F,F''F,c t j dt d| d|j t d t d t d z d|j t d t d t d z d|j t d t d t d z d gdd y # t
$ r}t
d
| Y d }~y d }~ww xY w) Nr zLINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (z, '' z', 'z')T
)capture_outputr2 u FTS fallback失败: )
subprocessrunr replacechrrE r+ )r r_ r r rL s r* r r L s * wZ[`Zaadejerersvwysz| AC }D EH IK EL }L fM eN NR S\ Sd Sd eh ik el nq rt nu vy z| v} n} S~ R C DK DS DS TW XZ T[ ]` ac ]d eh ik el ]l Dm Cn np
q
, * $QC ())*s B6B9 9 CCC c |r^t j j t j j t j j t
d at dt dt d| d |rt dt t \ }}t j }|j j t g }t! d| dz D ] }t d| t# || }|st d | d
n^t% | }|st d | d n@t d
t' | d |j) | t+ j, d t dt' | d d} d}
t/ |d D ] \ } } t d| dt' | d| d dd d t1 | d | }
|
s&t d |
dz
}
t+ j, d a| d xs |
d }| d xs |
d }t3 |||| d ||
d t t4 |
j7 d g |
j7 d d d! }|r| dz
} t d" n|
dz
}
t d# t+ j, d$ |j9 t d%d& t d'| d(|
| S ))u 主爬虫zhuoqiu_test.dbu
开始爬取 z - u ，u 页u 本地测试模式，DB: r4 u
列表页 u ❌ 第u 页获取失败，停止u ⚠️ 第u 页无文章，停止u ✅ u 条u
总计 u 条待爬详情r r$ r5 r% r_ N2 z...rF u ❌ 详情获取失败g?rg r r r F)r_ rF rg r| r r r r u ✅ 入库u ⏭️ 已存在g ?r z2==================================================u 完成! 新增: u
, 跳过: ) ospathr dirnameabspath__file__r r+ r COLUMN_NAMEr r: r; r1 updater= r9 rV rw r r rC rD enumerater r r r? close) max_pageslocalr r rG all_articlespagerq rr new_count
skip_countidxartdetail final_title
final_dateoks r* crawlr V s
'',,rwwrwwx/H IK[\- {#k]#i[DE 'y 12 ID# G OO7# LaQ ' j T7+ )D6!9: ; 4 ,tf$9: ; fSZL %&E"
1
)C %&&678 IJ lA. S auAc,'(3w<+<*=S AB c%j'2 * , !OJ JJsO 'l5fWo _:z(:
D# u:" n-" !::mR8 K7 (
NI , !OJ $ &
3= @ JJL"VH
9+Z
|<= r, __main__u' 霍邱县行政审批信息公开爬虫) descriptionz --max-pagesu 最大爬取页数)r defaulthelpz--local
store_trueu 本地测试模式)r r )r r )NNr8 r )r F)/__doc__sysr rm rC r argparser r r urllib.parser r r: bs4r r warningsfilterwarningsr> r r r r rT rR PAGE_SIZE TOTAL_PAGESr= r+ rM rV rw r r r r r __name__ArgumentParserparser add_argumentint
parse_argsargsr r r r, r* r
sE > = = = + . ! & #
5!
" .
JO' <&*#LJZ0&R*FR z
$X
$
$1Z
[F
CAUV
,=QR D DNN$**5 r,