j% d Z ddlZddlZddlZddlZddlZddlZej j dej j ej j e ddl Z ddl
mZ ddlZ ej Zej% dedd ej) \ ZZdZdZd Zej4 dkD r ej4 nd
Zd d d
dZ h dZ d Z d Z d Z d Z!d Z"e#dk( r e" yy)uM Crawler for 淮南市大通区人民政府 - 通知公告 v2 (FIXED encoding) N)
BeautifulSoupz--pagesu 限制页数)typedefaulthelpu+ 淮南市大通区人民政府-通知公告zhttps://www.hndt.gov.cn6787542 zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36zUtext/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentAcceptzAccept-Language>) pdddldth1h2h3h4h5h6hrlioltdthtruldivnavpreformmainasidetabletbodytfoottheadcenterdialogfigurefooterheaderhgroupaddressarticlecaptiondetailssectionfieldset
blockquote
figcaptionc | yddl }ddlm g fd | dj }|j dd| }|j dd | }|j S )
ul 块级边界出换行、行内节点直接拼接、
出换行（≈ 浏览器看到的换行结构）。N r )NavigableStringc F | j D ] }t | rj t | *t |dd dk( rj d Lt |dd t
v r+j d | j d | y )Nnamebr
)children
isinstanceappendstrgetattr _BLOCK_TAGS)nodechr5 outwalks "/root/gov_crawler/crawl_hndt_v2.pyrC zbody_text..walk. s --
B"o.
3r7#VT*d2
4 VT*k9
4 R
4 R
z[ \t\r\f\v]*\n[ \t\r\f\v]*r9 \n{3,}
)rebs4r5 joinsubstrip)el_retr5 rB rC s @@@rD body_textrP & sb z#
C H
A -tQ7A 61%A 779rE c ` t j | t d }d|_ |j S )u> GET with forced UTF-8 — server omits charset in Content-Type )headerstimeoututf-8)requestsgetHEADERSencodingtext)urlrs rD get_pager] A s$ S'26AAJ 66MrE c N t | d }g }|j dt j d }|s|S |j dd D ] }|j d }|s|j d d
}|s,|j d d
xs |j
d
}|j dd }|r|j
d
nd
} |j d r
t |z }n,|j d st dz |j d z }|j ||| d |S )N html.parserr doc_listclass_r F) recursiveahrefr4 titleTrL spandate/http)r[ rf ri ) r findrH compilefind_allrW get_text
startswithBASElstripr< )
htmlsoupitemsr r rd re rf date_spanri s
rD extract_listrw H s } -D E
4
; 7 8B
kk$%k0 B GGCL uuVR gr")?K iibjj9I.JiK ))#. CEIIfb)EyyGPRPTPTU T2Jekk#6Fr6J ''/ 3J c): :E""QugRwa#89""U5#AB C )) &Euf%%***? TCJ . 56
+ &ffVS#.446LL%E3'kkmLL% & ++e$&&FG4::< $P[Ke!L ccac ccrE c " g } t dt dz D ] }|dk( r t dnt dt d| }t d| dt d| t t
| }|s
t d n@t d t | d
| j | t j d t d
t | d
d}d}t |dd 5 }t | d D ] \ }} t d| dt | d| d r| d d d n| d d d ddd t
| d }
t |
\ } } }
}| xs | d | d |
| xs | d | xs | d t d |d }|j! t# j$ |d d!z |dz
}t d" t j d% d d d t d&| d'| d( t |d) 5 }t | D ]; \ }}|d*k\ r n1t# j& | }t d+|dz d,|d d d- = d d d t d. t) j* g d/d0 t j d dd l}|j/ d1 }|j1 d2 j2 }|j5 |j7 t d3| d
t d4 t) j* d5d6|gddd7d89 }t t |j8 d:kD r|j8 d;d n |j8 t d< |j/ d1 }|j1 d= |j1 d> |j1 d? |j1 d@ |j5 |j7 t) j* g dAddB t dC t dD| dE y # t $ r}t d | Y d }~d }~ww xY w# t $ r`}t d#| |j! t# j$ | d | d d$| d | d t d d$d d d!z |dz
}Y d }~d }~ww xY w# 1 sw Y xY w# 1 sw Y YxY w)FNr z/zwgk/tzgg/index.htmlz/content/column/z ?pageIndex=u [页码 rj z] u → 无数据，停止u
→ 提取 u 条u → 失败: g ?u
共 z/tmp/hndt_v2_import.jsonlr wrU )rY r rf ( r[ rR z...r T)endflushri u 通知公告)rf page_urlr} publish_datesummary site_nametagsr Fr r9 u ✓u ✗ r4 g333333?u
✅ JSONL: z (u 条) — 确认前3条编码:r\ z z. 2 u!
🗑️ 删除旧乱码数据...) systemctlstop
search_app)capture_outputz/mnt/data/search.dbz7DELETE FROM gov_raw WHERE page_url LIKE '%hndt.gov.cn%'u 删了 u
📥 导入新数据...python3z!/root/gov_crawler/import_jsonl.pyi, z /mnt/data)r rZ rT cwdi ipu
🔨 重建 FTS...zDELETE FROM gov_searchzDELETE FROM gov_search_v3z4INSERT INTO gov_search(gov_search) VALUES('rebuild')zINSERT INTO gov_search_v3(title, site_name, source_url, publish_date) SELECT title, site_name, page_url, publish_date FROM gov_raw)r startr )r rT u ✅ 搜索服务已重启u
✅ 完成！u 条，编码已修复) range
PAGE_COUNTrq COLUMN_IDprintrw r] lenextend Exceptiontimesleepopen enumerater SITE_NAMEwriter r loads
subprocessrunsqlite3connectexecuterowcountcommitclosestdout) all_itemspager[ ru e output_pathcountfiitemrs rf ri r} r recordlinerecr dbdeletedr\ s rD r r s Iaa (
04 +,$GWXaWbbmnrms?t
a
|2cU34 # /E01 M#e* T2 3 U #
3
F3y>"$
'(-K
E
k3 1 Q A. GAt AaS#i.)$w-DM#2,>UYZ_U`adbdUe+ffijps{
A
U ,4@4F1tWk " 3d7m $U &$($8DL $ 5W
!* *#.
6>EF
e
JJsO? D
M+ b/M
NO
k3 31 | 3GAtAv**T"C Bqse2c'l3B / 01 2 33
./NN6tLJJqM . /BjjRS\\GIIKHHJ IgYd
#$
%& #F T&*s UA S]S 0!((45/ahh?
! . /BJJ'(JJ*+JJEFJJ N OIIKHHJNN7VXY
%& OE7"8
9:c N1#& ' 8
QCj!
!'] $U !$(L #G}!* *#% $ !& ' *. . /
' F3 3sd !O&5)O& AQ7'A>P %Q7#A
R& P/PP Q4AQ/)Q7/Q44Q77RR__main__)$__doc__sysosrH r r r pathinsertdirnameabspath__file__rV rI r argparseArgumentParser_AP add_argumentintparse_known_args_ARGS_r rq r pagesr rX r? rP r] rw r r __name__ rE rD r s S * * * 277??277??8#<= > h an E !q 9 !KK!OU[[
De0 * 609dx^;B zF rE