j_ , d Z ddlZddlZddlZddlZddlZddlZd Z eej dk r e
d ej d ej d Z eej dkD rej d ndZ
ej dd Z ej e ZdZdZ ee 5 ZeD ]F Zej/ Zes ej0 e Zej5 d
xs d dd Zej5 d
xs d dd Z eej5 d xs d Z ej5 d xs d dd Z ej5 d xs ddd Z ej5 d xs g Z e r ejB e d nd Z"ejG de f ejG dee e e e e e"f ejG d jI d Z%e re dd nedd Z&ejG de%ee e&f edz
ZI ddd ejS ejG d jI d Z*ejW ejX e e
d e
d e d e d e* y# e'$ rZ(edz
ZY dZ([(dZ([(ww xY w# 1 sw Y xY w)!uL Server-side JSONL importer v2: 保留表格结构(转|分隔行)与链接URL Nc | syt j | } g fd}t j d|| t j t j
z } d }t j d|| t j t j
z } t j dt
d | t j
} t j d d| } t j d
d | } t j d t
d d
z | } d }t D ]! \ }}| j d| d || } # | j S )u^ 正文清洗: 保留 结构转 | 分隔行, a 链接转 [文本](URL), 其余转段落 c h j | j d dt dz
dS )Nr @@TABLE @@)appendgrouplen)mtabless import_jsonl_v2.py _hold_tablez strip_html.._hold_table s.
aggaj!VQ r** z]*>.*?
flagsc t j dd| j d j }| j d }|r d| d| dS |S )N<[^>]+>r r [z]())resubr
strip)r txthrefs r
_hold_linkz strip_html.._hold_link sN ffZQWWQZ0668wwqz%(3%r$q!2d2r z.]+href=["\']([^"\']+)["\'][^>]*>(.*?)zB]*>
r z[ \t]+ z\n{3,}r c t j d| t j }g }|D ] }t j d|t j }|D cg c]' }t j dd| j ) }}|j ddj
| z dz t d j
| S c c}w )
Nz]*>([\s\S]*?)r z ]*>([\s\S]*?)r r z| z | z |r )r findall
IGNORECASEr r r joinchr)trowslinesrcellscs r _restore_tablez"strip_html.._restore_table s zz5q
N :AJJ @!2==YE@EF1RVVJA.446FEF LL
5 11D8 9 : 2w||E"" Gs ,Cr r )
htmlunescaper r DOTALLr" r$ enumeratereplacer )textr r r+ ir% r
s @r
strip_htmlr3 s$ == D
F+ 66,k4ryySUS`S`G` aD3 66CZQU]_]f]fikiviv]v wD
66WY\]_Y`bfnpn{n{ |D
66*b$ 'D
66)S$ 'D
66)SWQY -D# &! @1||gaSO^A->?@ ::<r r z+Usage: import_jsonl.py [label]r data SEARCH_DBz/root/search.dbtitler i page_urli content publish_date site_nameunknownd attachmentsF) ensure_asciizTDELETE FROM gov_search WHERE rowid IN (SELECT rowid FROM gov_raw WHERE page_url = ?)zINSERT OR REPLACE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?)zSELECT last_insert_rowid()zWINSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)z SELECT COUNT(*) FROM gov_rawu ✅ z: +z errz total=)-__doc__jsonsqlite3sysosr r, r3 r argvprintexit
jsonl_pathlabelgetenvdb_pathconnectdbaddederrorsopenfliner loadsitemgetr6 r7 r8 r9 r; attdumpsattachments_strexecutefetchone new_rowidsummary Exceptionecommittotalcloseremove r r rd s R ' ' '!F sxx=1
78 CHHQK
XXa[
388}q( f
"))K!2
3W__W
* $ #zz| 4::d#DXXg&,"ds3E ,2ET:H )!4!:;G HH^ 4 :CR@L +.;)TcBI ((=)/RCEHjdjj5 AbO
JJmpxoz {
JJ m'<Ho^
#?@IIKANI'.gdsmE$3KG JJiE9g6
QJEC#$L
12;;=a@
* wc%VHGE7; < aKF G$ $s7 3J
D/I2=J
2J7J<J
JJ
J