=1j
D d Z ddlZddlZddlZddlZ eej dk r ed ej d ej d Z eej dkD rej d ndZ
eej dkD rej d ndZ ej dd Z
ej e
Zej! d
ej! d ej! d ej! d
ej! d ej! d ej! d ej# dZdZ ee d 5 ZeD ]W Zej/ Zes ej0 e Zej5 d xs ddd Zej5 d xs ej5 d xs ddd Z ej5 d xs dj/ Z ej5 d xs ddd Z e
xs ej5 d xs d dd Z e xs ej5 d xs d dd Z ej5 d xs g Z!e!r ejD e!d!" ndZ#e resedz
Z ed#edd &ej! d$e f ej! d%ee e e e e e#e f edz
ZZ ddd ej# ejM ed(e d)e y# e$$ r!Z%edz
Z ed&e% d'edd Y dZ%[%dZ%[%ww xY w# 1 sw Y axY w)*u import_jsonl_raw.py - 直接导入 JSONL 到 gov_raw (保留正文 HTML 原样)
不走 import_jsonl_v2 的 strip_html 剥平逻辑 — 正文 content 保持 /
结构,
search_app 检测到 /
就当 HTML 渲染, 表格样式完整保留。
JSONL 字段: page_url(或 url), title, content, publish_date, site_name, attachments(数组或空)
用法: python3 import_jsonl_raw.py [site_name] [script_name]
script_name 缺省: 优先取 JSONL 内 script_name 字段, 再缺省 'import_jsonl_raw'
N zAUsage: import_jsonl_raw.py [site_name] [script_name] SEARCH_DBz/root/search.dbzPRAGMA journal_mode=WALzPRAGMA busy_timeout=300000zPRAGMA synchronous=NORMALzkCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5(
title, site_name, summary, tokenize=trigram
)zCREATE TRIGGER IF NOT EXISTS trg_gov_raw_fts_ins AFTER INSERT ON gov_raw BEGIN
INSERT INTO gov_search(rowid, title, site_name, summary)
VALUES (new.id, new.title, new.site_name, new.summary);
ENDzCREATE TRIGGER IF NOT EXISTS trg_gov_raw_fts_del AFTER DELETE ON gov_raw BEGIN
DELETE FROM gov_search WHERE rowid = old.id;
ENDzCREATE TRIGGER IF NOT EXISTS trg_gov_raw_fts_upd AFTER UPDATE ON gov_raw BEGIN
DELETE FROM gov_search WHERE rowid = old.id;
INSERT INTO gov_search(rowid, title, site_name, summary)
VALUES (new.id, new.title, new.site_name, new.summary);
ENDzutf-8)encodingtitlei page_urlurli content publish_date site_nameunknownd script_nameimport_jsonl_raw attachmentsF) ensure_asciiz [ERR] missing title/url: z&DELETE FROM gov_raw WHERE page_url = ?zINSERT INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments, script_name) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?, ?)z [ERR] z: u ✅ 导入完成: +z err)'__doc__jsonsqlite3sysoslenargvprintexit
jsonl_path
force_site force_scriptgetenvdb_pathconnectdbexecutecommitaddederrorsopenflinestriploadsitemgetr r
r r
r r attdumpsattachments_str Exceptioneclose import_jsonl_raw.pyr: s0 sxx=1
MN CHHQK
XXa[
MA -SXXa[2
!#((ma /sxx{R
"))K!2
3W__W
$ %
' (
& '
*w' 01 0zz| 0 4::d#DXXg&,"ds3E ,EE2uMHxx *0b779G HH^ 4 :CR@L"Ptxx '<'I 4C&PI&_488M+B+XFXZ^[^*_K((=)/RCEHjdjj5 AbO5! 3D#J<@A JJ?( M JJ r'<Ho_jk
QJE10 0:
E7$vh/ 0 0 aKF HQCr$t* . / / 05 0 0sC L2D K->L 2K-2L-L2LLLLL