<1j dZddlZddlZddlZddlZddlZddlZddlZdZejddZ dZ dZ dZ d Zed k(reyy) uQimport_jsonl_auto.py - 调度器自动导入器 (根治"只抓不导入") 用途: 日跑调度器跑完脚本后自动调用, 把脚本输出的数据导入 search.db。 支持两类输出: 1. --file 脚本写文件型 (直接导入) 2. --stdout 脚本 print 到 stdout 型 (智能解析) stdout 解析支持三种格式: A. JSON_BEGIN:...:JSON_END 包裹 (crawl_hequ 等) B. 整体 JSON 数组 print(json.dumps(list, indent=2)) (crawl_shuozhou 等) C. 逐行 JSON 对象 (JSONL) (crawl_baixiang 等) 字段归一化: title, url/page_url/link, date/pub_date/publish_date, content, attachments, site_name 用法: python3 import_jsonl_auto.py --file /tmp/x.jsonl [--script crawl_x.py] [--site "站点名"] python3 import_jsonl_auto.py --stdout /tmp/x.out [--script crawl_x.py] [--site "站点名"] Nz/root/gov_crawler SEARCH_DBz/root/search.dbcFg}tjd|tj}|rY tj|j d}t |tr d|vr|d}|St |tr|}|S|g}|S|j} tj|}t |tr|St |trd|vrt |dtr|dS|gS |jD]v}|j}|s|ddvs tj|}t |tr|j|n!t |tr|j|x|S#t$rYwxYw#t$rYwxYw#t$rYwxYw)uF从脚本 stdout 文本中提取 JSON 数据记录列表 (dict 列表)zJSON_BEGIN:(.*?):JSON_ENDitemsrz[{)researchSjsonloadsgroup isinstancedictlist Exceptionstrip splitlinesappendextend)textrmdatastrippedlineobjs import_jsonl_auto.pyparse_stdout_textrs E .bdd;A ::aggaj)D$%'T/W  L D$'LLzz|H  zz(# dD !K dD !$:d7mT#BG}$6M "##% zz|  7d? jj&c4(LL%T*LL%  L;         sN>E6*E6>E6&F;+F'FAF6 FF FF F F ct|jdxs|jdxsdjdd}t|jdxs(|jdxs|jdxsdjdd}t|jd xs;|jd xs(|jd xs|jd xsdj}t|jd xs;|jdxs(|jdxs|jdxsddd}t|jdxsdjdd}|jdxsg}t|tr t j |}t|tsg}||||||dS#t $rg}Y(wxYw)Ntitlenameipage_urlurllinkicontent content_htmlbodyr publish_datepub_datedatedate_pub site_named attachments)rr!r$r'r,r.)strgetrr r r rr)itrr!r$pubsiteatts rnormalize_itemr5Js w7266&>7R 8 > > @# FE266*%NN"&&.NBOUUWX]Y]^H"&&#grvvn'=ggSUSYSYZ`SageghnnpG bff^$hz(:hbffVnhPRPVPVWaPbhfh ijmkm nC rvvk"(b ) / / 1$3 7D &&  %2C#s **S/C c4 Wd3 HH  C s,G G*)G*c |s tdytjt}|j d|j d|j d|j d|j d|j d|j d |j d \}}|D]}t |}|d r|d s|d z }|xs |dxsd}|xsd} |drtj|ddnd} |j d|d f|j d|d |d |d|d||d | | f|d z }|j |jtd|d|d|S)Nu [AUTO-IMP] 无有效记录rzPRAGMA journal_mode=WALzPRAGMA busy_timeout=300000zPRAGMA synchronous=NORMALzsCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5( title, site_name, summary, tokenize=trigram )zCREATE TRIGGER IF NOT EXISTS trg_gov_raw_fts_ins AFTER INSERT ON gov_raw BEGIN INSERT INTO gov_search(rowid, title, site_name, summary) VALUES (new.id, new.title, new.site_name, new.summary); ENDzCREATE TRIGGER IF NOT EXISTS trg_gov_raw_fts_del AFTER DELETE ON gov_raw BEGIN DELETE FROM gov_search WHERE rowid = old.id; ENDaCREATE TRIGGER IF NOT EXISTS trg_gov_raw_fts_upd AFTER UPDATE ON gov_raw BEGIN DELETE FROM gov_search WHERE rowid = old.id; INSERT INTO gov_search(rowid, title, site_name, summary) VALUES (new.id, new.title, new.site_name, new.summary); END)rrrr!rr,unknownimport_jsonl_autor.F) ensure_asciir z&DELETE FROM gov_raw WHERE page_url = ?zINSERT INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments, script_name) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?, ?)r$r'u [AUTO-IMP] ✅ 导入 +u (跳过 )) printsqlite3connectDB_PATHexecutecommitr5r dumpsclose) records force_site force_scriptdbaddedskippedr1nr, script_nameatt_strs r write_recordsrL]s  ,-  !BJJ()JJ+,JJ*+JJ  JJ  JJ  JJ  IIKNE7  2 z: qLG =!K.=I "9&9 FG FV$**Q}-EB\^ ;a m=MN  n wZ:) a6GTUV`Tacjlw x    IIKHHJ %eWIgYa @A LcJtj}|jdd|jdd|jddd |jd dd |j}|jrzt j j|jr,t j j|jd k(r-td |jtjd g}t|jdd5}|j}dddjjdr t!|ng}|sat|jdd5}|D]:}|j}|s |j#t%j&|< dddt+||j,|j.}tjd |j0rt j j|j0s-td|j0tjd t|j0dd5}|j}dddt!}t+||j,|j.}tjd tdtjdy#1swYxYw#t($rYwxYw#1swYRxYw#1swYxYw)Nz--fileuJSONL 文件路径)helpz--stdoutu 脚本 stdout 输出文件路径z--scriptr u脚本名 (写入 script_name))defaultrOz--siteu强制 site_nameru' [AUTO-IMP] 文件不存在或为空: zutf-8ignore)encodingerrors)[{u% [AUTO-IMP] stdout 文件不存在: zWUsage: import_jsonl_auto.py --file | --stdout [--script X] [--site Y]r)argparseArgumentParser add_argument parse_argsfileospathexistsgetsizer;sysexitopenreadr startswithrrr r rrLr3scriptstdout)apargsrCfr$rrGrs rmainrisR  "BOOH#7O8OOJ%GOHOOJ1QOROOHb/AOB ==?D yyww~~dii(BGGOODII,F!,K ;DII;G H HHQK $))gh ? 1ffhG 07 0J0J:0V#G,\^dii'(C qD::   {{ww~~dkk* 9$++G H HHQK $++ A Q668D #D)gtyy$++>   cdHHQK9  %     sHK/;L $K<9L 4L/K9< L L L  L  LL"__main__)__doc__r r<r_r[rrV subprocessBASEgetenvr>rr5rLri__name__rMrrqs\"8777 "))K!2 3,`H&+\*Z zFrM