z
)titler datesummarycontent) refinditerDOTALLgroupsearchr BASEsubstripappend) htmlitemsmblockhref_mpdf_pathpdf_urltitle_mr$ r% date_mr& r' s r parse_listr: ) ss E [[ bii ' 159 <<?$) )) H%QSQZQZ[ii UW\^`^g^ghG z2w}}Q'78>>@ 7B ll1o&a Q'8&,,q/9JKD 1 ;&wi/dejdkkop C' R L c t j t } | j d | j d | j | j y )NPRAGMA journal_mode=WALaH CREATE TABLE IF NOT EXISTS quality_results ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT NOT NULL, domain TEXT, title TEXT, url TEXT UNIQUE, content TEXT, publish_date TEXT, summary TEXT, crawled_at TEXT DEFAULT (datetime('now','localtime')) ))sqlite3connect QUALITY_DBexecutecommitclose)conns r init_dbrE X sB ??: &DLL*+LL KKMJJLr; c d}t j t }|j d | D ]N } |j d|d f |j dt t |d |d |d |d |d f |d z }P |j |j |S # t $ r}t d | Y d }~d }~ww xY w) Nr r= z'DELETE FROM quality_results WHERE url=?r zrINSERT INTO quality_results (site_name, domain, title, url, content, publish_date, summary) VALUES (?,?,?,?,?,?,?)r$ r' r% r& r u ❌ DB: ) r> r? r@ rA SITE_NAMEDOMAINr printrB rC )r2 newrD itemr s r storerL i s C ??: &DLL*+ & & LLBT%[N S LL EFDM4;YQUV\Q]_cdm_no 1HC & KKMJJL J & L$ % % &s A B"" C+B>>Cc t d t j t } | j dt f j }| j |s t d yt j d }|j d d}d}|D ]J }|\ }}}} } |j d |||xs d dd | xs d | xs d dd |d f |j dkD r|d z }L |dkD r3|j |j d|f |j |j d|f j d } |j t d| dt | d| d y# t $ r} t d| Y d} ~ d} ~ ww xY w)u@ 将本地数据直接写入 search.db（服务器本地模式）u 📤 同步到 search.db...z_SELECT title, url, content, publish_date, summary FROM crawl_results WHERE domain=? ORDER BY idu 本地没有数据Nr r= r r zwINSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, summary, site_name, tags) VALUES (?,?,?,?,?,?,?)r i i, r z Error: zINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z OK /u 条同步到 search.db (DB共u 条)) rI r> r? r@ rA rH fetchallrC total_changesr rB fetchonelen) rD rowsdst site_name new_countrr$ r r' pub_dater& r totals r sync_to_serverrZ { s )* ??: &D <<i hj JJL $% //+ ,CKK)*'II #12.sGXw # KK) gmWf 5x~2-R#& 27 1$Q # 1} Q L KK89, HHP ST VEIIK E)Ac$i[(FugT RS# # IaS/ " " #s ?E,, F 5FF __main__u 新和成环境信息爬虫) descriptionz--test store_trueu 测试模式)actionhelpz--syncu 仅同步 z2==================================================u 🚀 u ❌ 列表页获取失败r u 📋 解析到 u 条记录u 🔬 测试模式： u ✅ r$ ( z... | r% z | r iu ...（共u 条，未存库）u ✅ 入库 u 条u 📊 范围: z ~ u 📄 全部为PDF文档u 🏁 完成)4__doc__sysosr( jsonr> sslurllib.requestr base64 subprocessr urllib.parser pathdirnameabspath__file__BASE_DIRjoinr@ SERVER_SSH SERVER_DBSERVER_SEARCH_DBLIST_URLr- rG rH _create_unverified_contextr r r r: rE rL rZ __name__argparseArgumentParserparser add_argument parse_argsargssyncexitrI r1 r2 rR testrK n r; r