ujpdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z e jddZ dZ ejjejjeZdZej$ddZdZd Zd Zd Zd Zd ZddddZdZdZdZdZdZdZ e!dk(re yy)uGcrawl_hipac.py — 江苏淮安工业园区管委会 公告公示爬虫Nignorezhipac.huaian.gov.cnu淮安工业园区 SEARCH_DBz/root/search.dbz;http://hipac.huaian.gov.cn/articleCommonController/lists.dozhttps://hipac.huaian.gov.cn5632 0000000064a8f16d0164ae1f25dd00ff )newskiperrorsctjt}|jd|jd |jd|j |j y#tj$rY6wxYw)NzPRAGMA journal_mode=WALaYCREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT, source_url TEXT, page_url TEXT, title TEXT, publish_date TEXT, date_rank INTEGER DEFAULT 0, summary TEXT, status TEXT, category TEXT DEFAULT '', visits INTEGER DEFAULT 0, content TEXT DEFAULT '', tags TEXT DEFAULT '' )zCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5( title, content, site_name, content='gov_raw', content_rowid='id', tokenize='unicode61' ))sqlite3connectrexecuteOperationalErrorcommitclose)conns /root/gov_crawler/crawl_hipac.pyinit_dbrsr ??9 %DLL*+LL        KKMJJL  # #   sA//BBc tjt} |jd||||tt |df|j dkDrS|jd|fj}|r |jd|d||t ftdxxdz cc<ntdxxdz cc<|jy#tj$rYIwxYw#td xxdz cc<Y?xYw#|jwxYw) NzINSERT OR IGNORE INTO gov_raw (title, content, publish_date, page_url, source_url, site_name, summary, tags) VALUES (?,?,?,?,?,?,?,?)rz'SELECT id FROM gov_raw WHERE page_url=?z@INSERT INTO gov_search(rowid, title, site_name) VALUES (?,?,?,?)r r r ) r rrrDOMAIN SITE_NAME total_changesfetchoneIntegrityErrorstatsr)titleurlcontentdatesummaryrrow_ids r store_recordr%(s ??9 %D ]WdCGRP R    !\\"KcVT]]_FLL!c"()UGY!GI %LA L &MQ M --  h1 s<AC0C #CCCCCC/-C22Dcntt|ttdtd}ddddd} |j t ||dd }|j jd ijd g}g}|D]}|jd djjddjdd}|sF|jdd} | s[|jtd| ||jdxsdddd|S#t$r} td|d| Yd} ~ yd} ~ wwxYw)Nr)topicpagepagesizerrdeptid Mozilla/5.0z0application/x-www-form-urlencoded; charset=UTF-8Ahttp://hipac.huaian.gov.cn/cmsweb/zwgk/gyyq/index.html?topic=5632XMLHttpRequest) User-Agentz Content-TypeRefererzX-Requested-WithF)dataheaderstimeoutverifyvaluelistr   path/ releaseTimer)r rr"u ⚠️ API错误(第u页): )TOPICstr PAGE_SIZERDEPTIDpostAPI_URLjsongetstripreplaceappend DETAIL_DOMAIN Exceptionprint) sessionpage_numr1r2ritemsresultitemrr:es rfetch_list_pagerR>sCCMs9~XZgn oD*]*name="ArticleTitle"[^>]*content="([^"]+)"rr7r8r9z(.*?)z - z@]*class="content_body"[^>]*>(.*?)\s*\s*z4]*class="content_d"[^>]*>(.*?)\s*z]*>.*?)flagsz]*>.*?u(公开日期[：:]\s*(\d{4}-\d{2}-\d{2})z<[^>]+>z\s+)rr!r"r#)rangerDtextlentimesleepresearchIgrouprErFDOTALLsplitsubrI) rKr attemptrMhtmlrmr!cr"r#rQs r fetch_detailriQsR8(*' * CE :A66D4y3 1 xE PRVXZX\X\]A ((*2263?GGcRII5tRYYGGGAJ,,.66vsCKKDRUVE~ % E1 5b 9G ]_cegenenoAIIUW[]_]f]fgGGAJ$$&FF7QbiiPRPTPTnUFF5r1BIIbddNSD EtLA$ffZW];GDKbffVS'288:4C@QVG"wQXY YM(*R  *{DJJqM *sAJIJ K %KK c tjt}|jd|dtfj }|j |rytd|dddddd t||d}|std td xxd z cc<yt|dxs|d|d|d|jdxs|jdd|dtdt|jdddy )Nz6SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?r Fu 📄 r(z...rT)endflushu ✗ 详情失败r rr!r"r#u ✅ B) r rrrrrrrJrirr%rDr[)rKrPrexistsds r process_itemrq}s ??9 %D \\RUYZ_U`bkTl m v v xFJJL e GDM#2&'s +4@Wd5k*A  !"E(Oq$8O7,tG}d5k1Y<vInZ^ZbZbcikmZnpqr{p|} E#aeeIb)*+1 -. c ddl}|j}|jdd|jdd|jdtd|j }t |j r tdytj}|jjd d i|jd d d tj}d}|jrtnd}t!d|dzD]}|j"r||j"k\rnwt%||}|sngtd|dt'|d|D]E} |j"r||j"k\rmt)|| r|dz }tj*dGtj|z } tdddt,ddt,ddt,dd| dd t,ddkDryy)Nrz--full store_true)actionz--syncz--test)typedefaultu,--sync 已弃用，脚本直接写 search.dbr.r+r,r0FrUru📃 第u页 (u条)g?r9z2==================================================u 🏁 新增:r u 跳过:r u 错误:r u ⏱️ z.0fs)argparseArgumentParser add_argumentint parse_argsrsyncrJrequestsSessionr2updaterDr\full TOTAL_PAGESrYtestrRr[rqr]r) ryparserargsrxstart total_new max_pagespnrNrPelapseds rmainrs  $ $ &F 6 6 sA6    D I yy%FGAIIlM23EE MWYbgEh IIKEI#yy aIAy1}% 99dii/2&e E#e*T23 DyyY$))3UAt$i1ni JJsO  iikE!G BvhnU5\N(5=/RWX`RaQbbklstwkxxy z{ U|arr__main__)"__doc__osr^sysr\rCr subprocessrwarningsfilterwarningsrr:dirnameabspath__file__BASE_DIRrgetenvrrBrHr=r@rr?rrr%rRrirqr__name__rrrrsM3333!   77??277??84 5 BIIk#4 5 G-  ,  1*&,&*X @ zFrr