uj!dZddlZddlZddlZddlZddlZdZdZdZejjejjejje dZdZdd iZd Zd Zd Zd Zedk(reyy)u 中国石化安庆分公司 - 公示信息爬虫 http://apw.sinopec.com/apw/csr/environmental_health/ CMS: SharePoint，无WAF，可直接用 requests Nu"中科炼化-项目环保三同时z[http://zrpc.sinopec.com/zrpc/social_responsibility/environmental_protection/simulta_environz\http://zrpc.sinopec.com/zrpc/social_responsibility/environmental_protection/simulta_environ/zquality_results.dbzroot@1.94.217.116z User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36cg}tjd|D]I}d|jdz}|jdj}|j ||dK|setjd|D]L}d|jdz}|jdj}|s9|j ||dN|S)u"从列表页HTML提取所有条目z]*title="([^"]*)"[^>]*href="(/zrpc/social_responsibility/environmental_protection/simulta_environ/\d+/news_[\w_]+\.shtml)"http://zrpc.sinopec.com)titlehrefzuhref="(/zrpc/social_responsibility/environmental_protection/simulta_environ/\d+/news_[\w_]+\.shtml)"[^>]*>([^<]+))refinditergroupstripappend)htmlitemsmrrs #/root/gov_crawler/crawl_zrpc_env.pyget_list_itemsrs E [[\^bc5(1771:5   " ud345 VX\] =A,qwwqz9DGGAJ$$&E ud;<  = Lcddl} |j|td}d|_|j}d}tjd|tj}|r4tjd d|jd j}|sLtjd |}|r4tjd d|jd j}d}tjd |}|r%|jd }|ddd|ddd|ddd}tjd|tj}|r|jd } tjdd| } tjdd| } tjdd| } d} tjd| | tj} tjdd| } tjdd| } tjdd| } | j} nd} | rt| dkrd|d } ||| |d!S#t $r}dt |icYd}~Sd}~wwxYw)"u-访问详情页提取标题、日期、正文rNheaderstimeoututf-8errorz%
(.*?)
z<[^>]+>rz([^<]+)z .*z/(\d{8})/news_[\w_]+\.shtml-z 08:00:00zD
(.*?)
\s*
]*style=\'display:none\'>[^<]*
z&]*__ControlWrapper[^>]*>|u^\s*页面内容\s*c|jd}|jd}|jdrd|z}d|d|dS)Nr/rz z)r startswith)rrtexts r make_abs_linkz!get_detail..make_abs_linkQsG771:D771:Ds#047tfBtfD1 1rz9]*href="([^"]+\.(pdf|docx?|xlsx?))"[^>]*>([^<]+))flagsz style="[^"]*"z]*>|z  z

z

)rpub_datecontenturl)requestsgetHEADERSencodingr$ Exceptionstrr searchDOTALLsubr r Ilen) r+r,rrerrr)d content_htmlr%r*s r get_detailr;)s2! LLgrL : vv E :D"))LAz2qwwqz288:  II/ 6 FF;AGGAJ7==?EH 0#6A GGAJeWAa!fXQq1vhi8 Y[_acajajkAwwqz vvLbR^_ vvG\Z vv4b,G  2 vvZ+\G vv/\B vv4b,G vvil; $$& c'lR'wd#  q !Q  !s+H77 I I IIc|syg}|D]}|djdd}|djdd}|jddxsdjdd}|jdd}|jd td |d |d |d |d td |d d dj |zd z}d}t |d5} | j |dddtjddd|td|gddd}|jdk7r.td|jtj|ytjdddtd|d|gddd}|jdk(r tdntd|jddtj|tjdddtd gdddtd!y#1swYxYw)"u5同步到远程服务器: SCP SQL 文件 + SSH 执行Nr'z''r+r*rr)zoINSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary, date_rank) VALUES ('z','z');zBEGIN;zCOMMIT;z/tmp/sync_apw.sqlwscpz-ozConnectTimeout=10:T<)capture_outputr$rru SCP失败: sshzsqlite3 /root/search.db < z && rm xu 同步OKu 同步失败: zsqlite3 /root/search.db "INSERT INTO gov_search(gov_raw) SELECT rowid FROM gov_raw WHERE rowid NOT IN (SELECT rowid FROM gov_search);"u FTS更新完成)replacer-r SITE_NAMEjoinopenwrite subprocessrun SERVER_SSH returncodeprintstderrosunlink) records sql_linesr7rr+r*datesqltmpfs rsync_to_serverrYps I  ' ""3-hsD)55B'-266sDAuuZ$!{#cU#eWCyD6YKWZ[_Z``c e   RWWY' ') 3C C c31   t%8#*Qse?TU&*r CA||q ahhZ() #t%8*4SEFH&*s DA ||q l  $3 012IIcNNNE4!4j_`"&T2? /s G""G,c Ltdtddl}g}t}t ddD]}|dk(r t dz}nd|d}td|d d d  |j |td }d|_|j}t|}|stdn"|D cgc] } | d|vs | } } | rt| t|dzkrtt|dntt| d| D]} | ddd} td| dd d t| d} |j| d| j drtd| dbt| j dd dkr td|j| tdt| dd |s td!ytd"t|d#ddl}t#j$t"j&j)t*d $|j-t*}d}|D]>} |j/d%t|d|d&|d|d'f}|j0dkDr|dz }@|j3|j/d(j5d}|j7td)|d*|td+t9|td,y#t$r}td|Yd}~Od}~wwxYwcc} w#YxYw)-Nu 爬取: rr r"zdhttp://zrpc.sinopec.com/zrpc/social_responsibility/environmental_protection/simulta_environ/default_z.shtmlu 第u页: rT)endflushrrru请求失败: u0 条rg333333?u 条 (已爬)u 条r2u 详情: z... rzFAIL: r*r(u内容过短，跳过zOK (u字)u未爬取到有效数据u 共获取 u 条有效记录)exist_okzkINSERT OR IGNORE INTO quality_results (site_name, title, url, content, publish_date) VALUES (?, ?, ?, ?, ?)r+r)z$SELECT COUNT(*) FROM quality_resultsu入库: 新增u, 累计u同步到服务器...u完成)rOrGr,setrangeBASE_URLr-r.r/r$r0rr6r;addr sqlite3rQmakedirspathdirnameDB_PATHconnectexecuterowcountcommitfetchonecloserY)r, all_details visited_urlspage_nr+r7rr8ri new_itemsitem title_shortdetailrdconn new_countcurtotals rmainr{s0 Jyk "#K5L2,.7 Q;S.CyyyAAGHC fXU#48  S'2 >A AJ66D t$ 'N !&G16,)FQG GC NSZ#-== SZL . /  Y %& 7Dw-,K L T2$ GV -F   T&\ *zz'"vg/016::i,-2-.   v & DVI./05 6! 7=.7`  () L[)**: ;<KK(48 ??7 #DI   ,,}AgJ%!I,* NC||aQ   KKM LL? @ I I KA NEJJL N9+XeW 56 !"; (OG  N1#& '  HZ  s0+K4% L2L;L4 L=LLL#__main__)__doc__sysrQr rKtimerGrbLIST_URLrfrHrgabspath__file__rhrMr.rr;rYr{__name__rrrs%$$ 1 h i '',,rwwrwwx'@ACW X D  $DN)XXv zFr