uj{rdZddlZddlZddlZddlZddlZddlZddlmZmZddl m Z ddl Z e jddZ dZdZd Zej"ed z j%d Zd ZddiZej,Zej0j3edZdZdZedk(reyy)u]Crawl 云南湖柏环保科技 - 行业新闻 http://www.ynhbhb.com/index.php?list_1_33.html N)datetime timedelta) BeautifulSoup SEARCH_DBz/root/search.dbu!云南湖柏环保 - 行业新闻zhttp://www.ynhbhb.comz.http://www.ynhbhb.com/index.php?list_1_33.htmliG)daysz%Y-%m-%dz User-Agentz(.*?)rruB\s*-\s*行业新闻\s*-\s*云南湖柏环保科技有限公司\s*$r divshowinfo)class_scriptstyle)sessiongetencodingtextr#r$DOTALLIr%rr'rr!r decomposestr) rrespr,tmrrawr.contags r4 fetch_detailrN>s{{3{+ yy *D"))BDD. AB E hhqk!\^`beflln } -D ))E*) -C <<7 34 C MMO c#hnn&&& "9's *DDcdtjv}tdtdt j t }|j}d\}}}|rdnt}d}d}tj} td|dzD]3} | dk(rt} n td| d } tj| d } d | _| j } t'| }|std| dnt)d|D}|r!|dz }|dk\r| |k\rtd| dnd}|D][}|dr|dt*kr|dz }|j-d|dfj/}|r|dz }G t1|d\}}|xs|d}|d}t3t5j6|dj9j;ddddz}|rt3|j=ddnd}|rt?j@dd|ddn|}t?j@d d!|jC} |j-d"|t|d|d|||||f |jDdkDr|dz }tj$d$^tj| z }td| d%|d&|d'|d(|d)|d*d+ 6|jG |j}|j-d,tf|j-d-tfjI}|D]}|j-d.||jGtd/tK|d0|jMtj| z }td2|d*d3|d4|d5|y#t"$r1}td | d|tj$dYd}~Jd}~wwxYw#t"$r3}td|tj$d|dz }Yd}~d}~wwxYw#t"$r}td#||dz }Yd}~d}~wwxYw#t"$r}td1|Yd}~d}~wwxYw)6N incrementalz=== z ===)rrrrrrz/index.php?list_1_33_z.htmlr8r9r;z [ERROR] Page z: rz Page z: 0 items (end)c3NK|]}|ds |dxr |dtkyw)rN) CUTOFF_DATE).0items r4 zmain..xs.ddW[\bWcd6lAtF|k'AAds %%z: all old, stoppingrzNSELECT 1 FROM gov_raw WHERE page_url=? AND content IS NOT NULL AND content!=''rz [ERROR] Detail: rlrrz<[^>]+>z\s+ zINSERT OR IGNORE INTO gov_raw(id, site_name, source_url, page_url, title, publish_date, content, summary, date_rank) VALUES(?,?,?,?,?,?,?,?,?)z [DB] g333333?rz: +z (skipped z, >3y z) [z.0fzs]zPDELETE FROM gov_search WHERE rowid IN (SELECT id FROM gov_raw WHERE site_name=?)zCSELECT id, title, site_name, summary FROM gov_raw WHERE site_name=?zRINSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES(?,?,?,?)z FTS: z records rebuiltz FTS error: z Done (z s). Inserted z , Skipped z, Old )'sysargvprint SITE_NAMEsqlite3connectDB_PATHcursor MAX_PAGEStimerangeLIST_URLr)rArBrCrD Exceptionsleepr5allrRexecutefetchonerNr&hashlibmd5encode hexdigestreplacer#r'rrowcountcommitfetchallr close)is_incrementalconncinsertedskipped old_total max_pagesconsecutive_old min_pages start_timepglist_urlrIraw_htmler-all_oldrThasrcontent final_title final_dateint_id date_ranksummaryelapsedc2rowsrs r4mainrXs"chh.N D 4 ! ??7 #D A#* Hgy#IOIJAy1}%Cm 7H"#8EBH ;;x;4D#DMyyH 8$ GB4/ 0 dedd  q O!#it#678O# DF|V { :Q ))lostyozn|}GGIC1  !-d5k!:w 04=KfJT%[%7%7%9:DDFsKRPTYZFMH JJsOG# J))+ * t1YKs8*Jwivi[X[\cdg[hhjklGCmJ KKM # [[] ehqgstzz_bkamnwwy pA JJkmn o p  D {"234 JJLiikJ&G HWSMxj 7)6R[Q\ ]^S  OB4r!- . JJqM  > *1#./ 1 1   ( sm$1  $ # aS!""#sa#*N-O39PB Q O$&OO P!(PP P> P99P> Q# QQ#__main__)__doc__requestsr#r^rZrcrkrrbs4rosgetenvr`r]r)renowstrftimerRrbHEADERSSessionrAheadersupdater5rNr__name__r6r4rs100( "))K!2 3 / " ;x||~ u 55?? K P  (   w%N4__B zFr6