ujAdZddlZddlZddlZddlZddlZddlmZdZdZdZ dedZ d Z d ejvZ d Zd Zd ZdZdZdZedk(reyy)u;Crawler for naimanqi.gov.cn - 通知公告 /xwzx/tzgg_7894/N)datetimez%http://221.199.189.183/xwzx/tzgg_7894zwww.naimanqi.gov.cnu"奈曼旗人民政府-通知公告z]+>)resubstrip)htmls #/root/gov_crawler/crawl_naimanqi.pyget_textr s+ 66&#rvvj"d; < B B DDc tj|tdd}d|_|jdk(r |j SdS#t $r }Yd}~yd}~wwxYw)NF)headerstimeoutverifyzutf-8)requestsgetHEADERSencoding status_codetext Exception)urlres r crawlrsO LLgr% H #-qvv747 s>AA AAcg}tjd|tjD]}|jdj }|j drt dz|ddznt dz|jdz}|jt|jd||jdj f|S)NzL]*>.*?]*href="\.([^"]+)"[^>]*>(.*?)\s*]*>(.*?).//) rfinditerDOTALLgroupr startswithBASE_URLlstripappendr )r itemsmprs r parse_listr0s E [[hjnprpypy zF GGAJ   () T(:hnqu$3QRQYQYZ]Q^@^ hqwwqz*C1A1A1CDEF Lrctjd|tjr=ttjd|tjj dnd}dD]"}|j |dj }$tjd|tj}|s%tjd|tj}|r|j dj n|}d}tjd|}|r|j d}g}tjd|tjtjzD]}|j dj } |jt|j d | jd rtd z| d dzntd z| jd zf||||fS) Nz(.*?)r!r)u_奈曼旗人民政府u - 奈曼旗人民政府u 通知公告zT]*class="[^"]*lis_list_part2_content[^"]*"[^>]*>(.*?)\s*\s*zD]*class="[^"]*trs_editor_view[^"]*"[^>]*>(.*?)\s*z(20\d{2}[-/]\d{2}[-/]\d{2})zN]*href="(\./[^"]*\.(?:pdf|doc|docx|xls|xlsx|zip|rar|txt))"[^>]*>(.*?)r$r"r#) rsearchr'r r(replacer r& IGNORECASEr,r)r*r+) r titlesufr.contentpub_datem2attsamaus r parse_detailr= sVXV_V_`wy}@B@I@IWJHRYY6biiHNNqQ RPREU/ c2&,,./ ikoqsqzqz{A II]_cegenen o$%aggaj 5GH 14 8B bhhqk8 Dkkkmqsus|s|@B@M@MtMNB XXa[    Xbhhqk*r}}UYGZHsNRV,C`hkn`nqsqzqz{~q`A BB '8T ))rc tjt}|j}d}|D]\}}}}}} |j d|f|j r.| rddj d| Dzdznd} | r|dz| zn|} |j d ||| |rt|dd n||xs|tf|d z }|j|j|S) Nr'SELECT id FROM gov_raw WHERE page_url=?z
z
c34K|]\}}d|d|dyw)z zN).0tus r zsave..8s*6qdhdegh1#EWXYWZZ^7_6qsz
r z^INSERT INTO gov_raw(title,page_url,content,summary,publish_date,site_name) VALUES(?,?,?,?,?,?)rr!) sqlite3connectDB_PATHcursorexecutefetchonejoinr SITE_NAMEcommitclose) r-conncinsr5rdsr7pdr:ahfcs r saverX1s ??7 #D A C-2)sBT ;cVD ::<AE (6;;6qlp6q+q qt| |KM$&Wt^b G r#rg8G#4Tc#:5RTRZXZ\ef h q KKMJJL Jrc Jtrdnd}tdtd|dts"dgtddDcgc]}d |d  c}zndg}d }t |dD]2\}}t t d |sd n|}|s"t|}|sng}|D]\} } } tr]tjt} | j} | jd| fj}| j|rjt | }|sxt|\}}}}|j!|xs| | | |||ft#j$d|r#t'|}||z }td|d|dntd|dts,|dk\s3ntd|dycc}w)N INCREMENTALFULLz=== z [z] ===rr!Cindex_z.htmlrr#z index.htmlr?g?z Page z: +z newz : 0 itemsz === Done: z new ===)rZprintrNrange enumeraterr*r0rGrHrIrJrKrLrPr=r,timesleeprX)modeipagestipir/r r-pi_itemsrCrDdrQcuexh2dtctrUatns r mainrrAs'=VD D 2dV5 )*DORDuQ|rysuA''' 2  0 Wag h ) E*" %@ zFr