ljddlZddlZddlZddlZddlZddlZej jddZddddZ dZ d Z d Z d Z d Zd ZddZdZdZdZddZedk(r"dej,vr edyedyy)NDB_PATHz/root/search.dbz]*style="height:80px[^"]*"[^>]*>.*?]*href=(["\'])([^"\']+)\1[^>]*title=(["\'])([^"\']+)\3[^>]*>.*?.*?]*style="color:\s*#[Bb]4[Bb]4[Bb]4[^"]*"[^>]*>(\d{4}-\d{1,2}-\d{1,2}))rtitledate)refinditerDOTALLgroupstripr%COL_URLr)htmlitemspatternmr"r*date_strfull_urls rextract_list_itemsr8#s EYG [[$ 2N''!*""$   "771:##%w0  LLE8L M N Lrcd}tjd|tj}|r |jdj }nFtjd|tj}|r|jdj }d}tjd|}|r|jdj }d}g}d}tjd|tj}|r|jd}|rtj d|tj tjzD]{}t||jd} tjd d|jd j } |j| xs| jd d | d }g} tj d|tjtj zD]} | jdjdr!| j| jdD| jd} tjd d| } tjdd| } tjdd| } | j } | s| j| dj| }|||||ttdS)Nz@]*style="color:#fff;font-size:30px[^"]*"[^>]*>([^<]+)rz(.*?)z%(\d{4}-\d{1,2}-\d{1,2})\s*\d{2}:\d{2}z
]*>(.*?)
zP]*href=(["\'])([^"\']*\.(?:pdf|doc|docx|xls|xlsx|rar|zip))\1[^>]*>(.*?)r'z<[^>]+>rr)namerz+]*>(.*?)

|]*>(.*?)rzr? content_htmlamlink_url link_textr$elemp_texts rextract_contentrN/so E UW[]_]f]fgA   " II-tRYY ? GGAJ$$&EH :DAA771:##%GKL 4dBIIFAwwqz ++qsBDBFBFIKIRIRBRS `B":rxx{;Hz2rxx{;AACI    (LX^^C5H5LU]^ _ `KK NP\^`^g^gjljnjn^no )Dzz!}''1 TZZ]+A B7 5sFC 2v6LL( )++e$w&Y PPrFcddl}g}t}tt}t |}t dtzt dt |z|D]/}|d|vs |j|d|j|1t dt |z|r|ddD]}t d|ddd zt|d}t||d}t d |dzt d |d zt d t |d|dddfzt dt |dz|dD]}t d|dd|dddy|jtd} | jd| j} d} d} t |} t|D]\}}| jd|df| jr| dz } 2t|d}t||d}|drt!j"|ddnd}|ddzt$z}|dr|d|dddzz }| jd|d|d||d xs|d |d|d ||d!|df | dz } | d"zdk(s| j't d#| | fz| j'| j)t d$| | | fzy)%Nrz [Page 1] %sz Found %d itemsrz Total unique items: %drz === Detail: %s ===r*2z Title: %sz Date: %sr+z Content (%d chars): %sr>z Attachments: %dr?z - r<z: P<)r zPRAGMA busy_timeout=30000z)SELECT id FROM gov_raw WHERE page_url = ?rF) ensure_asciir:r=zINSERT INTO gov_raw (title, content, summary, publish_date, page_url, site_name, attachments, group_name, source_url) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)r@r/r)z Progress: %d/%d inserted...z- Done. Inserted: %d, Existing: %d (total: %d))sqlite3setrr1r8printlenaddrrNconnectrexecutecursor enumeratefetchonejsondumpsrDcommitclose) test_moderU all_items seen_urlsr2r3itemresultattconncinsertedexistingtotalidxattachments_jsonsummarys rcrawlrqWsJII g D t $E -' !" s5z )*# ;i ' MM$u+ &   T "# $s9~ 56bqM GD (4="+== >d5k*D$T4;7F -&/1 2 ,/ 0 ,F94E0FyHYZ^[^H_/`` a %F=,A(BB Cm, GF SZ_EF G G  ??7B? /DLL,- AHH  NEy)G T =U ~N ::< MH $u+& tE{3TZ[hTi4::f]&;%Poq/C')3 )  sVI.t44 4G m G_fY/&.:XDQWLZ`afZgiop{i|OQWX_Q`bhinbo p qA  a<1  KKM 1Xu4EE F!G" KKMJJL :hRW=X XYr__main__z--testT)rc)zutf-8)F)r,sysosr_timer environr rrrDrEPER_PAGE MAX_PAGESrr1rr%r8rNrq__name__argvrrr|s((( **..$5 6PZ0  l   $ =  &PP2Zh z388   r