ujRdZddlZddlZddlZddlZddlmZmZmZddlm Z ddl Z ddl Z e je jjdZdZej"ddZej&ej(ed z j+d Zd d iZdZdZddZdZdZdZdZdZe dk(reyy)u 贵溪市人民政府-环境审批 (www.guixi.gov.cn) ======================================== CMS: 大汉版通 (Hanweb) - JPage AJAX 列表API: /module/web/jpage/dataproxy.jsp (POST) 详情: /art/YYYY/M/D/art_COLUMNID_ARTICLEID.html,
含正文 N)datetime timedeltatimezone)quoteu贵溪市-环境审批z6http://www.guixi.gov.cn/module/web/jpage/dataproxy.jsp SEARCH_DBz/root/search.dbiG)daysz%Y-%m-%dz User-Agentz>>JKRRS[\\JT+E3&*cI  MM)c/J6YPWac d vv  )00s1vcr{CDs99B33 C-<'C((C-cg}tjd|D]n}|jd}|jd}|jdj}|j dsd|z}|j |||dp|S)NzL(\d{4}-\d{2}-\d{2})\s*]*href="([^"]+)"[^>]*title="([^"]*)"r httpzhttp://www.guixi.gov.cn)titleurlpub_date)refinditergroupstrip startswithappend)htmlitemsmr<r;r:s r4 parse_listrF/s E [[W  I771:ggaj   "~~f%+c1C uShGH I Lctdd}|sytjd|}|rt|j dSdS)Nr rztotalrecord>(\d+))r5r=searchintr?)rCrEs r4 get_totalrK?s> 1a D  &-A3qwwqz?&Q&rGc< tj|td}d|_|j}|j d}|dkr|j d}|dkry|j dd|}|dkry||d }d}d}tt|D]K}|||d zdk(r!|d zt|k\s ||d zd vr|d z }/|||d zdk(s;|d z}|dk(sF|d|}ntjdd|tjtjz}tjdd|tjtjz}|jS#t$r0} tdj!t#| d dYd } ~ yd } ~ wwxYw)z using depth countingr)rrrzid=zoomrz id="zoom"z r z
z]*>.*?)flagsz]*>.*?z ! detail fail: {}r)r#getr&r'r(findrfindrangelenr=subDOTALLIr@r)r*r r+) r;r2rCidx div_startsectiondepthcontentir3s r4 fetch_detailr`Gs LLgr : vvii " 7))K(C 7JJvq#. q=yz"s7|$ Aq1~'QqSCL-@GAaCLT^D^ 1Q38+ A:%alG &&5r7"))TVTXTX.Y&&3R RTRVRVW}}  #**3q6#2;78s2AE"E"7AE" E"BE"" F+&FFc |syddl}|jt}|jd|jdd\}}|D]} |jdt|j dxsddd|j d d|j d d|j d xsddd dd f|j dkDr|dz }n|dz }|j|j||fS#t$r}|dz }Yd}~d}~wwxYw)N)rrrzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALzwINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, tags) VALUES (?,?,?,?,?,?,?)r:rMir;r^r< u 环境审批r ) sqlite3connectrexecute SITE_NAMErR total_changesr)commitclose)rDrcdbokfailitr3s r4to_dbrnhs   #BJJ()JJ*+HB  JJ)VVG_*DS1FF5"%FF9b)VVJ'-2s3" !#a ',IIKHHJ t8O   AID s BC33 D <DD c \tdjttdjtt }tdj||dk(r tdyd}||zdz |z}g}t dt |ddzD]}t|}|snt|}|std j|nyd}|D]*}|d r |d tkr|j||dz },|r|dd nd } |r|d d nd } td j|t|| | ||dk(sntdjt||s tdyt|D]l\} }tdj| dzt||dddddt|d} | |d<tdjt| nt|\} }tdtdj| |y)Nz === {} ===u日期阈值: {}u总记录数: {}ru 无记录rr z [Page {}] emptyr<rz. [Page {}] {} items ({} ~ {}), within 3yr: {}u 共{}条待抓取详情u 无需抓取z [{}/{}] {}... r:(rMT)r/flushr;r^z{}Bu === 完成 ===u新增: {}, 跳过: {})r*r rfCUTOFFrKrUminr5rFrBrV enumerater`rn)totalr- total_pages all_itemsr,rCrDnewitemflr_r^rkrls r4mainr~s, ,  i ()  # #F +, KE  # #E *+ z kH8#a'H4KIa[!,q01t$ 4  %,,T2 3  DJD$4v$=   T " 1HC   %*E!HZ s%*E"Ij ! >EEdCPUJXY[\^abc !8 %( % , ,S^ <=  nY'*4  ''!S^T']3B=OPVX`detE{+!Y ell3w<() * YHB  " ) )"d 34rG__main__)r)!__doc__r=sysostimerrr urllib.parserr#urllib3disable_warnings exceptionsInsecureRequestWarningrfr%getenvrnowutcstrftimertr&r!r"r5rFrKr`rnr~__name__rGr4rs22++BBC % E BIIk#4 5 hll8<<(9%+@@ J J: VZ [  ,  'B F15h zFrG