uj dZddlZddlZddlZddlZddlmZmZmZddlZddl Z ddlZe je jjdZ dZejddZdZej$ej&ed z j)d Zd d iZdZdZdZdZddZedk(rdej:vZerdndZe e de zdzezejZ!ee\Z"Z#e de$e%eje!z dzdze de$e"zdze$e#zyy) u 宜昌市生态环境局-环境影响评价 (hbj.yichang.gov.cn) ============================================ CMS: 宜昌市政府网站群（自研） 列表: list-42531-{page}.html, 每页15条 详情: content-42531-{id}-1.html,
含正文 用法: python3 crawl_yichang_hbj.py # 全量(最多55页+日期过滤) python3 crawl_yichang_hbj.py --test # 测试 5 条 N)datetime timedeltatimezoneu+宜昌市生态环境局-环境影响评价z,http://hbj.yichang.gov.cn/list-42531-{}.html SEARCH_DBz/root/search.dbiG)daysz%Y-%m-%dz User-Agentz>Bcbg}tjd|tjD]}|jd}|j ds|j drd|znd|z}|jdj }|jd}|j |||d |S) Nzm
.*?]*href="([^"]+)"[^>]*>([^<]+).*?
(\d{4}-\d{2}-\d{2})http/zhttp://hbj.yichang.gov.cnzhttp://hbj.yichang.gov.cn/)titlerpub_date)refinditerDOTALLgroup startswithstripappend)htmlitemsmrr'r(s r parse_listr3%s E [[ 6 bii I ggaj~~f%7:~~c7J-3PlorPrC   "771: uShGH I LcL tj|tdd}d|_|j}d}t j d|t j}|D]3}t jdd|j}|s+|dk7s1|}n|s7t jd |}|r|jd j}|jd }|d kry|jd d |} | d kry|| d} | jd} | d kryd } | } t| t| D]K}| ||dzd k(r!|dzt| k\s | |dzdvr| d z } /| ||dzdk(s;| d z} | d k(sF|dz} nd }d}t| t| D]Y}| ||dzd k(r!|dzt| k\s | |dzdvr|d z }/| ||dzdk(s;|d k(r| | |j}n|d z}[t jdd|t jt j z}t jdd|t jt j z}|j}||fS#t"$r$}t%dt'|ddzYd}~yd}~wwxYw)z2Extract full title + article body from detail pager Fr rz]*>(.*?)z<[^>]+>u宜昌市生态环境局z(.*?)r"zclass="txtcontent-div"rz z
z]*>.*?)flagsz]*>.*?z ! detail fail: P)r6r6)rrrrrr)findallr+subr.searchr,findrfindrangelenIrrr)rrr0 full_titleh1sh1tr2idx div_startsectionnry_posdepth after_nryi content_depthcontentjrs r fetch_detailrQ4s: LLgr% H vv jj/ryyA Bz2r*002AQ44     148AWWQZ--/ ii23 7JJvq#. q=yz",,34 Q; wG - Aq1~'QqSCL-@GAaCLT^D^ 1Q38+ A: !AI  y#g,/ #Aq1~'QqSCL-@GAaCLT^D^" 1Q38+ A%%i288:G"  #&&5r7"))TVTXTX.Y&&3R RTRVRVW--/7""  !CF3BK/0sLBI6I6 AI6!I6:I6AI6* I65AI6B$I66 J#?JJ#c J|syddl}|jt}|jd|jdd\}}|D]} |jdt|j dxsddd|j d d|j d d|j d xsddd dd f|j dkDr|dz }n|dz }|jdtf|j|j||fS#t$r}|dz }Yd}~d}~wwxYw)NrrrzPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALzxINSERT OR REPLACE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, tags) VALUES (?,?,?,?,?,?,?)r'r6irrOr( u环境影响评价r"zINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?) sqlite3connectrexecute SITE_NAMEr total_changesrcommitclose)r1rUdbokfailitrs rto_dbr`rs2   #BJJ()JJ*+HB  JJ)VVG_*DS1FF5"%FF9b)VVJ'-2s3( !#a ',JJ \   IIKHHJ t8O  AID s BD  D"DD"c  g}t}tdtdzD]}t|}|snt |}|st dt |zdznd}|D]F}|d|vr |j|d|dr |dtkr1|j||dz }H|r|ddnd}|r|ddnd} t dt |zd zt t|zd z|zd z| zd zt |z|dk(snt d t t|zdz|r(|dd}t dt t|zdz|syt|D]\} }t dt | dzzdzt t|zd z|dddzdzddt|d\} } | |d<| r| |d<t t t| dz| rdndzt|\} }| |fS)Nr"z [Page z] empty, reached endrrr(?] z items (z ~ z), new: z Total: z items within 3yrrz TEST mode: z itemsrSz [r$r'(z... r6T)endflushrOBz title_fixed)setr@ MAX_PAGESr r3rraddCUTOFFr/rA enumeraterQr`)test all_items seen_urlsrr0r1newitemflrMrCrOr]r^s rcrawlrusMIIaQ'$ 4  *s4y(+AA B  DE{i' MM$u+ &JD$4v$=   T " 1HC %*E!HZ s%*E"Ij ! j3t9$t+c#e*o= JQNQVVYZZ]ggjmnqjrrs !8 +, +C N+ +.A AB bqM  oC N 33h>? Y'Q4 ec!A#h$s3y>'::TADMRUSUDVVY__egost*4;7 G!Y &DM c#g,#%JBOP QYHB t8Or4__main__z--testTESTFULL[rd)rnz Time: r"sz New: z Skip: )F)&__doc__r)sysostimerrrrurllib3disable_warnings exceptionsInsecureRequestWarningrXrgetenvrrjnowutcstrftimerlrr r3rQr`ru__name__argvrnmodert0r]r^rroundr4rrsQ 22 ++BBC : ; BIIk#4 5 hll8<<(9%+@@ J J: VZ [ <|&P'R z sxx D6vD G # /D 4 '( B$HB *s5R34 4s :; )c"g  )CI 56r4