ujdZddlZddlZddlZddlZddlZddlZddlZddlmZm Z dZ dZ dZ ejddZd Zd Zd d d ZgdZdZdZdZdZdZedk(reyy)u1濉溪县-环评意见征集爬虫 (GET分页版)N)datetime timedeltau濉溪县-环评意见征集zhttps://www.sxx.gov.cnzOhttps://www.sxx.gov.cn/zwgk/public/column/1981?type=4&catId=4741481&action=list SEARCH_DBz/root/search.dbg?'z(.*?)z/href="([^"]+)"[^>]*class="title"[^>]*>(.*?)z!([^<]+)<[^>]+>)titlerdatezpageCount[^:]*:(\d+)r) refinditerDOTALLgroupsearchsubstripappendint) htmlitemsmliaspanr"rr#pc_m page_counts r parse_listr5s E [[;T299 ME WWQZ IIH"bii Xyy=rB FF:r1771:6<<>E''!*""$C,04::a=&&(bD LL5dC DE 99,d 3D'+TZZ]#J * c tj|tdd}d|_|j}d}tjd|}|r|jdj}|sLtjd |}|r4tjd d|jdj}d}tjd |tj}|r|jdj}|rt|d krFtjd |tj}|r|jdj}|||rddSddS#t $r}dt |dcYd}~Sd}~wwxYw)Nr Fr rr )contenterrorz)ArticleTitle["\' ]*content=["\']([^"\']+)rz([^<]+)u[-_—]\s*濉溪.*$z#class="wzcon[^"]*"[^>]*>(.*?)dzgkwz_container[^>]*>(.*?)empty)r8r"r9)rrrrrrstrr$r(r'r*r)r&len)rrr-rr"r/r8s r fetch_detailr>/sB0 LLgr% H vv E >EA   "  II/ 6 FF12qwwqzBHHJEG 8$ JA''!*""$ c'lS( II8$ J ggaj&&(Gw TTG TT+ 0A//0s0E## F, E?9F?Fc&tD]}||vsyy)NTF) EIA_KEYWORDS)r"kws ris_eiarBKs  ; r6c ttjdk\}|rttjdnd}t dt d|rdndt j}g}td}|s t dyt|\}}t d |td|dzD]}|dkDr5tjtt|}|s/t|\}}|D]W} t| d s| d r0 t j| d d } || z j } | |kDrF |j#| Yt d t|d|s t dyt%j&t(} | j+} t-}|D]=} | j/d| df| j1s*|j3| d?| j5|Dcgc] }|d|vs |}}t dt|dg}t7|D]\}} tjdt9| d}|j;drd|dvrt d|dzd|dY|j#|j;d xs| d | d| d |j;ddt=j>dd|j;ddjAd|dzdzdk(st d|dzdt|d d}t%j&t(} | j+} |D]E} | j/d!| d | dt | d"dd#| d| d f| jBdkDr|dz }G| jG| j5t d%|d|dkDrt%j&t(} | j+} |D]p} d}| j/d&| df| j1}|s.|d}t=j>dd| dxsddd'}| j/d(|t | d |fr| jG| j5t d)t d*|d+y#Y~xYwcc}w#tD$r}t d$|Yd}~d}~wwxYw),Nr!rrz=== z === u增量u全量u无法获取列表页u 总页数: r"r#z%Y-%m-%du 筛选出 u 条环评相关记录u 无新数据z-SELECT page_url FROM gov_raw WHERE page_url=?ru待抓详情: u 条g333333?r9r;z [u ] 失败: r8r r)r"rr#r8 content_plain r/u] 已抓取...zINSERT OR IGNORE INTO gov_raw (title, page_url, site_name, summary, content, publish_date) VALUES (?, ?, ?, ?, ?, ?)rDiu 入库失败: u入库: z'SELECT id FROM gov_raw WHERE page_url=?izVINSERT OR IGNORE INTO fts_search(rowid, site_name, title, summary) VALUES (?, ?, ?, ?)uFTS更新完成u=== 完成, 新增 u 条 ===)$r=sysargvr,r SITE_NAMErnowrr5rangetimesleep PAGE_DELAYrBstrptimedaysr+sqlite3connectDB_PATHcursorsetexecutefetchoneaddclose enumerater>rr$r)r*rowcountrcommit)is_incremental cutoff_daysrJ all_itemsr-r. total_pagesr_itemdt days_diffconncexistingitto_fetchsuccessidetailinsertedrridrowsummarys rmainrqQs]a'N&4#chhqk"$K D 5^ J KL ,,.CI 1 D  %&#D)E; K } %&aq)# !8 JJz ""4(D!$'HE1 #D$w-(F|!**4<DB!$rI ;. /   T " ##* Js9~&&< =>  n ??7 #D AuH& ADK>R ::< LLe %& JJL&Dr"U)8*CDHD N3x=/ ./GX&=4 3d5k* ::g 7&/#A C!uJvg&78 9 ZZ(9DM;Lzz)R0VVJFJJy"4MNTTV    ER<1  C!uAc(m_N; <="H ??7 #D A * * II-gU Yo&t,d9otF|M N zzA~A  * KKMJJL HXJd #$!|w' KKM CDC II?$u+ P**,C!f&&Ri1FBGN ry$w-AC C     z 23[$ED * $QC( ) ) *s1-R/: R7R7AR</R4< SSS__main__)__doc__jsonr$osrLrGrrQrrrIBASE_URLrgetenvrSrN CUTOFF_DAYSrr@rr5r>rBrq__name__r6rr{s71111( * # \ "))K!2 3   Q(  n  U8 o4b zFr6