Hj'.dZddlZddlZddlZddlZddlZddlZddlmZddl Z ejdejjejjeZejj#deejj#dejj%edddZdZd Zej,d d Zd Zej2Zej6j9d ddddZdZdZddZ dZ!dZ"dZ#ddZ$e%dk(rOdZ&ejNddZ(e)e(D])\Z*Z+e+dk(s e*dze,e(ks e-e(e*dzZ&+e$e&yy#Y;xYw)un 薛城区人民政府-审批信息 (枣庄市生态环境局薛城分局) 爬虫 站点: www.xuecheng.gov.cn N BeautifulSoupignorez..crawleru薛城区-审批信息zwww.xuecheng.gov.cnzhttp://www.xuecheng.gov.cn SEARCH_DBz/root/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagec |sytjt}|jd|jd|jd|jd|jd|j dx}}|D]} |j dxsddd }|j d xs|dd }|j d xsd}|j d xsddd }|j dxsd} |j dxs| } |j dxsd} |jdt | | ||||d| f |jdkDr|dz }n|dz }|j  |jdt f|j |jtd|d|y#|dz }Y?xYw#t$r} td| Yd} ~ Kd} ~ wwxYw)NzPRAGMA journal_mode=WALzPRAGMA busy_timeout=8000zPRAGMA synchronous=NORMALa"CREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT, source_url TEXT UNIQUE, page_url TEXT, title TEXT, publish_date TEXT, summary TEXT, content TEXT, status TEXT, category TEXT, tags TEXT, attachments TEXT )zCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5( title, site_name, summary, content='gov_raw', content_rowid='id' )rtitleisummarycontent publish_date source_urlurl attachmentszINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, attachments) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)activezINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)z [FTS ERROR] u [DB] 新增: u , 跳过: ) sqlite3connectrexecutecommitget SITE_NAME total_changes Exceptionprintclose) itemsdbokskipitemr r rpub_daterpage_urlres crawl_xuecheng.py insert_dbr)s   #BJJ()JJ)*JJ*+JJ  JJ  IIKMB XXg&,"ds3Exx *3eTc:Ghhy)/RG06B]*>.*?r flagsz]*>.*?z )resubDOTALL IGNORECASEstrip)htmls r( clean_htmlr3Tsj 66,b$bii"-->W XD 66.D BMM@Y ZD 66-T ;D ::<c|sy|}tjd|tjtjz}g}t |D]1\}}d|d}|j ||f|j ||d}3tjdd|tj}tjd d |tj}tjd d|tj}tjd d|tj}tjd d|tj}tjdd|tj}tjdd|}tjdd|}tjdd|}tjdd|}tjdd|}tjdd |}tjdd|}tjdd|}|D](\}}t|}|j |d|zdz}*|jS)Nr z]*>.*?__TABLE___rz  r+z

 zzzzz5<(?!/?(?:table|tr|td|th|thead|tbody|tfoot|img))[^>]*>z  z<z&&z\n{3,}z[ \t]+z-(?<=[\u4e00-\u9fff])[ \t]+(?=[\u4e00-\u9fff])) r-findallr/r0 enumerateappendreplacer.html_table_to_htmlr1)r2texttables placeholdersitblphmds r( extract_textrJ[s  D ZZ2D"))bmm:S TFLF#(3s" RI&||CQ'( 66,d"-- @D 66'64r}} =D 66)T4r}} =D 66(D$bmm @H--h]*>(.*?)rz/]*href="([^"]*)"[^>]*title="([^"]*)"[^>]*>u*]*>(\d{4}年\d{1,2}月\d{1,2}日)z(\d{4}-\d{2}-\d{2})r u"(\d{4})年(\d{1,2})月(\d{1,2})日-rN)r rdate)BASE_URLrTsessionpostencodingrCrrr-finditerr/groupsearchr1matchzfillr@rV)page_numrr^rr2r'r mlilinkrMr date_mdate_strraw_datem2s r(fetch_list_pagerws JR SC CM "D LL4L 4 vv E [[/ryy A  WWQZyyKRP zz!} 1 ##%H"MYY5r:F ||AH?JB hhqk]!BHHQK,=,=a,@+A288A;CTCTUVCWBXY# ??6244  - 6 LC  (,- s+F// G8G G Gc2tjd|tjtjz}|s6tjd|tjtjz}|rC|j d}t t |}t|dkDr|jStjd|tjtjz}|rAt t |j d}t|dkDr|jStjd|tj}|r|j d}t |}t |}|jdDcgc]@}|jst|jdkDs1|jB}}|Dcgc]tfd d Dr}}|rd j|Sy cc}wcc}w) u从详情页提取正文z_]*class="[^"]*(?:TRS_UEDITOR|trs_paper|trs_web)[^"]*"[^>]*>(.*?)\s*\s*zM]*class="[^"]*(?:TRS_UEDITOR|trs_paper|trs_web)[^"]*"[^>]*>(.*?)rrz]*class="[^"]*(?:zwnr|detail-page|article-content)[^"]*"[^>]*>(.*?)(?:\s*){1,3}(?:]*class="[^"]*(?:foot|footer|share|print)z]*>(.*?)r8c3&K|]}|v ywN).0xls r( z.extract_content_from_detail..s- Q!V- s) u首页u 信息公开u 政务公开u 网站地图u 联系我们u版权u 无障碍u登录u注册r9r ) r-rkr/r0rjrJr3lenr1splitanyjoin) r2rp body_htmlrCbmbodycleanedrliness ` r(extract_content_from_detailrs tvz|~}F}FIKIVIV}V WA IIfhlnpnwnwz|{H{HoH IGGAJ Jy12 t9r>::<  egkmomvmvy{yFyFmF GAJqwwqz23 t9r>::<  -tRYY ?B xx{T"G$$(JJt$4Yq c!'')nWXFXYY! q- > - *    ;;u% % Z sH%HHH5Hc< tj|d}d|_|j}d}t j d|}|r|jdj}|sbt j d|}|rJ|jdj}t jdd |}|j}|r|}d }t j d |}|r|jddd }t|} || |d S#t$r }Yd}~yd}~wwxYw) u获取详情页r)r_r`Nz3]*name="ArticleTitle"[^>]*content="([^"]*)"rz([^<]+)u([-–—]\s*薛城区政府信息公开$r z.]*name="PubDate"[^>]*content="([^"]*)"r)r rrd) rfrrhrCrr-rkrjr1r.r) detail_urlror2r'r tttrtdprs r( fetch_detailrs KK BK / vv E I4 PB  !!#  YY0$ 7  !!#ABBJA AH Dd KB 88A;s#)$/Gw AA1 s*D DDc rtddtdttdtdtd}|s tdytdt|dd }|tzdz tz}|r||kr|}td |d |t |}t d |dzD]}t|}|std |d|d#|Dchc]}|d }}|Dcgc] }|d|vs |} }|j| td |d|dt|dt| d tdt|ddx} } g} t|dD]\}} td|dt|d| ddddt| d}|std| dz } K|jdd}|rt|dkrtd| dz } ~|jdxs| d}|jd xs| d }| j|| d| d|dd!|dd"|dd#| dz } td$t|d%|tddtd&| d'| | r t| tdtd(t|d)| d*| ycc}wcc}w)+Nr8z<============================================================u🚀 u 📥 获取列表页1...ru ❌ 列表页无数据u 第1页: u 条iu 总记录: u , 总页数: rau 第 /u 页: 空ru 页: u 条 (新增 )u 共获取: rz [z] r <z...u ❌ 请求失败rr u ❌ 正文为空rdr)r rrrr rru ✅ 正文: u 字, 日期: u 📦 写入数据库... 成功: u , 失败: u✅ 完成! 共处理 u 条，成功 u ，失败 ) rrrwr PAGE_SIZElistrangeextendr?rrr@r)) max_pagesitems1 total_records total_pages all_itemspnr rF seen_urls new_itemssuccessfaildb_itemsr$detail content_textdb_titler%s r(runrs! Bvh- E)  VH &' Q F  () KF }D )*M 9,q0Y>KY,  M- k] CDV IA{Q'[# F2$a }I6 7 '01!QuX1 1 %C15)BQC C# rd!K=s5zl-IGWWXYZ[ OC N+4 01GdHY*M4 A3aI'r$w-*<)=SABd5k* ' ) AID zz)R0 s<014 ' ) AID ::g&7$w-::f%5fu+;$SbM~#   1  L 12. KL7M: Bvh- ,WIZv FG( VH "3y>"2. TXSY Z[[2Cs J/+ J48J4__main__rz--pages)r)r r{).__doc__sysosr-rwarningsrequestsrSr urllib.parserWfilterwarningspathdirnameabspath__file___HEREinsertrrDOMAINregetenvrrSessionrfheadersupdater)r3rJrBrwrrr__name__max_pargvargsr?rFargrintr|r4r(rs0//! 125277<<tY78 $  ' BIIk#4 5  (   DO'32l@ )X FBBH\V z E 88AB