ujZdZddlZddlZddlZddlZej j ddddlmZm Z dZ dZ dZ dZ d Zejd d ZddlZddlZdd Zd ZdZdZdZdZedk(r8ejZeedejez ddyy)u 孟州市 (mengzhou.gov.cn) - 公示公告 https://www.mengzhou.gov.cn/xwdt/gsgg/ 分页: index.html (p1), index_2.html (p2), ... index_{N}.html 详情: /YYYY/MM-DD/NUMBER.html Nz/root/gov_crawler) fetch_page clean_htmlu孟州市公示公告zhttps://www.mengzhou.gov.cnz%https://www.mengzhou.gov.cn/xwdt/gsggz 2023-06-01 SEARCH_DBz/root/search.dbc|s tdytjt}|j d|j dd\}}|D]} |j d|j dddd|j d d|j d d|j d xsddd |j d xsddd |j dxsddd |j dddd|j dxsdddf |j dkDr|dz }n|dz }|j|j dtf|j|jtd|d|dy#|dz }Y;xYw)Nu ⏭ 无数据zPRAGMA journal_mode=WALzPRAGMA synchronous=NORMAL)rrzINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) site_nameurltitleipub_date summarycontentactivetagsdrzINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?u 💾 入库: 新增u, 跳过u, FTS已同步) printsqlite3connectrexecuteget total_changescommit SITE_NAMEclose)itemslabeldbokskipitems #/root/gov_crawler/crawl_mengzhou.py insert_to_dbr%s     #BJJ()JJ*+HB  JJ5R(#.r"r"'"(b$3/*%+Sb1)$*DS12&"txx/52t< 7 !#a %*IIKJJILUKWXIIKHHJ !"XdV> BC   AIDs CFF c6|dk(r tdStd|dS)Nrz /index.htmlz/index_z.html) LIST_PREFIX)pages r$ get_list_urlr)9s) qyk**]'$u --c.g}tjdtj}|j|D]Z}|j d}|j dj }|j d}|sG|j |||f\|S)u)提取列表页的 (title, url, date_str)u]*class="colRtitle"[^>]*href="(https://www\.mengzhou\.gov\.cn/\d{4}/\d{2}-\d{2}/\d+\.html)"[^>]*>·\s*(.+?)\[(\d{4}-\d{2}-\d{2})\]r)recompileDOTALLfinditergroupstripappend)htmlrpatternmhrefr date_strs r$ parse_listr:>s Ejj C G   d #2wwqz   "771:  LL%x0 1 2 Lr*cd|d|d}tj||}|sy|j}d}|}|t|kr|dkDrtjd|d||d}tjd|d ||d} | snh|r:|j | j kr|dz }||jz }n|dz}|| jz }|t|kr|dkDr|||d z S) u<提取指定class的元素内HTML，正确处理嵌套标签]*>Nrrz(\s|>)z)r.searchendlenstart) r5tag class_namer6r7rBdepthi next_open next_closes r$ _extract_divrIOs3%z*W5G '4 A  EEGE E A c$i-EAIII3%v.QR9 YY"SE|T!"X6   *Z-=-=-?? QJE  A QJE ! !A c$i-EAI ac?r*cLt|d}|syd}tjd|tj}|r6|j dj }tj dd|}d}tjd|tj}|r|j d}d}t|d d }|r|j }tj d d|tjtjz }tj d d|tjtjz }tj dd|}tjd|tj}|r|j d}t|}|xsd|xsd|xsdfS)u*获取详情页的标题、正文、日期utf-8encoding)NNNr z(.*?)ru(\s*[-–—]\s*孟州市人民政府\s*$zE]*name=["\']PubDate["\'][^>]*content=["\'](\d{4}-\d{2}-\d{2})divzcms-article-boxz]*>.*?)flagsz]*>.*?z\s*style="[^"]*"z'
(.*?)
N) rr.r?r0r2r3subIrIr)r r5r r7r rrawdetail_ms r$ fetch_detailrTesX cG ,D  E )4;A   "BBNH Z\`bdbfbfgA771:G tU$5 6C ))+&&5r7"))TVTXTX.Y&&3R RTRVRVW&&,b':99GRTR[R[\ nnQ'GW% =D'/T8+;t ;;r*c Ttdddtddg}tdtdzD]}t |}td|d|ddd t |d }|s td n`t |}|s tdnFtdt|d|j|t|dkstdntdt|d|s tdygt}}t|dD]\}\}}} ||vr|j|| r | tkr,td|dt|d|ddddd t|\} } } | r| }| r| } tjdd| xsdj!dd} tjdd| } |j#t||| xsd| xsd| tdtd| t%j&d|r t)|d td!t|dy)"N z2==================================================u 🏠 ru 📄 第 u 页 (z)... T)r@flushrKrLu❌ 请求失败u0 条u✅ u 条u (末页)u 📊 列表总计: u⚠️ 无数据z [/z] 2z...z<[^>]+>r i,z\s+)rr r rr rrg333333?mengzhouu ✅ 完成! 共 )rrrange MAX_PAGESr)rr:rAextendset enumerateaddTHREE_YEARS_AGOrTr.rPr3r4timesleepr%)all_listr(r r5r all_itemsseenrFr r9dtrr rs r$mainrjs4 Bvhgi[6( 34HaQ'4  D6uD1s$G#0 $ % 4  'N  SZL%& u:? ,  " !#h- 56   !#%tI%.x%;! !E3 $;   ?2  A3aH bs C8cN ,S 1GX E H&&S'-R8>>@#F&&g."U3}"(.b   XJ  3/4Y + s9~.d 34r*__main__u ⏱ 耗时: z.1fs)mz)__doc__sysosr.rdpathinsert crawler_librrrBASEr'r^rcgetenvrrr%r)r:rIrTrj__name__t0rr*r$rys &'. # $5  BIIk#4 5  !DF. ",<>85t z BF LR,A ./r*