Gj7dZddlZddlZddlZddlZddlZddlZddlZddlm Z ejdddl m Z ddl Zejj!ejj#eZejj)deejj)dejj+edddZd Zd Zej2d d Zd ZdZdZddZdZdZ dZ!ddZ"e#dk(rOdZ$ejJddZ&e'e&D])\Z(Z)e)dk(s e(dze*e&ks e+e&e(dzZ$+e"e$yy#Y;xYw)uk 旌德县人民政府-重大行政决策预公开 爬虫 (Playwright) - 修复版 站点: www.ahjd.gov.cn N BeautifulSoupignore)sync_playwrightz..crawleru%旌德县-重大行政决策预公开zwww.ahjd.gov.cnzhttps://www.ahjd.gov.cn SEARCH_DBz/root/search.dbc |sytjt}|jd|jd|jd|jd|jd|j dx}}|D]} |j dxsddd }|j d xs|dd }|j d xsd}|j d xsddd }|j dxsd} |j dxs| } |j dxsd} |jdt | | ||||d| f |jdkDr|dz }n|dz }|j  |jdt f|j |jtd|d|y#|dz }Y?xYw#t$r} td| Yd} ~ Kd} ~ wwxYw)NzPRAGMA journal_mode=WALzPRAGMA busy_timeout=8000zPRAGMA synchronous=NORMALa"CREATE TABLE IF NOT EXISTS gov_raw ( id INTEGER PRIMARY KEY AUTOINCREMENT, site_name TEXT, source_url TEXT UNIQUE, page_url TEXT, title TEXT, publish_date TEXT, summary TEXT, content TEXT, status TEXT, category TEXT, tags TEXT, attachments TEXT )zCREATE VIRTUAL TABLE IF NOT EXISTS gov_search USING fts5( title, site_name, summary, content='gov_raw', content_rowid='id' )rtitleisummarycontent publish_date source_urlurl attachmentszINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, attachments) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)activezINSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.site_name=? AND r.id NOT IN (SELECT rowid FROM gov_search)z [FTS ERROR] u [DB] 新增: u , 跳过: ) sqlite3connectrexecutecommitget SITE_NAME total_changes Exceptionprintclose) itemsdbokskipitemr r r pub_daterpage_urlres crawl_ahjd.py insert_dbr(s   #BJJ()JJ)*JJ*+JJ  JJ  IIKMB XXg&,"ds3Exx *3eTc:Ghhy)/RG06B]*>.*?r flagsz]*>.*?z )resubDOTALL IGNORECASEstrip)htmls r' clean_htmlr2Msj 66,b$bii"-->W XD 66.D BMM@Y ZD 66-T ;D ::<c|sy|}tjd|tjtjz}g}t |D]1\}}d|d}|j ||f|j ||d}3tjdd|tj}tjd d |tj}tjd d|tj}tjd d|tj}tjd d|tj}tjdd|tj}tjdd|}tjdd|}tjdd|}tjdd|}tjdd|}tjdd|}tjdd|}tjdd |}tjdd|}tjdd|}|D](\}}t|}|j |d|zdz}*|jS)Nr z]*>.*?__TABLE___rz  r*z

 zzz <__THEND__>zz <__TDEND__>z<[^>]+>z  z<z&&\n{3,}z[ \t]+z-(?<=[\u4e00-\u9fff])[ \t]+(?=[\u4e00-\u9fff])) r,findallr.r/ enumerateappendreplacer-html_table_to_htmlr0)r1texttables placeholdersitblphmds r' extract_textrKTs   D ZZ2D"))bmm:S TFLF#(3s" RI&||CQ'( 66,d"-- @D 66'64r}} =D 66)T4r}} =D 66(D$bmm @H--h]*class="[^"]*m-detailbox[^"]*"[^>]*>(.*?)\s*(?:]*class="[^"]*(?:is-footer|footer))zI]*class="[^"]*m-detailbox[^"]*"[^>]*>(.*?)\s*\s*ru 文本下载rNz<(?:div|p|table)[^>]*>u8(?:用微信扫描二维码|分享至好友和朋友圈)u小]字体z*]*class="[^"]*col-md-12[^"]*"[^>]*>r9uA(?:用微信扫描二维码|分享至好友和朋友圈|打印).*r r*r7c3,K|] }|dv yw)rN).0xliness r' z.extract_content_from_detail..snaAqMns)u发布时间：u 来源：u 浏览次数r_r8z]*>(.*?)u无障碍浏览.*?注册uA主办单位.*|联系电话.*|邮箱.*|备案号.*|政府网站.*r>c3&K|]}|v ywNrc)rdrels r'rgz.extract_content_from_detail..s- Q!V- s)u首页u 走进旌德u 新闻中心u政府信息公开u 政务服务u 互动交流u 解读回应u 县长之窗u 无障碍u 长辈版u登录u注册u 网站地图u 联系我们u 版权声明u 页面纠错u 隐私保护)r,searchr.r/groupfindlenstartrKr2endr-splitr0anyjoin)r1m content_htmltdxafter_tdbody_m body_startend_m body_htmlrDcleanedfont_pos after_fontend_posrlbmbodyrfs ` @r'extract_content_from_detailrs |CEGENENQSQ^Q^E^ _A IIbdhjljsjsvxwDwDkD Ewwqz / 7#C#n*=$=$>?HYY8(CF 3~#66G  "]_klvlw_xy ,Z U[[]8R SI ,Z[ 9I#Jy$9:#J|$<=!.G||F+H2~"<<12~(Q,$X\]3 #PR\] *6::<= 9I'nnX6G{$-hw$7 ' 2D' 3D#G,vvZ\^`dlnluluv$(JJt$4Yq c!'')nWXFXYn3mnn!"IEn3mnn{{5! t9q=::<  -tRYY ?B xx{T"G$vv12t<vvZ\^`devvi.446$(JJt$4Yq c!'')nWXFXY! q- > - *   ;;u% % 9Z&Z s0+P P P  PP<PP0Pc |j|d|jdd|j}g}tjd|D]}|j d |j dj }|j}|jd|}d }|d k7r|||j }|d vst|d krtjd ||j|jdz} | r| j dnd } t fd|Dr|j|t z| d|S)u3用 Playwright 获取列表页并返回文章列表 Ntimeout networkidle'zI]*href="(/OpennessContent/show/\d+\.html)"[^>]*title="([^"]*)"[^>]*>rr`zr r^)u| 文件u文件r raz%]*>(\d{4}-\d{2}-\d{2})i,c3<K|]}|dtzk(yw)rN)BASE_URL)rdrGrNs r'rgz%fetch_list_with_pw..s>11U8x$.>s)r rdate)gotowait_for_load_stater r,finditerrnr0rrrorprmrtrAr) page list_page_urlr1rrvr a_enda_closeinner date_spandate_strrNs @r'fetch_list_with_pwrsFIImUI+]E: <<>D E [[T wwqz   "))FE* b=w'--/E . .#e*q. IIFQUUWUVUZUZU\]`U`Hab )29??1%>>> LL$  1: Lr3cV|j|d|jdd|j}d}tjd|}|rJ|j dj }tjdd |}|j }|r|}t|}||d S) u$用 Playwright 获取详情页内容rrrrNz([^<]+)ru"[-–—]\s*旌德县人民政府$r )r r ) rrr r,rmrnr0r-r)rrr1 detail_titlettt content_texts r'fetch_detail_with_pwrsIIc5I!]E: <<>DL ,d 3B HHQK    FF8"a @ GGI L.t4L r3ctddtdttdt5}|jj d}|j dd}|j }td td }t||}|r|nd }td t|d t|}td |dzD]~} td| d} t|| } |D chc]} | d } } | D cgc] } | d| vs | }} |j|td| d|dt| dt|d tdt|d dx}}g}t|dD]\} }td| dt|d|dddd t||d}|r|jdstd|dz }^|jdxs|d}|jdd }|j!||d|d|d!dd"|dd#|d d$|dz }td%t|d&|d!|j%dddtddtd(d)r t'|tdtd*td+|d,|ycc} wcc} w#t"$r}td'||dz }Yd}~nd}~wwxYw#1swYxYw)-Nr7z<============================================================u🚀 T)headlesszoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zzh-CN) user_agentlocaleu 📥 获取列表...z,/XxgkContent/showList/836/112385/page_1.htmlr`u 页1: u 条rz&/XxgkContent/showList/836/112385/page_z.htmlru 第 /u 页: u 条 (新增 )u 共获取: rz [z] r <z...r u ❌ 正文为空r rr)r rrrr r ru ✅ 正文: u 字, 日期: u ❌ 错误: u 📦 写入数据库... 成功: u , 失败: u✅ 完成! 共处理 u 条，成功 u ，失败 )rrrchromiumlaunch new_contextnew_pagerrrplistrangeextendr@rrrArrr() max_pagespbrowserctxr first_urlitems1 total_pages all_itemspnrrrG seen_urls new_itemssuccessfaildb_itemsr#detaildb_titlerr&s r'runrsZ Bvh- E)  VH  9a**##T#2!!I" ||~ &'j LM #D)4#,i!  V T*+L ;?+ _BJDRDNC&tS1E+45a55I5$)GqQuXY-FGIG   Y ' F2$a }F3u:,mCPYNK[[\] ^  _  c)n-T23$ A. GAt Cs!C N+2d7mCR.@-AE F -dDK@VZZ %:/1AID!::g.?$w-%zz)R8 %"&u+;$(L"$5'~+#%!1 L(9':.fWX- 6  s9v Bvh- ,WIZv FG( VH "3y>"2. TXSY Z[_6GB s+,  k99shB7K0. J<:K0 K KBK0%2KK0A4K K0< K0 K-K("K0(K--K00K9__main__rz--pages)r)r rk),__doc__sysosr,jsonrmathwarningsrTrfilterwarningsplaywright.sync_apir urllib.parserXpathdirnameabspath__file___HEREinsertrurDOMAINrgetenvrr(r2rKrCrrrr__name__max_pargvargsr@rGargrpintrcr3r'rsS2111!/ 125277<<tY78 3  $ BIIk#4 5 32l!H Od%P0G\T z E 88AB