ujxdZddlZddlZddlZddlZddlZddlmZmZddlm Z m Z ddl m Z ddl Z dZdZedd d Zej"j%d d Zd ZdZddiZdZdZdZdZdZedk(reyy)uO洪湖市-生态环境-通知公告 爬虫 (Playwright翻页 + requests详情) N)datetimedate)ThreadPoolExecutor as_completed)sync_playwrightzhttp://zwgk.honghu.gov.cnzAhttp://zwgk.honghu.gov.cn/list.shtml?column_id=38559&page_index=1iDB_PATHz/root/search.dbu#洪湖市-生态环境-通知公告z User-Agentz Mozilla/5.0c 2g}t5}|jjdddg}|j}|j t ddt jdtd td zD]}|jd }|D]_}|jd }|rd |vr|j|,|s/|jd rAt|z}d |vsO|j|atd|dt!|dd|tks|j#d|d zd} | s|j#d} | r&| j%t jdn|j'ddd|S#1swY|SxYw)u%使用Playwright翻页收集所有URLTz --no-sandboxz --disable-gpu)headlessargsdomcontentloadedi0u) wait_untiltimeoutz#documentsUL li ahrefz honghu.gov.cnhttpu 第u页: u 条总URLflushz a[curpage="z"]za[curpage="2"]N)rchromiumlaunchnew_pagegotoCOL_URLtimesleeprange TOTAL_PAGESquery_selector_all get_attributeappend startswithBASEprintlenquery_selectorclickclose) all_urlspbrowserpagepgitemsarfull_urlnext_btns !/root/gov_crawler/crawl_honghu.pycollect_all_urls_via_playwrightr7s~H  a**##T8Y#Z! '&8% H 1 ;?+ B++,?@E 2v.Ot3OOD)$//&"9#d{H&(2 1 2 E"U3x=/ O?> Os%B4F F  F "9F A&F  Fcxdtj|zdz}tj||}|sy|j}d}|}|t |dz krL|dkDrG|||dzdk(r||dzd vr|dz }n|||d zd k(r|dz}|dz }|t |dz kr|dkDrG|||d z j S) u0深度计数法提取指定class的div内部HTMLz]*class="[^"]*z [^"]*"[^>]*>rr rrz   rz)reescapesearchendr)strip)html class_namepatternmstartdepthis r6extract_content_by_depthrK:s':)>>PG '4 A  EEGE E A c$i!m  !A#;& T!A#Y2N%N QJE !AaC[H $ QJE Q c$i!m  ac? ""c tdD]} tj|dt}d|_|j dk7s |j stjdY|j }d}tjd|}|r|jd j}|s7tjd |}|r|jd j}d D]1}|j|s|dt| j}3t|d }|r>tj d d|}tj dd|}|j}|s t|d}|s t|d}|sFtjd|tj"}|r|jd j}d}tjd|}|r|jd }|s)tjd|}|r|jd }|s)tjd|}|r|jd }|sPtjd|}|r8d|jd d|jdd|jd}||||fcS|dddfS#t$$r%} |dkrtjdYd} ~ d} ~ wwxYw)N)rheaderszutf-8rr9z#

(.*?)

rz(.*?))u-洪湖市人民政府u_洪湖市人民政府u--政府信息公开z zw-center-txtz z]*>\s*\s*z bt-contentcontentz1
]*>(.*?)
\s*z(zw-box-infos[^>]*>.*?(\d{4}-\d{2}-\d{2})u)发布时间[：:]?\s*(\d{4}-\d{2}-\d{2})z(\d{4}-\d{2}-\d{2})z/t(\d{2})(\d{2})(\d{2})\d*/20-)r!requestsgetHEADERSencoding status_codetextrr r?rAgrouprCendswithr)rKsubDOTALL Exception) urlattrrDtitlerGsuffixrRpub_datees r6process_detailrgMsQx>=  S"g>A AJ}}#166 1 66DE @$GA ((*II5t<GGAJ,,.Ef 9>>&)!-CK<0668E 9 /t_EG&&G<&&!:BH!--/24F24CIIRTXZ\ZcZcdggaj..0GH EtLA771:IIJDQ wwqzHII4d; wwqzHII~ tT "" Qw 1  s&AJ+%BJ+:F'J++ K4KKc tjjdd}tdtd|ddtddt }td t |d d|std dytjt}|j}d }d }tt 5}|Dcic]}|jt||}}t|D]} | j!\}} } } |dz }|dzd k(rtd|dt |dd| E| s| } td| ddd| sd} t#j$| dd} | t&kr |j)dt| || | dd| d f|j*d kDr|dz }|dzd k(s|j/ ddd|j/|j)dtf|j1\}}}|j3tddtd|d dtd|d|d|d dycc}w#YxYw#t,$r}td|dYd}~d}~wwxYw#1swYxYw)!NMODEfullz=== z (mode=z) ===TruPlaywright收集URL...u 总URL: u条u 无数据r) max_workersru 详情 /z...u [WARN] 无正文: (z 2026-01-01 zINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, date_rank) VALUES (?, ?, ?, ?, ?, ?, ?)z [DB_ERROR] 2zTSELECT COUNT(*), MIN(publish_date), MAX(publish_date) FROM gov_raw WHERE site_name=?u === 完成 ===u 新增: u 累计: u条 (z ~ ))osenvironrVr( SITE_NAMEr7r)sqlite3connectr cursorr MAX_WORKERSsubmitrgrresultr fromisoformat CUTOFF_DATEexecuterowcountr_commitfetchoner,)modeurlsconnrw total_newdoneexecutorr`fut_mapfutrcrRredrfcntmin_dmax_ds r6mainrs ::>>&& )D D 74& .d; "$/ * ,D Js4yk %T2  m4( ??7 #D [[]FI D  4HLM8??>37<MM( C,/JJL )C AIDbyA~ $qT 37tD},U3BZL9F' &&x}5{?# 7 hsGXcr]GQOQ??Q&NIbyA~ 5 : KKM NNilukwx)CJJL T* Jyk %T2 Jse5s5' 34@GN"   7 aS)66 73sg7I7<IA-I7!I )I7,4I I7)I7I7 I I7 I4I/*I7/I44I77J__main__)__doc__jsonr?rrrrurrconcurrent.futuresrrplaywright.sync_apirrUr'rr|rsrVr rtrxrWr"r7rKrgr__name__rLr6rs"""#?/" M4B **..$5 6 1   ' "J#&@#F4An zFrL