uj d dl Z d dlZd dlZd dlZd dlZd dlmZmZ d dlmZm Z d dl
m Z dZ dZ
eddd Zej j! d
d Zd Zd
ZddiZd Zd Zd Zd Zedk( r e yy) N)datetimedate)ThreadPoolExecutor as_completed)sync_playwrightzhttps://www.zh.gov.cnz2https://www.zh.gov.cn/col/col1229821392/index.htmli DB_PATHz/root/search.dbu) 镇海区-生态环境分局-通知公告 z
User-Agentz Mozilla/5.0c X t j d| }|sy|j }d}|}|t | dz
k rW|dkD rR| ||dz dk( r| |dz dv r| |dz d k7 r|dz
}n| ||d
z d k( r|dz }|dz
}|t | dz
k r|dkD rR| ||d
z
j S ) u9 深度计数法提取 bt-content zoom clearfix 内部HTMLz&)researchendlenstrip)htmlmstartdepthis /root/gov_crawler/crawl_zh.pyextract_contentr! s
;TBA
EEGE
E
A
c$i!m
!A#;& T!A#Y2S%SAaCyC
!AaC[H
$ QJE Q c$i!m
ac? "" c z t d D ]v } t j | dt }d|_ |j
dk7 s |j st j d Y|j }d}t j d| }|r |j d j }|s7t j d
| }|r |j d j }t | }|sFt j d |t j }|r |j d j }d}t j d | }|r|j d }| j d
r| nt | z }||||fc S | d d d fS # t" $ r%} |dk rt j d Y d } ~ d } ~ ww xY w)N )timeoutheaderszutf-8 r
z!(.*?)
r z(.*?)z1]*>(.*?)
\s*z(\d{4}-\d{2}-\d{2})http)rangerequestsgetHEADERSencoding status_codetexttimesleepr r groupr r! DOTALL
startswithBASE Exception)
urlattrr titler contentpub_datefull_urles
r process_detailrA # s Qx % $ S"g>A AJ}}#166
1
66D E >EA
((*II 5t< GGAJ , , .E &d+GII RTXZ\ZcZcd ggaj . . 0G H 0$7A 771: !nnV 4s$*H eWh7 7E% L
tT "" Qw
1
s AF %D F F:F55F:c t } t 5 }|j j dddg }|j }|j t dd t j d t d d
D ] }|j }t j d | }|D ] }| j t |z |j d }|s nJ|j! d
} | rt# | |k r n'|j% t j d |j' d d d t) | S # 1 sw Y t) | S xY w)NTz --no-sandboxz
--disable-gpu)headlessargscommiti` )
wait_untilr& r$ r d z+href="(/col/col1229821392/art/[^"]+\.html)"za.layui-laypage-nextz data-pageg ?)setr chromiumlaunchnew_pagegotoCOL_URLr2 r3 r+ r= r findalladdr7 query_selector
get_attributeintclickcloselist)
all_urlspbrowserpagepgr urlsunext_btndps
r collect_all_urlsr_ M s) uH a**##T 8Y#Z ! 'h >
1
3- B<<>D:: LdSD
' TAX&
'**+ABH'' 4BRB NN JJsO
%& >'& >s D EEc t j j dd } t dt d| dd | dk( rIt dd t }t d t
| d
d |st d d y t j t }|j }d }d }t t
5 }|D ci c] }|j t | | }}t | D ]
} | j! \ }}
} } |dz
}|dz d k( r t d| dt
| dd |
F| s|
} t d| d | s@t# j$ d| }
|
r&|
j' d j) d d dnd} t+ j, | d d }|t. k r |j1 dt |
|| | d d | d f |j2 d kD r|dz
}|dz d k( s|j7 d d d |j7 |j1 dt f |j9 \ }}}|j; t d d t d | d
d t d | d | d | d!d y t d"d y c c}w # Y xY w# t4 $ r}t d| d Y d }~d }~ww xY w# 1 sw Y xY w)#NMODEfullz=== z (mode=z) ===T)flushu Playwright收集URL...u
总URL: u 条u 无数据r ) max_workersr 2 u 详情 r z...u [WARN] 无正文: z /art/\d{4}/r) z-01-01z
2026-01-01
zINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, date_rank) VALUES (?, ?, ?, ?, ?, ?, ?)z
[DB_ERROR] zTSELECT COUNT(*), MIN(publish_date), MAX(publish_date) FROM gov_raw WHERE site_name=?u
=== 完成 ===u
新增: u
累计: u 条 (z ~ )u 日跑模式: 检查第1页) osenvironr- print SITE_NAMEr_ r sqlite3connectr
cursorr MAX_WORKERSsubmitrA r resultr r r4 splitr
fromisoformat CUTOFF_DATEexecuterowcountr8 rE fetchonerT )moder[ connrn total_newdoneexecutorr9 fut_mapfutr< r= r> dmdr@ cntmin_dmax_ds r mainr e s
::>>&& )D D 74&
.d; v~
&d3 !
3t9+S)6 -t , w'
K
8 "HLPQSx~s;S@QGQ#G,
"03
-UGX "9>IdV1SYKs;4H= #G 06dC >37BGI"((1+"3"3C"8";!KKM7
" "<
mpyo{| "OO -UE
.
9+S)6
3%uUG3ugQ7tD +48O R$ ! ;M!-T::;5 " "sg =K J B)K
!J+K.4J""K+KKJ K" K+K ;K KKK__main__)r, r r2 rh rl r r concurrent.futuresr r playwright.sync_apir r7 rM rt ri r- r
rk ro r. r! rA r_ r __name__ r" r r s & & & # ? /
>4B
**..$5
6 7
'#&'#T0;9~ zF r"