nzj d Z ddlZddlZddlZddlZddlZddlZddlZddl Z ddl
m Z ej j ej j e Zej j# ed ZdZdZdZdZ e j. Zd d
iZdddd Zdd Zd
Z d Z d Z ddZ d Z d Z!e"dk( r e! yy)u
crawl_debaoenv.py — 广东德宝环境技术研究有限公司 公示信息爬虫
============================================================
站点: https://www.debaoenv.com/Downs.aspx
内容: 环评/验收等公示信息，正文为PDF附件，不提取PDF文本，
只记录标题 + PDF链接（可点击）。
模式:
- 列表页: HTML分页 (Downs.aspx, Downs.aspx?page=N)
- 详情页: Down.aspx?id=N → iframe内嵌PDF链接
- 存储: 标题 + PDF URL内嵌到content/summary
用法:
python3 crawl_debaoenv.py # 增量爬 (第1页)
python3 crawl_debaoenv.py --full # 全量爬所有页
python3 crawl_debaoenv.py --sync # 仅同步到服务器
N)urljoinzdebaoenv_results.db 德宝环境zwww.debaoenv.comzroot@1.94.217.116/root/search.dbz
User-Agentz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36)newskiperrorsc t j j | t }t j j ||t
}|j
j dd S # t $ r Y y w xY w)N)headers)timeoutcontextzutf-8replace)r ) urllibrequestRequestHEADERSurlopenCTXreaddecode Exception)urlr reqresps #/root/gov_crawler/crawl_debaoenv.pyhttp_getr $ si nn$$S'$:~~%%c7C%Hyy{!!')!<< s A+A. . A:9A:c t j t } | j d | j d | j | j y )NPRAGMA journal_mode=WALu? CREATE TABLE IF NOT EXISTS crawl_results (
id INTEGER PRIMARY KEY AUTOINCREMENT,
site_name TEXT DEFAULT '德宝环境',
title TEXT,
url TEXT UNIQUE,
content TEXT,
publish_date TEXT,
summary TEXT,
crawled_at TEXT DEFAULT (datetime('now','localtime'))
))sqlite3connectDB_PATHexecutecommitclose)conns r init_dbr% - sB ??7 #DLL*+LL
KKMJJL c d t j t } |j d| ||||f |j |j
dkD rt dxx dz
cc< nt dxx dz
cc< |j y # t dxx dz
cc< Y &xY w# |j w xY w)NzcINSERT OR IGNORE INTO crawl_results (title, url, content, publish_date, summary) VALUES (?,?,?,?,?)r r r r )r r r r! r"
total_changesstatsr# )titler contentdatesummaryr$ s r store_recordr/ > s ??7 #D
! C$ 0
! %LA L &MQ M
h1
s AB BB B/c t d t j t } | j d j }| j
|s t d yt j d }|j d d}d}|D ]J }|\ }}}} }
|j d |||xs d
dd | xs d
|
xs d
dd |d
f |j dkD r|d
z
}L |dkD r3|j |j d|f |j |j d|f j d } |j
t d| dt | d| d y# t $ r} t d| Y d} ~ d} ~ ww xY w)u@ 将本地数据直接写入 search.db（服务器本地模式）u
📤 同步到 search.db...zPSELECT title, url, content, publish_date, summary FROM crawl_results ORDER BY idu 本地没有数据Nr r r r zwINSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, summary, site_name, tags) VALUES (?,?,?,?,?,?,?) i i, r( z Error: zINSERT INTO gov_search(rowid,title,site_name,summary) SELECT r.id,r.title,r.site_name,r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z OK /u 条同步到 search.db (DB共 条)) printr r r r! fetchallr# r) r r" fetchonelen)
r$ rowsdst site_name new_countrr+ r r, pub_dater. etotals
r sync_to_serverr@ R s
)* ??7 #D <<j k t t vDJJL
$%
//+
,CKK)* II
#12.sGXw # KK) gmWf 5x~2-R#& 27
1$Q
# 1}
Q L
KK89,
HHP
ST
VEIIK E)Ac$i[(FugT
RS# # IaS/ " " #s ?E&& F/FFc 0 | dk( r
dt dn dt d| }t | }|st d| d g S g }t j d| D ] }|j d j
}|j d j
xs |j d j
}dt | }|j ||d
t j d | }t | D ] \ } }
| t | k r|| nd |
d
< |S )u1 获取列表页，返回 [{'title','url','date'}]r( https://z /Downs.aspxz/Downs.aspx?page=u ⚠️ 列表页u 加载失败zD]*title="([^"]*)"[^>]*>([^<]+) )r r+ u" 发布日期：(\d{4}-\d{2}-\d{2})r1 r- ) DOMAINr r4 refinditergroupstripappendfindall enumerater7 ) pager htmlitemsmhrefr+ full_urldatesiitems r fetch_list_pagerV s ,0AIHVHK
(XfXM^_c^d;eC C=D
"4& 56 E
[[O 8 wwqz !
"8aggaj&6&6&8 fXdV,
X678
JJuQxrV : Lr& c t | }|syd}t j d| }|r |j d j }d}t j d| }|r2|j d j }t dt | }d}t j d| }|r|j d }|r|j d d nd} |r
d
| d | d }
nd
| d}
|r| d| n|} |xs d|
||| dS )u8 抓取详情页，提取标题+PDF URL（不下载PDF）Nr1 z(.*?)r( z ]*src="([^"]+\.pdf)"rB u( 发布日期[：:]\s*(\d{4}-\d{2}-\d{2})r2 z 📄 下载PDF附件: z
zz
z
PDF: )r+ r, pdf_urlr- r. )r rF searchrH rI r rE split)
detail_urlrN r+ tmrY pmpdf_relr- dmpdf_namer, r. s r fetch_detailrb s$ J D
E *D 1B !!# G 4d ;B ((1+##%HVH-w7 D > EB xx{ *1w}}S!"%bH )QRZQ[[cdwd#,3wwi(G " r& c < dd l } | j }|j ddd |j dt dd |j d
dd |j d t dd
|j }t |j r t y |j rdn |j }t j }d}t d|dz D ] }t | }|st d| d nt d| dt | d |D ] }|j r||j k\ r at! j" t$ } | j' d|d f j) }
| j+ |
rt, dxx dz
cc< t d|d d d ddd t/ |d } | t d t, d xx dz
cc< t1 | d xs |d |d | d | j3 d! xs |j3 d!d | d" t, d# dkD rK|dz
}| j3 d$ r$| j3 d$d j5 d% d& nd'} t d(| n t d) t j6 d* t j |z
}
t d+d, t d-t, d# d.t, d d/t, d t d0|
d1d2 t, d# dkD r t y y )3Nr z--full
store_trueu 全量)actionhelpz--pagesr( u 页数)typedefaultrf z--syncu 仅同步z--testu
测试N条i' u ⏹ 第u 页无数据，结束u
📃 第u 页 (r3 z'SELECT 1 FROM crawl_results WHERE url=?r r u 📄 r+ # z...r1 T)endflushu ✗ 详情页失败r r, r- r. r rY r2 rX u 无PDFu ✅ u ⏭️ 重复g333333?
z2==================================================u 🏁 完成! 新增:u 跳过:u 错误:u ⏱️ z.0fs) argparseArgumentParser add_argumentint
parse_argsr% syncr@ fullpagestimerangerV r4 r7 testr r r r! r6 r# r* rb r/ getr[ sleep)rn parserargs max_pages
start_time total_newrM rO rU r$ existsdetailra elapseds r mainr s
$
$
&F
HE
QXF
KH
sALI D I yy tzzIJIaQ ' , % GD6!67 8
4&c%j\67 $ DyyY$)) 3??7+D\\9DK>hj
JJLf
"
GDM#2 ./s34 H!$u+.F~,-h1$ w04=U y!
6":dhhvr&:y!
U|a Q GMzzR[G\6::i4::3?CbjhZ()&' JJsOI$ , \ iikJ&G Bvh- u huV}oXeT\oM^
_` GGC=
"# U|a r& __main__) )r( )#__doc__osrF sysrv r
subprocessurllib.requestr ssl urllib.parser pathdirnameabspath__file__BASE_DIRjoinr SITE_NAMErE
SERVER_SSHSERVER_SEARCH_DB_create_unverified_contextr r r* r r% r/ r@ rV rb r __name__ r& r r s " C B B B
77??277??84 5
'',,x!6
7
$ $c$$&o 1*"(.Tb2(VGT zF r&