&jb d Z ddlZddlZddlZddlmZ dZdZedz Zdedz d d
Z d Z
eZ d Z d
Z
d Zd Zedk( r e yy)u
南昌经济技术开发区 - 生态环境
自定义CMS, WAF屏蔽分页, 仅爬第1页(20条)
https://ncjk.nc.gov.cn/ncjkq/sthj01/bmxxgk_list.shtml
N)urljoinz/root/search.dbzhttps://ncjk.nc.gov.cnz /ncjkq/sthj01/bmxxgk_list.shtmlzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36/zzh-CN,zh;q=0.9)z
User-AgentRefererzAccept-Languageu 南昌经开区生态环境c L | j t t dd }d|_ |j }t j d|t
j }g }|D ]G \ }}}t t | }|j |j ||j d I |S )u1 获取列表页（仅第1页，WAF屏蔽分页） Fheaderstimeoutverifyutf-8zd]*title="([^"]*)"[^>]*>.*?\s*]*class="time"[^>]*>(\d{4}-\d{2}-\d{2}))titleurldate) getLIST_URLHEADERSencodingtextrefindallDOTALLr appendstrip) sessionrhtmlitemsresultshrefr
date_strfull_urls /root/gov_crawler/crawl_ncjk.pyget_listr# s Hgr% HAAJ 66D JJo bii
E G!& eX8T*[[] NN$
N c n | j |t dd }d|_ |j }t
j d|t j }|r |j d j nd}|r>t
j d
d| }t
j d d| }|j }t
j d | }d}|r7d
}|D ]+ \ } }
t || } |d| d|
j dz
}- |dz
}|r||z
}|ddS # t $ r}dt | dcY d}~S d}~ww xY w)u 获取详情页内容r Fr r )contenterrorNzN]*class="article-content[^"]*"[^>]*>.*?(.*?) z\s+style="[^"]*"z\s+class="[^"]*"zF]*href="([^"]*\.(?:pdf|doc|docx|xls|xlsx|zip)[^"]*)"[^>]*>([^<]+)u> )r r r r Exceptionstrr searchr groupr subr r ) r r r r e content_mr' attsatt_htmlatt_hrefatt_nameatt_urls r"
get_detailr6 - sI 0 KKWbK G
vv
Y biiI -6iooa &&(2G&&,b':&&,b':--/ ::Q D H S"& _ Hh c8,G -y0B8>>CSBTT]^ ^H _ M!8 ,,? 0 A//0s ,D D4
D/)D4/D4c | j }d}|D ] }|d }|d }|d }|d }|s|j d|f |j r>t j dd| d d } t j dd| j }
|j d
|||| t ||d f |d z
} | j |S )
Nr r
r r' r )SELECT id FROM gov_raw WHERE page_url = ?z<[^>]+>r& zINSERT OR IGNORE INTO gov_raw (title, page_url, content, summary, site_name, publish_date, source_url, category) VALUES (?, ?, ?, ?, ?, ?, ?, ?)u 环评公示r) )cursorexecutefetchoner r. r SITE_NAMEcommit) connr cursaveditemr
r r' pub_datesummary
content_plains r"
save_to_dbrF U s
++-C
E W
5ky/< ?#H <<> &&R1$37z2w7==?
_
C'9h^ T
+, KKM Lr$ c V dd l } | j t }t j }t | }t
dt | |j }|D ]E }|j d|d f |j s*t
d |j y |D ]E }t ||d }|j d rt
d|d d d d
|d >|d |d < G t || }|j t
d t | d
| y )Nr z List items: r8 r z-Existing record found, skipping (incremental)r( z ERROR r
r z: r' zDone. Crawled: z, New: )sqlite3connectDB_PATHrequestsSessionr# printlenr: r; r< closer6 r rF )rH r? r r r@ rB detaildb_saveds r" mainrR r s! ??7 #D G W E LU
%& ++-C ?$u+ P <<> A B JJL ,GT%[1 ::g HT']3B / 06'?2CD E +Y , $&HJJL OCJr[ sy
# 7 7 D#~'
* *%-P:;: zF r$