uj d Z ddlZddlZddlZddlZddlZddlmZ ddlmZ dZ dZ
e
dz Z de
d z d
Z d Z
d Zd
ZddddddddZddZd Zd Zd Zedk( r e yy) uu
蓬莱区政府 - 建设项目环评 爬虫
JPAAS CMS, API分页
https://www.penglai.gov.cn/col/col30455/index.html
N)datetime)quotez/root/search.dbzhttps://www.penglai.gov.cnz7/api-gateway/jpaas-publish-server/front/page/build/unitzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z/col/col30455/index.html)z
User-AgentRefereru 蓬莱区建设项目环评z2https://www.penglai.gov.cn/col/col30455/index.html bulidstatic152awU93spIZ5G7CYnk12KGtcolumnu 当前栏目列表null30455) parseTypewebIdtplSetIdpageTypetagIdeditTypepageIdc t j }t j ||dd |d< | j t
|t dd }d|_ |j }|j di j d d
}t j d |t j }g }|D ] } t j d | }
t j d
| } t j d| } |
sH| sK|
j d }
|
j d s t |
z }
|j | j d j! |
| r | j d j! nd
d |S )u 获取列表页数据)pageNopageSizeF) ensure_ascii paramJson )paramsheaderstimeoutverifyutf-8datahtml z(.*?)zhref="([^"]*)"ztitle="([^"]*)"z([^<]*) http)titleurldate) BASE_PARAMScopyjsondumpsgetLIST_URLHEADERSencodingrefindallDOTALLsearchgroup
startswithBASE_URLappendstrip)sessionpage_no page_sizer rr r itemsresultsitematitle_mdate_mhrefs "/root/gov_crawler/crawl_penglai.py
get_list_pagerD $ s@
F **Y%O^cdF; HVWbQV WAAJ 668D 88FB # #FB /D JJ($ :EG II' .)) .52D9 771:D??6* $ NN q)//139 Q--/r
N c P | j |t dd }d|_ |j }t
j d|t j }|r |j d j nd}t
j d
|t j }|r |j d j nd}|r>t
j d d| }t
j d d| }|j }||dd
S # t $ r}dt | dcY d}~S d}~ww xY w)u 获取详情页内容r F)r r r r r! )contenterrorNz#(.*?)
r" zEz\s+style="[^"]*"z\s+class="[^"]*")r$ rG rH ) r+ r- r. text Exceptionstrr/ r2 r1 r3 r7 sub) r8 r% r; r er@ r$ content_mrG s rC
get_detailrO ? s 0 KKWbK G
vv
ii>biiPG(/GMM! " " $RE P biiI -6iooa &&(2G &&,b':&&,b':--/w <<) 0 A//0s ,D D%
D D% D%c | j }d}|D ] }|j dd }|j dd }|j dd }|j dd }t j j d } |sr|j d|f |j
r|rt j d d| d
d nd}
|j d ||||
t ||d
f t j d d| j } |j d|| ||t f |dz
} | j |S )u 批量入库r r$ r! r% rG r& z%Y-%m-%d %H:%M:%S)SELECT id FROM gov_raw WHERE page_url = ?z<[^>]+>N zINSERT OR IGNORE INTO gov_raw (title, page_url, content, summary, site_name, publish_date, source_url, category) VALUES (?, ?, ?, ?, ?, ?, ?, ?)u 环评公示zpINSERT OR IGNORE INTO gov_search_v3 (title, content, source_url, publish_date, site_name) VALUES (?, ?, ?, ?, ?)r" ) cursorr+ r nowstrftimeexecutefetchoner/ rL SITE_NAMEr7 commit) connr< cursavedr> r$ r% rG pub_daterT summary
content_plains rC
save_to_dbr` \ s8
++-C
E "%hhub!((9b)88FB'lln%%&9: ?#H <<> detaildb_savedr[ s rC mainrv s- CHH$I iI ??7 #D GKMKG
Y
wis+, gw/ 'y
; <
s5zl&)*
D e5Fzz'" &/!2341 ::i(1 $Y /DO QJE
de$Sdtxx ?RT$STx
q
*WIQ78 ++-C
G$u+ X<<> KMJJLOK= }KXeWfgh
1 U Y
X JJL OK= }K
WX+ %Ts G2
%G2
__main__) )__doc__ri r) r/ rd osr urllib.parser rh r5 r, r- rX SITE_URLrf r' rD rO r` rv __name__ rE rC r s
' O O D44
* ?
'
! 6=:#L:Yz zF rE