ztjg d Z ddlZddlZddlZddlZddlZddlmZmZ ej j dej j ej j e ddl
mZ ej ed z
j! d ZdZd d
iZd Zd Zd
ZddZedk( rU ej2 Zej7 de d ej; Z ee j> Z e!de d yy)uR crawl_wanning_xxgk.py - 万宁市-政府信息公开(生态环境0400) (TRS静态) N)datetime timedelta)push_to_searchdbiG )daysz%Y-%m-%du 万宁市-生态环境z
User-Agentz6Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0z/https://wanning.hainan.gov.cn/wanning/xxgk/0400c | dk( r
t dz }nt d| dz d} t j |dt d }d |_ t j d
|j t
j }g }|D ] \ }}}t j d d | j }t j d
d | }t | dk sd|vrV|j d r|} nF|j d rd|t d d z } n#|j d rd|z } n t dz |z } |j | ||d |S # g cY S xY w)Nr /z/index_ z.html Ftimeoutheadersverifyutf-8z6]*>(.*?).*?(\d{4}-\d{2}-\d{2})<[^>]+> \s+ t202httpz../../z&https://wanning.hainan.gov.cn/wanning/z https://wanning.hainan.gov.cn)urlpub_datetitle)BASErequestsgetHEADERSencodingrefindalltextDOTALLsubstriplen
startswithappend)
page_numr ritemsresultshref
title_htmlr r full_urls
'/root/gov_crawler/crawl_wanning_xxgk.pyfetch_urls_from_pager1 sE 1}Sjghl^51 LLgwu M
JJA
E G&+ P"j(z2z288:vr5) u:>V4 / ??6 "H
__X
&?$s8}~BVVH
__S
!6=HczD(HxXNO P N/ s $E Ec & t d D ]( } t j | dt d }d|_ n yd}t j d|j t
j }|rat j d d|j d
j }t j d d| j }|rd |vr|}|st j d
|j t
j }|rit j d d|j d
j }t j d| }t |t j }t j dd| j }d}t j d|j t
j }|r |j d
j }|sPt j d|j t
j }|r |j d
j }|sPt j d|j t
j }|r |j d
j }||fS # d }Y xY w)N r
Fr
r )r r r z]*>(.*?)r r z^begin-->|end-->$404z([^<]*)z--)keyr z3]*>(.*?)
z>]*>(.*?)
z3class="xxgk-syxl-m21023-xz2 clear"[^>]*>(.*?))ranger r r r r searchr" r# r$ groupr% splitmaxr& )r _attemptr* r mcandpartscontents r0 fetch_detailr@ / s !H S'75QA AJ y E
(!&&"))EHHUE*E3'--/E FF62u % + + -EG
H!&&RTR[R[\A''!*""$ IIWYZY_Y_acajaj k ggaj&&(G IILaffVXV_V_ ` ggaj&&(G '>? As $J Jr c F g }t }t | D ] }t | }|s nt d|dz dt | d |D ]{ }|d r
|d t
k r|d |v r |j
|d t |d \ }}|s|d }|j sZ|j ||d |d t |dd } t d
t | d |D cg c] }|d j s| } }| r t | d t | S c c}w )
Nz Page r z: z itemsr r r r )r r r site_namer? summaryz Total: r? wanning_xxgk) setr6 r1 printr& CUTOFFaddr@ r% r( SITE_NAMEr )
max_pagesrecordsseenpgr+ itr r? r* valids
r0 runrP V s: G 5DI $R(
1vRE
|623 B*~"Z.6"9%yD HHRY )"U) 4NE77 ==? NN r%ybn &7r
(
Ic'l^6
*+ 811Y<#5#5#7Q 8E 8 / u:
9s &D ?D __main__z--pages)typedefault)rJ zDone: z records)r )"__doc__sysosr r argparser r pathinsertdirnameabspath__file__ crawler_libr nowstrftimerG rI r r r1 r@ rP __name__ArgumentParserap add_argumentint
parse_argsargspagescntrF r0 rk s X & & & ( 277??277??8#<= > (
(,,.9%0
0 : :: F $ Q
R8 B%N: z "BOOICO3
==?D
#C F3%x
! rj