fj d Z ddlZddlZddlZddlZddlZddlZddlZddl m Z m
Z
ej j d ej j d ej j dej j! ej j# e ddlmZ dZdZd Zd
Zd Zd Zd
ZdedeedddZ d Z d Z d Z d dZ e!dk( r$dejD v Z# e e# Z$ e%de$ d yy) u[ crawl_funing_js.py - 阜宁县人民政府-通知公告 (Hanweb JPage + 创宇云防御WAF) N)datetime timedeltaT)line_buffering)push_to_searchdbu" 阜宁县人民政府-通知公告zhttps://www.funing.gov.cn66172421748 z /module/web/jpage/dataproxy.jsp1/u 阜宁县人民政府0)appidwebidpathcolumnidunitidwebnamepermissiontypec t j } | j j ddd | j t
dz d | S # t $ r Y | S w xY w)u 创建带 WAF cookie 的 sessionz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36zzh-CN,zh;q=0.9)z
User-AgentzAccept-Languager timeout)requestsSessionheadersupdategetDOMAIN Exception)ss $/root/gov_crawler/crawl_funing_js.py get_sessionr" " sf AIIs+
fslB'
H
H
s A A A c i t dt | i}t t z dz t j
j
| z } | j |d }d|_ |j dk7 r t d| d|j g S t j d |j t j }g }t! }|D ] } t j" d | }
t j" d
| } t j" d| } |
sH| sK|
j% d }
|
|vsa|j' |
|j) |
| j% d j+ | r| j% d ndf t d| dt- | d |S # t $ r }t d| d | g cY d
}~S d
}~ww xY w)uA 获取第N页列表数据，返回 [(href, title, pub_date), ...]page?r r utf-8 z List page z: HTTP z: Error Nz&zhref="([^"]+)"ztitle="([^"]*)"z\[(\d{4}-\d{2}-\d{2})\] z Page z: z
new items)DATAPROXY_PARAMSstrr DATAPROXY_BASEurllibparse urlencoder encoding status_codeprintr refindalltextDOTALLsetsearchgroupaddappendstriplen)sessionr$ paramsurlre cdata_blocksrecords
seen_hrefsblockhref_mtitle_mdate_mhs r!
fetch_listrK 1 s
4
4&#d)
4F
>
!C
'&,,*@*@*H
HC KKRK (
==C Lgamm_= >I ::1166299L GJ ,e4)) .65u= g QA
"q!MM!$**,'-FLLO2
GD6CL>
45 N1
TF(1#./ s AF G)G;GGc
l |j d r|nt |z } | j |d }d|_ |j }d}t j d | }|r7|j d
j }t | d
kD r|d n|d }d} t j d
| }|r!|j d
j dd } t j d|t j }
|
s)t j d|t j } | r| }
|
s|| dfS |
j r|
j d
n|
j } t j dd| } t j dd| } t j dd| } t j dd| } t j dd| } t j dd| } g }t! }
dD ] }t j" d|d|d|d | t j D ]~ }|j }t j d d| j% dd }|r(||
vsB|
j' | |j) | ed |v sd |v sn|j) | |sAt j d d| }t j d!d"| j% }|| |fS d"j+ | }t j d#d$ | }|| |fS # t $ r}t d| Y d}~yd}~ww xY w)%u; 获取文章详情，返回 (title, pub_date, content_html)httpr r r& z Detail error: N)r) r) r) r) z([^<]+)r( r u. 发布日期[：:]\s*(\d{4}[-/]\d{2}[-/]\d{2})r -uC (.*?)zid="zoom"[\s\S]*?z z ]*>[\s\S]*?z ]*>[\s\S]*?z ]*>z
z(?:begin|end)-->)ptableaimgz(?:]*/>|]*>.*?)<[^>]+>( zz fetch_detail.. s VQWWQZ@ )
startswithr r r0 r r2 r5 r3 r8 r9 splitr= replacer6 lastindexsubr7 finditerr< r: r; join)r> r@ full_urlrA rB htmltitlerZ partspub_date content_mzoom_mraw seen_keystag_nametag_htmlkeyr5 body_htmls r! fetch_detailrq V s nnV,s&3,H KK"K -
66D
E
+T2A
" Z!^b q H
CTJA771:%%c3/ N biiI 5tRYYG I h"" ) 3 3)//!
9JC
&&12s
;C
&&2B
all_recordsseen max_pagesr$ itemshrefrf rh rd irec detail_title detail_datert s r! runr s mGK 5D eIaQ ' 7D) %* !D%4 #'??6#:4
"" "# (!* $
3
[!1 2
34 K( 3-9'3u:-N* k7#3s7|G %8Z J $C N VVJG r s a 6 6 6 6 (
d +
d + 277??277??8#<= > ( 0 $
3
&
"JN&b* Z z
SXX
%C
#
C F3%x
! r\