(jl d Z ddlZddlZddlZddlZddlZddlZddlmZ dZ e dz Z
dZ dZ dZ
d d
iZd Zd d d
d d dd ddddddddgdddgdddddZd Zd Zd Zd Zd Zed k( r e yy)!uO Crawler for 滕州市-枣庄市生态环境局滕州分局-通知公告
http://www.tengzhou.gov.cn/zwgk/xxgkml/szbm/zzssthjjtzfj/
TRS CMS /irs/front/search 接口 (filters 组配分类+部门)
列表: POST /irs/front/search 71条/4页 (20条每页)
详情: curl 直连, 正文 div.zwnr, 标题 , 日期 成文时间
N)urljoinzhttp://www.tengzhou.gov.cnz/irs/front/searchz/root/search.dbu( 滕州市生态环境分局-通知公告zcrawl_tengzhou_sthjj.pyz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36z
2023-08-18 18d72ce8cb52849f_2024329561714 通知公告1)aggrName
aggrValuesfilterIdf_2024329501386u$ 枣庄市生态环境局滕州分局ALLtime all) appendixType
beginDateTimecodecodes
configCode
dataTypeId endDateTimefilters granularityisSearchForcedhistorySearchWordsorderBypageNopageSizesearchByc 2 t dt d| d y )N[z] T)flush)print SITE_NAME)msgs )/root/gov_crawler/crawl_tengzhou_sthjj.pylogr) s Ai[3%
- c t t }| |d< t j t |t
d }|j dk7 rt d|j y |j S )Nr )jsonheaderstimeout u 搜索接口 HTTP ) dict SEARCH_BODYrequestspost
SEARCH_URLHEADERS status_coder) r- )pagebodyrs r( fetch_searchr; # sW DDN
jtWbIA}} 01 668Or* c g }| j di j di }|j dg D ] }|j di }|j dd }|j d xs |j dd }|j d xs |j d d }|r|d
d nd}|s}|s|j ||j |d |S )
u( 从搜索响应提取 (url, title, date)datamiddle
listAndBoxurlr title_no_tagtitleztable-2r N
)r@ rB date)getappendstrip) r= itemsr> entrydr@ rB trD s r(
parse_listrL - s E
XXfb
!
%
%h
3FL"- M IIfb !eeE2 n%;w);
EE) 1fb 1q"v 5 LLu{{}dK LM Lr* c t j | t d }d|_ |j }d
d
d
d
g d d }t j d
|t j }|rT|j d j j dd
j dd
j }|r||d< t j d|t j }|s%t j d|t j }|rA|j d }t t j d| |d< t j dd
|t j }t j dd
|t j }|j |d< t j" d| D ] }|j d |j d j c} t% fd d D s&t j d | t j& snt) t* | }
|d j- |
d t j d!| }|st j d"| }|rN|j d d#t/ |j d d$d#t/ |j d% d$|d&< t j d'| }|r"|j d j |d(< |S # t
$ r9}t
d| j d d dt | dd Y d}~yd}~ww xY w))u- 详情页: 标题/正文/日期/文号/附件r, )r. r/ zutf-8u 详情请求失败 /z: N2 r F)rB contentrD doc_no attachments has_tablez<title>(.*?)r u 滕州市政府信息公开--u _滕州市政府信息公开rB z&]*class="zwnr"[^>]*>(.*?)z+]*class="view[^"]*"[^>]*>(.*?)z]*>.*?)flagsz]*>.*?rQ z+]*href="([^"]+)"[^>]*>([^<]{2,80}) c 3 & K | ] }|v
y w)N ).0ktxts r( z fetch_detail..Y s D18Ds )u 附件u 下载u 文件z \.(docx?|xlsx?|pdf|zip|rar)$rS )namer@ uG 成文时间[：:]\s*<[^>]*>\s*([\d]{4})年([\d]{1,2})月([\d]{1,2})日u( ([\d]{4})年([\d]{1,2})月([\d]{1,2})日-02d rD u* 文\s*号[：:]\s*<[^>]*>\s*([^<\s]{3,40})rR )r3 rE r6 encodingtext Exceptionr) splitstrresearchDOTALLgrouprG replaceboolsubfinditeranyIr BASErF int) r@ r: htmleresultmrK inneramhreffullr[ s @r( fetch_detailrz < s LLgr :
vv b"SUdi
jF
)4;A
GGAJ & &'F K S STrtv w } } F7O
;T299MA IIDdBII V
"299Y#> ?{3RbiiP12uBIIN !KKMy++ LeT IB RXXa[%6%6%8ID#D%CDD Rqswy{y}y}H~ tT*}%,,c$-GH I \^bcA IIA4 H GGAJSTv
?FA 771:++-x MI !#))C."4!5RAs } EFs /L M /L==Mc f d} dt j v r]t j j d }|dz t t j k r t t j |dz } d}d}d}t j
}t
j t d }|j }t d| dz D ]* }t | } | st d| d |dz
}&| j di j d i }
|
j d
d } |
j d d } t d| d | d
| d t | }
|
st d| d n|
D ] }|j d|d f |j! r|dz
}0t# |d }|r
|d r|d sC|r|d rd|d d|d d|d< n)t d|d j% d d |dz
}|d xs |d xs d}|j dt& t( |d |d |||d d d j+ d d j+ d d d d!d|d |d" d t- j. |d# d$% d&|d' rdndt0 f |j2 dkD r|dz
}n|dz
}|j5 |d(z dk( sQ|dkD sXt d)| d*| d+| d,t j
|z
d-d. - |j7 t d/| d*| d+| d0t j
|z
d-d1 y )2Nr z--pagesr r, )r/ u 页u
接口失败r= pagertotal pageCountrN u : 共u 条u 无条目，停止z'SELECT id FROM gov_raw WHERE page_url=?r@ rB rQ z z
u 详情失败: rO rD z
1900-01-01aF
INSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, date_rank, summary, status,
category, visits, content, tags, industry, attachments, group_name, has_table, script_name)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
r0 publishedr rR rS F) ensure_asciiu 山东rT rC u 进度: 新增u 跳过u 失败z (z.0fzs)u 完成: 新增u 用时s) sysargvindexlenrq r sqlite3connectDBcursorranger; r) rE rL executefetchonerz rd r& r5 rj r- dumps SCRIPT_NAMErowcountcommitclose)pagesi total_new
total_skip
total_failt0conncurpgr= r| r}
page_countrH itemdetrD s r( mainr i sW
ECHH HHNN9 % q53sxx= Q(EIJJ
B ??2r *D
++-CAuqy! .rB #bT ' ( !OJ $(("5 '1%YY{A.
c"Qzl%wc 234 #bT,- . rD KKADK> R||~a
tE{+Cc'l#i.3w<'3DK=3w<.PX%YC N*4;+<+$v,>,D KK :tE{CL$c)nTc*223;CCCL^QIH
r4::c-.@u#Us; /!Q
||a Q a
KKM2~"y1}&yk GJr s . - - #
'
'
6 ' K L ]RF2&nRUV&6\jmn AnEU . *Z?hD zF r*