uj$1 d Z ddlZddlZddlZddlZddlZddlZddlZ ddl
Z
ddl Z e j d ddl
mZ ddlZddlmZ ej& j( j+ e dZdZdZd Zd
Zd Zed z Z d
Z dZ ddddZ dZ e!ejD D ]k \ Z#Z$e$jK d r e&e$jO dd d Z 4e$dk( s:e#dz e)ejD k sS e&ejD e#dz Z m e dk\ re ndZ*d Z+d Z,d Z-d Z.d Z/d Z0d Z1d Z2e3d k( r e2 yy# e$ r dZY w xY w# e($ r Y w xY w# e($ r Y w xY w) u
平顶山市生态环境局 - 环评公示 (sthjj.pds.gov.cn/channels/11330.html)
巨匠云 CMS (user.pds.gov.cn API)
列表: POST https://user.pds.gov.cn/api/Outer/T?parms=opt/dynamic/
body JSON: {"publishmentsystemid":10370,"channelid":11330,"contentid":0,"ajaxdivid":"ajaxElement_1_1","template":"<从页面提取的固定加密串>","pageNum":N}
template 从列表页 HTML 正则提取 (template" : "...."), 每页10条
响应HTML: 标题日期
详情: /contents/11330/{id}.html, meta ArticleTitle + div.content 正文 (div.article 内)
注意: SSL需verify=False; Content-Type text/plain
Nignore)
BeautifulSoup)InsecureRequestWarningzhttps://sthjj.pds.gov.cnz6https://user.pds.gov.cn/api/Outer/T?parms=opt/dynamic/u( 平顶山市生态环境局-环评公示u 河南zcrawl_pds_hjgs.pyz/mnt/data/search.dbz/channels/11330.htmliB, i( zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)
User-AgentAcceptzAccept-Language z--pages==z--pagesc v | syt j | } | j dd j dd j dd j dd } | j dd j dd } t j d d| } t j d
d| } | j S ) N u u u u u u ^[\s·\u00b7 ]+\s+)html_libunescapereplaceresubstrip)ts #/root/gov_crawler/crawl_pds_hjgs.py clean_titler C s ! A (B ''"5==hKSST\^`aA &# &&x5A
#R+A
vsA A 779 c | j t t dd }d|_ t j
d|j }|s t d |d S )z2Extract fixed template string from list page HTML. Fheaderstimeoutverifyutf-8z0template["\']?\s*[:=]\s*["\']([^"\']{100,})["\']z template not found in list pager )get LIST_PAGEHEADERSencodingr findalltext RuntimeError)sessionrtplss r get_templater* N sL Iw5 IAAJ
::I166 RD <== 7Nr c t j t t dd||dd }| j t
|j
d t d dd d
d }|j S ) Nr ajaxElement_1_1)publishmentsystemid channelid contentid ajaxdividtemplatepageNumF) ensure_asciir r ztext/plain;charset=UTF-8)r z Content-Typer )datar r r ) jsondumps
PUB_SYS_ID
CHANNEL_IDpostAPI_URLencoder" r% )r' r1 pagebodyr( s r
fetch_listr> X ss :: ) &
D
[[
!!,/ 6
A
66Mr c h g }t j d| t j D ] }|j d }t |j d }|j d j }|sPt j j t | }|j |||f |S )NzHhref="(/contents/\d+/\d+\.html)"[^>]*>([^<]+)\s*([^<]+)r ) r finditerSgroupr r urllibparseurljoinBASE_URLappend) html_textitemsmhreftitledateabs_urls r
parse_listrQ n s E
[[S244
- wwqzAGGAJ'wwqz ! ,,&&x6
gud+,
- Lr c d}d}d}t rt | d }|j ddt j dt j i }|r |j d rt
|d }|j ddt j dt j i }|rM|j d r<|d j }t j d | } | r| j d
}|j d t j d t j
}
|
rN|
j d t j dt j
} | r t | }n t |
}|sB|j d t j dt j
} | r t | }|sAt j d| t j } | rt
| j d }|sIt j d| t j t j z } | r| j d
}|s)t j d | } | r| j d
}|||fS )Nr html.parsermetaname ArticleTitle)attrscontentPubDatez(\d{4})-(\d{2})-(\d{2})r divarticle)class_z5]*class="[^"]*content[^"]*"[^>]*>(.*?)\s*)
r findr compileIr r r matchrD strsearchrC )
rJ page_urlrN rO content_htmlsoupmtmdddmartccrL s
r parse_detailrl ~ s E
DL Y
6
YYvfbjj "$$.O%PY
Q
"&&# 9
.E
YYvfbjjRTT.J%KY
L
"&	
##%A4a8Bxx{iibjjRTT&BiC %
:rtt(DEB "2w "3x 5J)EFB "2w IINPY[][_[_ `
+E IIUW`bdbfbfikimimbm n 771:L II0) < 771:D $ $$r c
| sddg fS t j dd| t j } t j dd| t j } t j dd| t j t j z } t j dd| t j t j z } t rQ t | d }|j d D ]% }d
D ] }|j j |d ' t | } | }t j d d|t j }g i fd }t j d
||t j t j z }g fd}t j d||t j t j z }t j d| t j rdnd}t j dd|t j }t j dd|t j }t j dd|t j }g } |j d D ] }
|
j } | st j dd| } t j | } | j dd j dd } t j d d| } | s~j! D ] \ }
}| j |
| } t# D ] \ }}| j d | d | } | j% | j dj' | }j! D ] \ }
}|j |
| } t# D ] \ }}|j d | d | } t j d d| }t j d d| }t j | }t j d!d| }t j d"d| }t j d#d| }t) }g }|j d D ]X }t j d$dt j dd| }|s2||v r7|j+ | |j% | Z dj' | j }||fS # t $ r Y w xY w)%Nr r z
)flagsz ]*>z]*>.*?z]*>.*?rS T) styleclasslangdiralignvalignwidthheightborder cellpadding cellspacingz;]*src="[^"]*fileTypeImages/icon_[a-z]+\.gif"[^>]*/?>c | j d }| j d }t j dd| }t j | j }t j dd| j }|s5t j j |j d d xs d}t j j | }j ||f d t d
}d | d | d
|< |S )Nr r@ <[^>]+>r u ^附件[:：]\s*?r u 附件__LINK____z z)rD r r r r r ospathbasenamesplitrE rF rG rI len) rL rM innertxtrP key attachments link_protectrc s r
_link_replz html_to_text.._link_repl s wwqz
ffZU+ $**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*\*+2. 'y0B3%t L S
r z&]*href="([^"]+)"[^>]*>(.*?)c h j | j d dt dz
dS )Nr __TBL__r r~ )rI rD r )rL
table_protects r _tbl_replz html_to_text.._tbl_repl s3 QWWQZ(]+A-.b11r z]*>.*?z ]*>r zz
z
z(]*>z
r{ r r
z z [\s\u200b\u200c\u200d\ufeff]+r r~ z
__LINK__\d+__z __TBL__\d+__z[ \t]+\nz\n{3,}z [ \t]{2,}r )r r rC r_ r find_allrW popra Exceptionrb r r r r r rK enumeraterI joinsetadd)rX rc _souptagattrcontent2r r has_tablepartsblockbplainkvitbloutseenfinalpr r r r s ` @@@r html_to_textr sC 1byff]BrttAIvvgz8244@HvvlD("$$?HvvA2xWYW[W[\H E ' KKM z2q)!!%(
fc*228SA7UC &&( DAq !QA
. 0FAs GA3b/3/A 0
QWWY ++e
C ""$ 1kk!Q M* 03kkGA3b/3/0
&&!2s
+C
&& "c
*C
C
C
&&dC
(C
&&FC
(C
&& sC
(C 5D E
YYv
ffVR
B!:; $;
Q ++e
"
"
$C ; &&c s AS7 7 TTc
| j d|t f }|j ry d} ddlm} || }| j d|||||t t t ||f
}|j dkD ryt j dd| } t j | } t j d d
| j d d } | j d |j |t | f y
y# t
$ r Y w xY w# t j $ r=}
dt# |
v r*t% j& d t) | ||||| cY d }
~
S d }
~
ww xY w)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?Fotherr )classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)r{ r r r
zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)TlockedrA )execute SITE_NAMEfetchone crawler_libr r
GROUP_NAME SCRIPT_NAMErowcountr r r r r lastrowidsqlite3OperationalErrorra timesleep store_record) connurlrN rX rO r curindustryr summaryes r r r sF
,,PSVXaRb
cC
||~ 5(/H ll
+ #ugtY
KQZ\d e
< r rQ r r r# r% rl r r r r r r r commitcloser )sr r1 new_count
skip_count seen_urlsr< rJ r rK rP rN ldater( dhtmldtitleddaterd rX r _ plain_lenoks r mainr + s AIIW ??7B /DLL,-LL*+AH #2 uS]O7
KSWXIJIa!$ ) "1h5I 9%
dV8CJ
y1#6dCa
s0
J*,J. J+
J&&J+. K7KK__main__)4__doc__sysr r r r5 htmlr urllib.parserE r warningsfilterwarningsbs4r ImportErrorr $requests.packages.urllib3.exceptionsr packagesurllib3disable_warningsrH r: r r r r r! r8 r7 r"
_MAX_PAGESr argvr a
startswithintr
ValueErrorr r r r* r> rQ rl r r r __name__ r r r s !! G * *+A B &
B 6
!
- -
@O'
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA,
&%R_'D D=3@ zF a
M:
s5 D; 6E 2E;EE EEE E