z~j% ` d Z ddlZddlZddlZddlZddlZddlZddl Z ddl
Z
dZ dZ dZ
e de de
ZdZd Zd
Zd Z dd lmZ d
ZdZddddZdZ eej6 D ]3 \ Z Z e j= d s e e jA dd d Z5 edk\ rendZ"dZ# ejH d ejJ ej e#dz z
Z&d Z'd%dZ(d Z)d Z*d Z+d Z,d&d Z-d Z.d! Z/e0d"k( r dZ1d#ej6 v rd$Z1 e-e"e1 yy# e$ r dZY w xY w# e!$ r Y w xY w)'uB
G批 秦皇岛市生态环境局-公告公示 (AJAX 分页)
http://sthj.qhd.gov.cn/home/list?pcode=...&code=...
列表: 标题 (含日期)
分页: POST /home/listPage {pageNumber: N, ...} (totalPageValue=21)
详情: /home/details?code=...&id=N -> div.content.fix
Nzhttp://sthj.qhd.gov.cnz1NDAyOGFiODE2MzI5MTVjZDAxNjMyOTFhZjY1ZjAwMTY%ce%b3z1NDAyOGFiODE2MzI5MTVjZDAxNjMyOTIxMTYxMDAwNjM%ce%b3z/home/list?pcode=z&code=u( 秦皇岛市生态环境局-公示公告u 河北zcrawl_qhd_sthj_gsgg.pyu 环境保护)
BeautifulSoupz/root/search.dbutf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-Language z--pages==iG z%Y-%m-%diQ c | syt j | } | j dd j dd j dd j dd } t j dd| } | j S )N u u u u z\s+ )html_libunescapereplaceresubstrip)ts (/root/gov_crawler/crawl_qhd_sthj_gsgg.py clean_titler 0 so ! A (B ''"5==hKSST\^`aA
vsA A 779 c t d D ]q } dd l}dd l}|j |j j
|j
| t |d }|j dk( rt |_
|j c S s y # t $ r Y w xY w)N r F)headerstimeoutverify )
rangerequestsurllib3disable_warnings
exceptionsInsecureRequestWarninggetHEADERS status_codeENCencodingtext Exception)urlr _r r rs r fetchr+ 9 s
1X $ $ $W%7%7%N%N O S'75 QA}}#
vv
$ s A+B B
B
c n ddl }ddl}|j |j j t
dt dt d}t t }d|d< d|d < |j ||d
t | id d
}|j dk( rt |_ |j S y# t $ r Y yw xY w)uH AJAX 分页: POST /home/list/listPage?code=..&pcode=..&refresh=..&url=..r Nz/home/list/listPage?code=z&pcode=z3&refresh=bmV3c0xpc3Q%ce%b3&url=L2hvbWUvbGlzdC5odG1sXMLHttpRequestzX-Requested-Withz%http://sthj.qhd.gov.cn/home/list.htmlReferer
pageNumber F)r datar r r )r r r r r BASE_URLCODEPCODEdictr" poststrr# r$ r% r& r' )page_idxr r api_urlhr* s r
fetch_listr; G s W / / F FG4TF'%DFG W
A ,A:AiL
MM'1L#h-3PZ\ejM k ==C AJ66M
s AB( ( B43B4c g }t |S t | d }|j dd D ] }|j dd j }d|vr(t |j dd xs |j d }|rt
| d
k rhd}|j g d }|r7t j d |j }|r|j d
}t j j t | } |j || |d t! }
g } |D ]/ } | d |
vs |
j# | d | j | 1 | S )N html.parseraThrefr@ r z
/home/detailstitler )lidivtdtr20\d{2}-\d{1,2}-\d{1,2}r )rA r( pub_dater( )r find_allr! r r get_textlen find_parentr searchgroupurllibparseurljoinr2 appendsetadd)
htmlitemssoupr> r@ rA pdparentmr( seenuniqits
r
parse_listr_ Z sH E } -D
]]3T]
* CuuVR &&( $ & AEE'2 . H!**4*2HIE
Q
89 4foo6GHAWWQZll""8T2
uSbAB C 5D
D
e9D HHRY KKO Kr c z | syt | d }t }|j ddg D ] }|j |j dd D ] }|j dd j }|j
d xs5 t j j |j d
d xs d }|r|j d
r|j t j j t | }|j# d|d }||_ |j' | |j g d D ] }|j |j ddg D ]9 }|j
d r|j) g d r*|j ; d dlm} g }
|j ddg D ]# } | j. dk( r|
j1 t3 | .| j. dk( s>| j5 d rPg } | j6 D ] }
t9 |
dd }|dk( r| j1 t3 |
0|dv r| j1 t3 |
O|r| j1 t3 |
l| j1 | t3 |
dj; | j }|s|
j1 d| d & |j ddg D ] }|j5 d r|j. dk( r%|j) d r7|j) g d rKdj; d |j6 D j }|r)t= | d k\ r|
j1 d | | d |j dd D ] }|j dd j }|j
d xs5 t j j |j d
d xs d }|so|j d
rt j j t | }||v r|j? | |
j1 d | d | d! |
rd"j; |
S | | j S )#Nr r= scriptstyler> Tr? r@ rB ?r u 附件)z javascript:#_blank)r@ target)spanbstrongfontemiusrE p)imgtabler> )escaperq namerp )r> rq brzz
rF )ro rq ulc 3 T K | ] }t |t st | " y w)N)
isinstancer7 ).0cs r z%extract_clean_text.. s LAAs9KQLs ((r z z
z
) r rT rJ decomposer! r rK ospathbasenamesplit
startswithunwraprP rQ rR r2 new_tagstring replace_withfindrV rr rs rS r7 rM contentsgetattrjoinrL rU ) content_htmlrX seen_urlstaga_tagr@ r& full_urlnew_arr partselinnerry rs r rE directa_ins r extract_clean_textr w s } 5DI}}h 01
s . "yy$**,~~D~)]RWW-=-=djjoa>P-Q]U]t';< LLN <<''$7 Sx A
5! " }} QR
}}eS\* ||$|'9N0O MMO EmmWcN+ ,
77g LLR !
WW^BNN7$;E[[
1 q&$/5=LLQ( 11LLQ(LLQ(LLA 0
1 $$&A s1#T]+#,$ }}eT]+ X ??7 #E(9 88E? 88( ) L LLRRT c&kQ& LL3vf~ .d3 4LL4L0 XD88FB'--/D==t=,`0@0@C QRAS0T`X`DDOO,@ A!<<//$?y(
h' |H:5GvX VW XX( "'6;;u HF<,@,F,F,HHr c dddd}t | d }|j d }|r t |j d |d< t j
d| d d }|r|j
d |d
< |j d d
xs( |j d d
xs |j d d }|r?|j d d
D ] }|j t t | |d< |S )Nr )rA publish_datecontentr= rA TrB rH i@ r r rE r )class_z content fixr )idmbx) r r r rK r rN rO rJ r{ r r7 )rV resultrX title_tagr[ content_divr s r parse_detailr s 2"
=F } -D '"I%i&8&8t&8&DEw
,d5Dk:A!"~ ))E')2~diimi6\~`d`i`ijot}`i`~K ''e'< C MMO .s;/? @y Mr c dx}}t }t d| dz D ] }t | }|s nt | }|s n|D ] }|d |v r |j |d |d r
|d t
k r1t
t |d xs d } | d xs |d | d< | d xs |d | d< t | || | d r|dz
}|dz
} t j d t d
| d | d y )
Nr r r( rI r rA r r g333333?zResult: z stored, z skipped) rT r r; r_ rU _CUTOFF_DATEr r+ _storetimesleeplog)
pagesjson_outstoredskippedr pagerV rW r^ details
r crawlr s" FWIa# $ 4
B%yI% MM"U) $*~"Z.<"?!%5 "2"8b9F $Wo <GF7O%+N%;%Mr*~F> " 62x (i ! 1
3+, (6()G9H56r c
H t |d |d | d | d | d | d xs dd d t t d }|rAt |dd
5 }|j t j |d d
z d d d y t j t d }|j d |j } |j d|d |d |d |d |d |d |d |d |d f |j |j y # 1 sw Y y xY w# t $ r Y (w xY w)Nr( rA r r r i ) site_name
source_urlpage_urlrA r r summarycategory script_namer> r )r% F) ensure_ascii
< )r zPRAGMA busy_timeout=60000zINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name) VALUES (?,?,?,?,?,?,?,?,?)r r r r r r ) SITE_NAMECATEGORY SCRIPT_NAMEopenwritejsondumpssqlite3connectDB_PATHexecutecursorcommitr' close)r r^ r rowfconnry s r r r sA iuI ~.)$9%+Tc2 "
C
(C'
2 @a
GGDJJs7$> ? @ ??7B /DLL,-
A
*
s< 0#j/3w<
#i.#i.#j/SVWdSe
g h
JJL! @
s *D 2AD D D! D!c 2 t dt d| d y )N[z] T)flush)printr )msgs r r r s Ai[3%
-r __main__z--jsonz/tmp/crawl_test.jsonl)r0 )r N)2__doc__sysr| r r rV r urllib.parserP r r r2 r4 r3 LIST_URLr
GROUP_NAMEr r bs4r ImportErrorr r$ r"
_MAX_PAGES enumerateargvrl r> r intr
ValueError_PAGES _DAYS_CUTOFFstrftime localtimer r r+ r; r_ r r r r r __name__r r r r s H G G G #;:Z(vdV < 6
& !
@O'
chh DAq||J QWWS!_Q /0J "QA t}}Z lUZFZ8Z)[\ &:>IB$78 @. zH388* &( g M" s# D 9D%D"!D"%D-,D-