P~j* N d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl
m Z ddl m
Z
dZdZdZd Zd
Z ej& e d z
j) d
Zej, j/ dd ZddiZd Zd Zd!dZ d Z d Z d Z d"dZ e!dk( rejD dd Z#dZ$dZ%d d jM e# v r7 e' ejP d d jM e# jS d Z$d e#v re#e#jU d dz Z% e e$e% yy)#u]
长垣市人民政府 - 公示公告 (changyuan.gov.cn/sitesources/cyxrmzf/page_pc/xxgk/zfxxgkml/gsgg)
CMS: sitesources 定制 (listN.html, 详情 article{hash}.html)
列表: a[href=.../gsgg/article{hash}.html] title
分页: list2.html ...
详情: 正文 div.view.cm / #zoomTY
用法: python3 crawl_changyuan_gsgg.py [--pages=N] [--json out.jsonl]
N)datetime timedelta)
BeautifulSoup)urljoinu" 长垣市人民政府-公示公告zhttp://www.changyuan.gov.cnzUhttp://www.changyuan.gov.cn/sitesources/cyxrmzf/page_pc/xxgk/zfxxgkml/gsgg/list1.htmlzwww.changyuan.gov.cnu 河南iG )daysz%Y-%m-%dDB_PATHz/root/search.dbz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c 2 t dt d| d y )N[z] T)flush)print SITE_NAME)msgs )/root/gov_crawler/crawl_changyuan_gsgg.pylogr s Ai[3%
- c t j dd| xs d } t j dd| } t j dd| j } | S )Nz | |\u200b|\ufeff z\s+ u \.\.\.|…$)resubstrip)ts r clean_titler sK
-r17;A
vsA A
~r1%++-A Hr c ` t j | t | }d|_ |j S )N)headerstimeoututf-8)requestsgetHEADERSencodingtext)urlr rs r fetchr% % s$ S'7;AAJ 66Mr c N t | d }g }|j dd D ] }|j dd }d|vsd|vr t |j d d xs |j d
}|sPd}|j d d g }|r7t
j d
|j }|r|j d }|j |t t | |d t } g }
|D ]/ } | d | vs | j | d |
j | 1 |
S )N html.parseraThrefr* r article.htmltitler lidiv20\d{2}-\d{2}-\d{2}r )r- r# pub_dater# )r find_allr r get_text find_parentr searchgroupappendr BASE_URLsetadd) htmlsoupitemsr( r* r- r2 parentmseenuniqits r
parse_listrD + s& } -D E
]]3T]
*
]uuVR D G4$7 AEE'2 . H!**4*2HI e}- 0&//2CDA 771:
uWXt-DRZ[\
] 5D
D
e9D HHRY KKO Kr c F dddd}t | d }|j d }|rS|j d }t j dd| }t j dd| }|j |d< |j d d
d i }|r |j
d
r |d
d d |d< |d s/t j d| d d }|r|j d |d< |j dd }|s|j dd }|rt t | |d
< |S )Nr r- publish_datecontentr' r- Tr. u3 [_\-—]\s*(公示公告|长垣市人民政府)\s*$u$ [_\-—]\s*长垣市人民政府\s*$metanamePubDate)attrsrH
rG r1 i@ r r0 zoomTY)idzview cm)class_) r findr4 r r r r r6 r7 extract_clean_textstr)r< resultr= title_tagr meta_dater@ content_divs r parse_detailrX E s% 2"
=F } -D '"I T * FFI2q Q FF:B B'')w &(; "))Eh)/K iiii8 .s;/? @y Mr c | syt | d }|j ddg D ] }|j |j d D ]9 }|j dd t fddD s*|j ; |j d d
D ] }|j d d j }|j
d
xs5 t j j |j d d xs d}|r|j d r|j t t | }|j d |d }||_ |j# | |j g d D ] }|j g } |j d D ] }
|
j% d d
} | s|
j' d r+| j d d j }| j
d
xs5 t j j |j d d xs d}|s|j d r| j) |t t | f |j d D ]8 }
|
j% d d
s|
j' d r)|
j : |j ddg D ]9 }|j
d
r|j% g d r*|j ; ddlm} g }
|j ddg D ]J }|j. dk( r|
j) t1 | .|j. dk( s>|j' d rPg }|j2 D ] }t5 |dd }|dk( rB|j dd t fd d D r<|j) t1 | W|d v r|j) t1 | v|r|j) t1 | |j) | t1 | dj7 | j }|s6|
j) d | d! M t9 }| D ]7 \ }}||f}||v r|j; | |
j) d"| d#| d$ 9 |j dd%g D ]o }|j' d r|j. dk( r%|j% g d& r9dj7 d' |j2 D j }|r)t= | d(k\ r|
j) d | | d! |j d d
D ] } | j d d j }| j
d
xs5 t j j |j d d xs d}|so|j d r|t t | f}||v r|j; | |
j) d"|d) d#|d d$ r |
rd*j7 |
S | | j S )+u 输出纯 HTML：段落 、链接 、表格保留、图片 ![]()
、附件独立成段。
禁止混合格式（纯文本 + 原始 HTML 会触发 search_app HTML 分支导致换行折叠）。r r' scriptstyleimgsrcc 3 & K | ] }|v
y wN .0ksrc0s r z%extract_clean_text..h s aQqDyas )z
un-collectcollectsharefaviconqrcodeewmr( Tr) r* r. ?r u 附件)z javascript:#_blank)r* target)spanbstrongfontemiusr/ tabler0 p)r\ rw r( )escaperJ Nc 3 B K | ] }|j v y wr_ )lowerra s r re z%extract_clean_text.. s [1
,[s )z.gificon16fileattach)r( rw brzz
z z
td)rx rw ulc 3 T K | ] }t |t st | " y wr_ )
isinstancerS )rb cs r re z%extract_clean_text.. s LAAs9KQLs (( z
) r r3 decomposer anyr r4 ospathbasenamesplit
startswithunwrapr r9 new_tagstring replace_withrQ r5 r8 r< ry rJ rS contentsgetattrjoinr: r; len) content_htmlr= tagr\ a_tagr* r" full_urlnew_a attach_linksr/ a_inry partselinnerr rJ r seen_ar# keyr0 directrd s @r rR rR ] s } 5D}}h 01
}}U# wwub! a"`a a MMO
s . "yy$**,~~D~)]RWW-=-=djjoa>P-Q]U]t';< LLN 8T* Sx A
5! " }} QR
LmmD! Ewwsw& w/88FB'--/D==t=,`0@0@C QRAS0T`X`DDOO,@ A ##T78T+B$CD
E mmD!
773T7 "2>>'+B LLN }}eS\* ||$|'9N0O MMO EmmWcN+ ,
77g LLR !
WW^BNN7$;E[[
1 q&$/5= 55+D[6Z[[ LLQ( 11LLQ(LLQ(LLA 0
1 $$&A s1#T]++,. UF! K cSk &=
3
|C5(:4&IJ K }}eT]+ X ??7 #E(9 88( ) L LLRRT c&kQ& LL3vf~ .d3 4LL4L0 XD88FB'--/D==t=,`0@0@C QRAS0T`X`DDOO,@ AWXt 45&=
3 |CF83Ec!fXX VW XX$ "'6;;u HF<,@,F,F,HHr r c
t d| dt g }t }t d| dz D ] }|dk( rt n t j dd| dt } t | }t | }d } |D ]4 }
|
d
|vs |j |
d
|j |
| dz
} 6 t d| d t | d | d
t | |s nt j d g } t |d D ] \ } }
|
d r
|
d t k r t |
d
}t! | }
| j |
d xs |
d |
d
|
d xs |
d |
d t" dd | dz d k( rt d| dt | t j d |rct% |d d 5 }| D ]+ }|j' t) j* |d ! d"z - d d d t d#t | d$| y t- j. t0 }|j3 d% |j5 }d x}}| D ]W } |j3 d&|d' |d( |d( |d |d |d |d xs dd d) |d* d+f |j6 d kD r|dz
}n|dz
}Y |j9 |j; t d-| d.| d/ y # t $ r }t d| d| Y d }~ d }~ww xY w# t $ r(}t d|
d
dd d| dddd}
Y d }~d }~ww xY w# 1 sw Y OxY w# t $ r }t d,| |dz
}Y d }~d }~ww xY w)0Nz Crawl pages=z cutoff=r z list1\.html$listr, Pz ERR: r r# z: z items (z
new), total g?r2 z detail ir rF r- rG rH u 公示公告)r- page_urlrG rH site_namecolumnrM z /g?wr )r! F) ensure_ascii
zJSONL: z
items -> zPRAGMA busy_timeout=10000zINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name) VALUES (?,?,?,?,?,?,?,?,?)r r i r zcrawl_changyuan_gsgg.pyz
DB error: zResult: z new, z skipped) r CUTOFFr: rangeLIST_URLr r r% ExceptionrD r; r8 r timesleep enumeraterX r
openwritejsondumpssqlite3connectr executecursorrowcountcommitclose)pagesjson_out items_allrA rx r# r< er> newrC enrichedidxdetailfitemconnr storedskippeds r crawlr s ,ugXfX./I 5D
1eai
q&hbff_QCuox&X :D 4 B%y$E# $q
as"SZL]3y>:J KL
3#& H Y* R
j>bnv 5 FE#D!$'F G_37 5 "> 2 Dbn i("$
8q= "SE3y>*+ ,
3'*
(C'
2 Ea
E
4e31L2 AL? K; K66K;> L/ L**L/2L<? M&M!!M&__main__ z--pages=r z
--pages=(\d+)z--json)r r ) )r N)+__doc__r sysr r r r r r r bs4r urllib.parser r
r9 r DOMAINGROUPnowstrftimer environr r r r r r% rD rX rR r __name__argvargsr r r intr6 r7 indexr` r r r s7 2 1 1 1 ( 0 ( b
(,,.9%0
0 : :: F
**..$5
6 K L.
40^IBJ4Z z 88AB