P~jj& N d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl
m Z ddl m
Z
dZdZdZd Zd
Z ej& e d z
j) d
Zej, j/ dd ZddiZd Zd Zd!dZ d Z d Z d Z d"dZ e!dk( rejD dd Z#dZ$dZ%d d jM e# v r7 e' ejP d d jM e# jS d Z$d e#v re#e#jU d dz Z% e e$e% yy)#u/
中咨华宇 - 环评公示 (chinaeia.cn/channel.aspx?id=124)
CMS: ASP.NET (channel.aspx?id=N&page=N, 详情 content.aspx?id=N)
列表: a[href=content.aspx?id=N]
分页: ?page=2 ...
详情: 正文 div.nr3 (含 title 去后缀)
用法: python3 crawl_chinaeia_hpgongshi.py [--pages=N] [--json out.jsonl]
N)datetime timedelta)
BeautifulSoup)urljoinu 中咨华宇-环评公示zhttp://www.chinaeia.cnz*http://www.chinaeia.cn/channel.aspx?id=124zwww.chinaeia.cnu 北京iG )daysz%Y-%m-%dDB_PATHz/root/search.dbz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c 2 t dt d| d y )N[z] T)flush)print SITE_NAME)msgs -/root/gov_crawler/crawl_chinaeia_hpgongshi.pylogr s Ai[3%
- c t j dd| xs d } t j dd| } t j dd| j } | S )Nz | |\u200b|\ufeff z\s+ u \.\.\.|…$)resubstrip)ts r clean_titler sK
-r17;A
vsA A
~r1%++-A Hr c ` t j | t | }d|_ |j S )N)headerstimeoututf-8)requestsgetHEADERSencodingtext)urlr rs r fetchr% % s$ S'7;AAJ 66Mr c t | d }g }|j dd D ] }|j dd }d|vrt |j d }|s8d}|j d d
g }|rjt
j d |j }|rD|j d j d
d j dd j dd d d }|j |t t | |d t } g }
|D ]/ } | d | vs | j | d |
j | 1 |
S )N html.parseraThrefr* r zcontent.aspx?id=r lidiv# 20\d{2}[-/年]\d{1,2}[-/月]\d{1,2}r 年- 月/
)titler# pub_dater# )r find_allr r get_text find_parentr searchgroupreplaceappendr BASE_URLsetadd) htmlsoupitemsr( r* r4 r5 parentmseenuniqits r
parse_listrH + sA } -D E
]]3T]
*
]uuVR T ) AJJTJ 23 e}- @&//BSTA 771:--eS9AA%MUUVY[^_`cacd
uWXt-DRZ[\
] 5D
D
e9D HHRY KKO Kr c dddd}t | d }|j d }|r<|j d }t j dd| }|j |d< |j dd d
i }|r |j
d r |d d d
|d< |d sbt j d| d d }|rG|j d j dd j dd j dd d d
|d< |j dd xs |j dd }|s|j dd }|rt t | |d < |S )Nr r4 publish_datecontentr' r4 Tr+ u [_\-—]\s*中咨华宇\s*$metanamePubDate)attrsrL r3 rK r. i@ r r/ r0 r1 r2 dlnr3)class_r- ny_main) r findr7 r r r r r9 r: r; extract_clean_textstr)r@ resultrA title_tagr meta_daterD content_divs r parse_detailr\ E sW 2"
=F } -D '"I T * FF12q 9'')w &(; "))D)/Q499U593QK iiii8 .s;/? @y Mr c 6
| syt | d }|j ddg D ] }|j |j d D ] }d|_ |j d D ] }d|_ |j d D ]9 }|j d d t fd
d D s*|j ; |j d d
D ] }|j dd j
}|j d
xs5 t j j |j d d xs d}|r|j d r|j t t | } |j! d | d }
||
_ |j% |
|j g d D ] }|j g } |j d D ] } | j' d d
}
|
s| j) d r+|
j dd j
}|
j d
xs5 t j j |j d d xs d}|s|j d r| j+ |t t | f |j d D ]8 } | j' d d
s| j) d r)| j : |j ddg D ]9 }|j d
r|j' g d r*|j ; dd lm} g }|j ddg D ]J }|j dk( r|j+ t1 | .|j dk( s>|j) d rPg }|j2 D ] }t5 |d d }|dk( rB|j d d t fd d D r<|j+ t1 | W|d!v r|j+ t1 | v|r|j+ t1 | |j+ |t1 | dj7 | j
}|s6|j+ d"| d# M t9 }| D ]7 \ }}||f}||v r|j; | |j+ d$| d%| d& 9 |rd'j7 | S || j
S )(u 输出纯 HTML：段落 、链接 、表格保留、图片 ![]()
、附件独立成段。
禁止混合格式（纯文本 + 原始 HTML 会触发 search_app HTML 分支导致换行折叠）。r r' scriptstyledtpddimgsrcc 3 & K | ] }|v
y wN .0ksrc0s r z%extract_clean_text..l s aQqDyas )z
un-collectcollectsharefaviconqrcodeewmr( Tr) r* r+ ?r u 附件)z javascript:#_blank)r* target)spanbstrongfontemiusr, tabler- )rc r~ r( )escaperN Nc 3 B K | ] }|j v y wrf )lowerrh s r rl z%extract_clean_text.. s [1
,[s )z.gificon16fileattach)r( r~ brzz
z z
z
) r r6 decomposerN r anyr r7 ospathbasenamesplit
startswithunwrapr r= new_tagstring replace_withrU r8 r< r@ r rW contentsgetattrjoinr> r? ) content_htmlrA tagr` rb rc a_tagr* r" full_urlnew_a attach_linksr, a_inr partselinnercrN r seen_ar# keyrk s @r rV rV \ s } 5D}}h 01
mmD! mmD! }}U# wwub! a"`a a MMO
s . "yy$**,~~D~)]RWW-=-=djjoa>P-Q]U]t';< LLN 8T* Sx A
5! " }} QR
LmmD! Ewwsw& w/88FB'--/D==t=,`0@0@C QRAS0T`X`DDOO,@ A ##T78T+B$CD
E mmD!
773T7 "2>>'+B LLN }}eS\* ||$|'9N0O MMO EmmWcN+ ,
77g LLR !
WW^BNN7$;E[[
1 q&$/5= 55+D[6Z[[ LLQ( 11LLQ(LLQ(LLA 0
1 $$&A s1#T]++,. UF! K cSk &=
3
|C5(:4&IJ K "'6;;u HF<,@,F,F,HHr c
t d| dt g }t }t d| dz D ] }|dk( rt n
t d| } t | }t | }d} |D ]4 }
|
d |vs |j |
d |j |
| dz
} 6 t d| d t | d
| d t | |s nt j d g } t |d D ] \ } }
|
d
r
|
d
t k r t |
d }t | }
| j |
d xs |
d |
d |
d xs |
d
|
d t dd | dz dk( rt d| dt | t j d |rct! |dd 5 }| D ]+ }|j# t% j& |d d z - d d d t d!t | d"| y t) j* t, }|j/ d# |j1 }dx}}| D ]W } |j/ d$|d% |d& |d& |d |d |d |d xs dd d' |d( d)f |j2 dkD r|dz
}n|dz
}Y |j5 |j7 t d+| d,| d- y # t $ r }t d| d| Y d }~ d }~ww xY w# t $ r(}t d|
d dd d| dddd}
Y d }~d }~ww xY w# 1 sw Y OxY w# t $ r }t d*| |dz
}Y d }~d }~ww xY w).Nz Crawl pages=z cutoff=r z&page=Pz ERR: r r# z: z items (z
new), total g?r5 z detail ir rJ r4 rK rL u 环评公示)r4 page_urlrK rL site_namecolumnr3 z r2 g?wr )r! F) ensure_ascii
zJSONL: z
items -> zPRAGMA busy_timeout=10000zINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name) VALUES (?,?,?,?,?,?,?,?,?)r r i r zcrawl_chinaeia_hpgongshi.pyz
DB error: zResult: z new, z skipped) r CUTOFFr> rangeLIST_URLr% ExceptionrH r? r< lentimesleep enumerater\ r
openwritejsondumpssqlite3connectr executecursorrowcountcommitclose)pagesjson_out items_allrE ra r# r@ erB newrG enrichedidxdetailfitemconnr storedskippeds r crawlr s ,ugXfX./I 5D
1eai
q&h
&&< :D 4 B%y$E# $q
as"SZL]3y>:J KL
3#& H Y* R
j>bnv 5 FE#D!$'F G_37 5 "> 2 Dbn i("$
8q= "SE3y>*+ ,
3'*
(C'
2 Ea
E
4er s7 2 1 1 1 ( ' # 7
(,,.9%0
0 : :: F
**..$5
6 K L.
4.PIfJ4Z z 88AB