9|j> d Z ddlZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl m Z dZdZdZdZdZej& j) d d
Zej& j) d d Zd
ZdZddddZdZ eej8 D ]k \ Z Z e j? d r e e jC dd d Z4e dk( s:e dz e#ej8 k sS e ej8 e dz Zm edk\ rendZ$d Z%d#dZ&d#dZ'd$dZ(d Z)d Z*d Z+d Z,d Z-d Z.d! Z/e0d"k( r ejb e/ yy# e
$ r dZ Y #w xY w# e"$ r Y w xY w# e"$ r Y w xY w)%u
crawl_jslygc_hpgs.py - 江苏绿源-环评公示
http://www.jslygc.com/list_info.aspx?cid=74 (iframe: list_news_iframe.aspx?cid=74)
列表: ASP.NET WebForms, #news_list ul li > a[href=show.aspx?id=N] + 日期
分页: POST __EVENTTARGET=AspNetPager1&__EVENTARGUMENT=N (需携带最新 VIEWSTATE + Cookie)
详情: show.aspx?id=N → h1(完整标题) + #txt1(正文HTML, 保留表格) + #fj(附件)
注意: 列表标题被截断(无省略号), 完整标题必须从详情页 h1 提取
N)
BeautifulSoupzhttp://www.jslygc.comz2http://www.jslygc.com/list_news_iframe.aspx?cid=74u 江苏绿源-环评公示u 江苏zcrawl_jslygc_hpgs.pyDB_PATHz/root/search.db
JSONL_PATH utf-89 zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z
User-AgentAcceptzAccept-Language z--pages==z--pagesc | syt j | } | j dd j dd j dd j dd } t j dd| } | j S )Nr u u u u \s+ )html_libunescapereplaceresubstrip)ts &/root/gov_crawler/crawl_jslygc_hpgs.py clean_titler : so ! A (B ''"5==hKSST\^`aA
vsA A 779 c t | D ]J } | j |t | }|j dk( r|c S t j d|dz z L y# t $ r Y )w xY w)u& GET with retry (站点间歇性超时))headerstimeout N)rangegetHEADERS status_code Exceptiontimesleep)sessionurlretriesr irs r http_getr* C ss
7^ C' BA}}# $
1q1u9 s (A A&%A&c t | D ]K } | j ||t | }|j dk( r|c S t j d|dz z M y # t $ r Y )w xY w)N)datar r r r )r postr r! r" r# r$ )r% r& r, r' r r( r) s r http_postr. P su
7^ StWg NA}}# $
1q1u9 s )A A'&A'c |dk( rt | t }|&y||dt | d}t | t | }|yt dk( r!|j
j
t d n |j }t j d | }t j d
| }||r|j d n||r|j d fS |fS ) u; 获取列表页. page=1 GET; page>1 POST AspNetPager 回发r
)Nr r AspNetPager1) __VIEWSTATE__VIEWSTATEGENERATOR
__EVENTTARGET__EVENTARGUMENT)r, r r errorsz id="__VIEWSTATE" value="([^"]*)"z)id="__VIEWSTATEGENERATOR" value="([^"]*)") r* LIST_URLstrr. ENCcontentdecodetextr searchgroup) r% pagelast_vslast_vgr) r, r< vsvgs r
fetch_listrD \ s qy Wh ' 9 #$+ + "4y
gxd 3 9 69Wn199 C 2!&&D 6 =B ? FB "((1+2BHHQK SS7 SSr c X g }t j d| t j t j z D ]n }|j d }t |j d }|j d }|rt
| dk rPt d| }|j |||f p |S )u@ #news_list ul li: 标题 2025-07-09zP]*href="show\.aspx\?id=(\d+)"[^>]*>(.*?)\s*([0-9]{4}-[0-9]{2}-[0-9]{2})r
r z/show.aspx?id=) r finditerSIr> r lenBASE_URLappend) html_textitemsmnidtitledateabs_urls r
parse_listrU r s E
[[lnwy{y}y} AC AE AE zE F -ggajAGGAJ'wwqzE
Q JnSE2
gud+,- Lr c d}d}d}g }t rOt | d }|j d }|r t |j dd }|j dd }|r"dj d
|j
D }|j dd } | r| j
d d
D ] }
|
d } t |
j dd xs1 t j j | j d d } t j j || }
|j |
| f |r|j
d d
D ] }
|
d } t j d| t j" s.t |
j dd xs1 t j j | j d d } t j j || }
|
|D cg c] }|d c}vs|j |
| f |sJ|j d }|r7t j d|j }|r|j% d }||||fS c c}w )u5 h1 完整标题 + #txt1 正文HTML + #fj 附件列表r html.parserh1r Tr divtxt1)idc 3 2 K | ] }t | y w)N)r8 ).0cs r z parse_detail.. s "Aa3q6"As fjahrefrd ?r \.(pdf|docx?|xlsx?|zip|rar|txt)h2u" 日期[：:]\s*(\d{4}-\d{2}-\d{2})r
)r findr get_textjoincontentsfind_allospathbasenamespliturllibparseurljoinrM r r= rJ r> )rN page_urlrR rS content_html attachmentssouprX r[ ra rb rd txtrT xrg dms r parse_detailr{ s& E
DLK Y
6
YYt_
Ct <=Eyy6y* 77"A4=="AAL
YYuY
&
[[4[0
3y!!**S*"=>f"''BRBRSWS]S]^aSbcdSeBf ,,..x>""GS>2
3 ]]3T]2
;y99 ?rttL%ajjDj&ABjbggFVFVW[WaWabeWfghWiFjC $ll 2 28T BG [&Aqt&AA#**GS>:
; 4 BYY DbkkmT 88A;D $ k 11 'Bs - I%c
| sddg fS t j dd| t j } t j dd| t j } t j dd| t j t j z } t j dd| t j t j z } g }i }t r t | d }|j d d
D ] }t j d |j d
d t j s9|d
}|j d
xs1 t j j |j d d }t j j || }|j! ||f dt# | d} d| d| d|| < |j% | |j d D ] }
|
j d } | s|
j d
r*g } | D ] }
|
j d xs d}|st j j || }| j! d| dt j j |j d d xs d d | sdt# | d} d d j' | z d z || < |
j% | t) | } g fd }t j d || t j t j z } t j d!| t j rd"nd}t rQ t | d }|j d
D ]% }d#D ] }|j, j/ |d$ ' t) | } t j d d%| t j } t j d&d'| t j } t j d(d| t j } g }| j d) D ] }|j1 }|st j d*d| }t3 j4 | }|j7 d+d, j7 d-d, }t j d.d| }|s~|j9 D ] \ }}|j7 || } t; D ] \ }}|j7 d/| d| } |j! |j1 d)j' | }|j9 D ] \ }}|j7 || } t; D ] \ }}|j7 d/| d| } t j d0d| }t j d1d| }t j d2d| }t j d3d| }t3 j4 | }t j d4d'| }t j d5d)| }t j d6d,| }t= } g } |j d) D ]X }
t j d7dt j d*d|
} | s2| | v r7| j? | | j! |
Z d)j' | j1 }|||fS # t* $ r Y w xY w# t* $ r Y Lw xY w)8uA 清理正文, 保留表格 HTML, 附件链接转绝对 URL 段落r r z
)flagsz ]*>z]*>.*?z]*>.*?rW rb Trc rf rd rY re
__ATTACH____
pimgsrcz AI !'=9E ~~d+
. L .DIIMM$-.
. %jG ffWj'>Gff\4=Gff@"gUWUYUYZG E v& KKM z2q)!!%(
fc*228SA7UC &&( DAq !QA
. 0FAs GA3b/3/A 0
QWWY ++e
C ""$ 1kk!Q M* 03kkGA3b/3/0
&&!2s
+C
&& "c
*C
&& "c
*C
&&#R
-C
C
C
&&dC
(C
&&FC
(C
&& sC
(C 5D E
YYv
ffVR
B!:; $;
Q ++e
"
"
$C ; &&K " sG A[ +C[ >[ [ 0A/[ A[ A[& [#"[#& [32[3c |s| S | j r| gng }|D ] \ }}|j d| d| d dj | S )uC 附件区转 绝对URL
段落, 不保留 ul/li 包装r r r r )r rM rj )r: rv r rT rx s r append_attachmentsr s[ WIbE# N
|G9,>se8LMN ;;u r c | j dt f }|j D ch c] }|d c}S c c}w # t $ r
t cY S w xY w)u7 加载已入库 page_url 集合用于列表级预查重z.SELECT page_url FROM gov_raw WHERE site_name=?r )execute SITE_NAMEfetchallr" r )conncurrows r load_existing_urlsr " sN llKi\Z"%,,.13A111 u s )? :? ? AAc
| j d|t f }|j ry d} ddlm} || }| j d|||||t t t ||f
}|j dkD ryt j dd| } t j | } t j d d
| j d d } | j d |j |t | f y
y# t
$ r Y w xY w# t j $ r=}
dt# |
v r*t% j& d t) | ||||| cY d }
~
S d }
~
ww xY w)Nz7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?Fotherr )classify_industryzINSERT OR IGNORE INTO gov_raw (page_url, source_url, title, content, publish_date, site_name, group_name, script_name, has_table, industry) VALUES (?,?,?,?,?,?,?,?,?,?)r r r
r r zTINSERT OR IGNORE INTO gov_search (rowid, title, site_name, summary) VALUES (?,?,?,?)TlockedrF )r r fetchone crawler_libr r"
GROUP_NAME SCRIPT_NAMErowcountr r r r r lastrowidsqlite3OperationalErrorr8 r# r$ store_record) r r& rR r: rS r r industryr summaryes r r r + sF
,,PSVXaRb
cC
||~ 5(/H ll
+ #ugtY
KQZ\d e
< d t d |d d d | d d |d z
}4tA ||
|||| }|r.|d z
}jC |
t d!|d d d | d d n|d z
}tE jF d" tE jF d# t
rmtI t
d$d%& 5 } |D ]+ } | jK tM jN | d'( d)z - d d d t d*t
d t | d+d | |jQ |jS t d,| d-| d |dkD s|dkD rdS d S # t $ r } t d
| d| d Y d } ~ d } ~ ww xY w# t $ r#} t d|
d| d |d z
}Y d } ~ d } ~ ww xY w# 1 sw Y xY w).Nr < )r zPRAGMA busy_timeout=60000zPRAGMA journal_mode=WALu
已入库 u 条, 用于列表级预查重T)flushr 0C85029Dr
zPage z fetch error (all retries)z fetch error: z: found z itemszEmpty page, stop paginationz detail error (all retries): r r5 z detail error z: r
z skip empty: )rt rR r: publish_dater site_name
group_name script_namez
JSONL + ( z ()z + g333333?g ?wr )encodingF) ensure_asciir zJSONL written: z rows)z
DONE: new=z skip=)*requestsSessionr updater r r connectr r r printrK r _PAGESrD r" rU r* r: r; r9 r{ r r r r r rM r r r r r r# r$ openwritejsondumpscommitclose) r sr existing new_count
skip_count
jsonl_rowsr@ rA r? rN r rO rT rR rS dhtml_rdhtmldtitleddateru attsr: r atts_inall_atts plain_lenokfr s r mainr M s% AIIW D w3 01 ./%d+
3x=/)GHPTUIJJGGa!$ < *4Qgw*O 'Iw dV#=>dK ! 9%
dV8CJfETR
Jyk
|
4DAA
a 16Q6G E$~aS1 > &
y1#6dCa
L D DsB %N>#N."!N.&1O N+
N&&N+. O7OO O&__main__)rF r )r r )2__doc__sysrm r r# htmlr urllib.parserq r r bs4r ImportErrorrL r7 r r r environr r r r9 TOTAL_PAGESr
_MAX_PAGESr argvr( rb
startswithintrp
ValueErrorrK r r r* r. rD rU r{ r r r r r __name__exit r r r s !
# ? '
$
**..$5
6
ZZ^^L"
-
@O'
chh
DAq||J QWWS!_Q /0J
i AECM 1 SXXa!e_-J
"QA
T, '2Tk'\ D[7| z CHHTV k M2
s5 E "E E E
EEE E$#E$