j ` d Z ddlZddlZddlZddlZddlZddlmZ ddlmZ ddl Z e j dZ dZ dZ
ej dd Zd
Zd ej" v Zd d
ddZ ej( e Zej- Zh dZd Zd Zd Zd Z d Z d Z d Z e dk( r e yy)u`
多氟多新材料股份有限公司 - 公示公告 爬虫
https://www.dfdchem.com/News/5.html
N)
BeautifulSoup)datetimezhttps://www.dfdchem.comz#https://www.dfdchem.com/News/5.htmlu 多氟多新材料 SEARCH_DBz/root/search.db z
--incrementalzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentAcceptzAccept-Language>) pdddldth1h2h3h4h5h6hrlioltdthtruldivnavpreformmainasidetabletbodytfoottheadcenterdialogfigurefooterheaderhgroupaddressarticlecaptiondetailssectionfieldset
blockquote
figcaptionc | yddl }ddlm g fd | dj }|j dd| }|j dd | }|j S )
ul 块级边界出换行、行内节点直接拼接、
出换行（≈ 浏览器看到的换行结构）。N r )NavigableStringc F | j D ] }t | rj t | *t |dd dk( rj d Lt |dd t
v r+j d | j d | y )Nnamebr
)children
isinstanceappendstrgetattr _BLOCK_TAGS)nodechr3 outwalks "/root/gov_crawler/crawl_dfdchem.pyrA zbody_text..walk3 s --
B"o.
3r7#VT*d2
4 VT*k9
4 R
4 R
z[ \t\r\f\v]*\n[ \t\r\f\v]*r7 z\n{3,}z
)rebs4r3 joinsubstrip)el_retr3 r@ rA s @@@rB body_textrL + sb z#
C H
A -tQ7A 61%A 779rC c b t j | t dd }d|_ |j S )N F)headerstimeoutverifyzutf-8)requestsgetHEADERSencodingtext)urlrs rB
fetch_pagerY F s& S'2eDAAJ 66MrC c t t | d }g }|j d D ] }|j d }|r|j d nd}|j d }|s>|j d }|j dd }|r|sg|j d s t |z }|j |||d
|S ) N html.parserz.cbox-1.p_loopitemz.e_timeFormat-5.s_titleTrH r2 z.e_text-6.s_title ahrefhttp)titlerW date)r select
select_oneget_textrS
startswithBASE_URLr: ) htmlsoupitemsitem_divdate_eldate_strlinkr_ r] s rB
parse_listrm K s } -D EKK 45 F%%&?@3:7##$#/""#89
D
)xx#D v&d?D
uT8DE F LrC c
t | d }|j d }|r|j d n|d }|j d }|r|j d n|d }d}d}g } |j d }
|
rt |
}t |
}|
j
d
d D ] } | j d d
| j d } | s&
j d
d j d d } t
fddD sd| j |
j d r
nt
z d t | dk r'| r%d|d d| ddj d | D z }||||| d S ) Nr[ r Tr\ r_ z.e_timeFormat-8.s_titler` r2 z.e_richText-14a)r] r] /?r c 3 B K | ] }|j v y w)N)lower).0extr] s rB z parse_detail..o s e33$**,&es )z.docz.docxz.pdfz.xlsz.xlsxz.zipr^ )rV rW u
附件：
r7 c 3 : K | ] }d |d d|d d yw)ry rW rz rV zN )ru ro s rB rw z parse_detail..s s[ gl [\iuvwx}v~u @B CD EK CL BM MU hV gls )r_ r` content_html content_text attachments)r findrc rb r; rL find_allrS splitanyr: rd re lenrF )rf itemrg r detail_titlerj detail_dater} r~ r richro rV r] s @rB parse_detailr ] s } -D
4 B.02;;T;*d7mLoo 78G29'""".tF|KLLK ??+ ,D 4y s . pA55$D::D:)Dzz#r*005a8e2dee""D QWAX^fim^m#no
p <2 +%d5k]"\NBYZ]a]f]f gl `k gl ^l l $$" rC c x t j d| |t df t j y # Y y xY w)NzVINSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)r2 )cexecute SITE_NAMEconncommit)row_idr_ s rB sync_ftsr | s5
j5)R0 2
s 25 9c 4 | d }|j dd j dd }| d r| d d d nd}t j d|f t j }|r|d nd }|rt sn|ry | d
r| d
n| d }| d rt j | d d nd} t j d
|| d || d t |||f |xs t j }|rt || d y# t $ r} t d| Y d } ~ y d } ~ ww xY w)NrW zhttp://r2 zhttps://r~ i z)SELECT id FROM gov_raw WHERE page_url = ?r Fr} r ) ensure_asciizINSERT OR REPLACE INTO gov_raw
(page_url, title, content, publish_date, site_name, source_url, summary, attachments)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)r_ r` Tz
[DB ERROR] ) replacer r fetchone INCREMENTALjsondumpsr lastrowidr Exceptionprint)
r page_url display_urlsummaryrow existing_idcontent_to_storeattachments_jsonr es
rB insert_articler s6 E{H""9b199*bIK,0 ,@d>"4C(bGII9H;G
**,C #a&TK; /3N/CtN+nI]NRS`Natzz$}"5EJgi / tG}&6V
W.>
@ A
+ VT'] +
aS!"s (A
C6 6 D?DDc J g } t rdnt }t dt dt rdnd d t d|dz D ] }|dk( rt
}n|dz
dz }d| d }t d
| d | t
| }t | }t d
t | d |s nN|D ]G } t
|d } t | | }
| j |
d |d |
d |
d |
d |
d d I t dt | d} | D ] }t | s| dz
} t j t j t d| d y # t $ r}t d | Y d }~5d }~ww xY w# t $ r }t d|d d | Y d }~d }~ww xY w)N z=== u -公示公告 - u 增量u 全量u
爬取 === z1https://www.dfdchem.com/News/2021778837727387648-z-16.htmlzFetching page z: z -> Error fetching: z -> z itemsrW z -> Error detail r_ r` r} r~ r )r_ rW r` r} r~ r z
Total unique items: r z Inserted z new records (auto-syncs FTS))r MAX_PAGESr r rangeLIST_URLrY r rm r r r: r r r close) all_itemspages_to_fetchpagerW offsetrf r rh r detail_htmldetailinserteds rB r r s I%Q9N D + H+RR\
]^a !+, 19CQh"_FEfXXVC
tfBse,- c?D 4
c%j\() D
(e 5 %k48
E{ v &~ 6 &~ 6 %m 4
! @
"3y>"2
34H $ MH KKMJJL IhZ <
=>= )!- .
*4;-r!=>
s0 - E%E; E8 E33E8; F"F F"__main__)!__doc__ossysr sqlite3rR rE r r urllib3disable_warningsre r r getenvDB_PATHr argvr rT connectr cursorr r= rL rY rm r r r r __name__r| rC rB r s
$ 0
"))K!2
3
) DO0 ww KKM * 6
$ >
>,?\ zF rC