ڵj2 d Z ddlZddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
ddl m Z dZ
dZdZd Zd
Zd Zd Zd
dddZg dZd Zd Zd Zd Zd Zd Zd Z d Z d Z d Z d Z e!d k( r e yy) u
Crawler for 石大胜华新材料集团 - 信息公开 (Enterprise)
CMS: Custom PHP (Yunyou Network)
List: div.newslist > h3 > a[href], p.newslisttime (YYYY.MM.DD)
Pagination: list.php?catid=65&page=N, 6/page, ~29 pages
Detail: div.newsshowc content, title from div.newsshowt h3
Usage:
python3 crawl_sinodmc_xxgk.py # Full crawl
python3 crawl_sinodmc_xxgk.py --max-pages 1 # Incremental
N)datetime)
BeautifulSoup)urljoinz/root/search.dbu( 石大胜华新材料集团-信息公开u 信息公开u
企业-山东zhttps://www.sinodmc.comz)https://www.sinodmc.com/list.php?catid=65 zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentAcceptzAccept-Language) u 【打印本页】u 【关闭窗口】u 打印本页u 关闭窗口u 转载分享：u 浏览量：u 相关附件：u 相关稿件：u 扫一扫在手机打开u 附件：u 累计次数：c D g }| j D ]p }t |t r%|j }|s'|j | 9|j
dk( rp|j
dd j }|j d }|r'|r%t || }|j d| d| d |s|j | |j
d k( rR|j
d
d }|s|j
d d } t || }
|j | rd|
d nd|
d |j
d
k( r|j d ;t |d sI|j dd }|s`|j | s dj | j S )Nahref Tstrip
imgsrcalt ">查看图片br
get_text ) children
isinstancestrr
appendnamegetr r hasattrjoin) p_tag
detail_urlpartselemtr
textfullr r full_srcs '/root/gov_crawler/crawl_sinodmc_xxgk.pyrender_paragraphr* , sv E dC
A Q
YY#
88FB'--/D==t=,D z40 |D6D6 BC T"
YY%
((5"%Chhub)":s3 PS|H:5K L[ghpgq rH ZI J
YY$
LL
T:
&
c
.A Q1 2 88E? "" c g }| j d D ]L }t | }t |j d dk\ r|j | |j N |S )z>Extract elements as raw HTML. Remove from content_div.tableTr )find_allr lenr r decompose) content_divr" tablesr-
table_htmls r) extract_tables_as_htmlr5 J s_
F %%g. Z
u~~D~) *b 0 MM* %
Mr+ c g }| j d D ]b }|j dd }|st || }|j dd }|j |rd| dnd| d |j d |S )Nr r r r r r )r/ r r r r1 )r2 r" imgsr r r( r s r) extract_imagesr8 U s
D##E* ggeR z3/H''%$C KK3,xj0FGVbckbl mC UD
E MMO
Kr+ c J g }| j ddg D ] }|j t | | }t | | }t | j
D ]u }|j |j j }|dk( r|j d rCd|j
D ]U }t |t r |j } | s'| dz z
0|j dk( r1|j d rQ|j dd } | sg| dz z
p|j d
k( rd z
|j d k( rM|j d
d }
|j d } |
r| rt ||
} d| d| dz
| s؉| dz z
|j dk( rG|j dd }
|
s|j dd }t ||
}|rd| dnd| dz
7|j dd } | sN| dz z
X t! j" dd j st% fdt& D r|r
|d k7 s|j) |dk( s|j dd D ]M }|j dd st% fdt& D r2|r |d k7 s=|j) O x d j+ | }|r|d d j+ | z z
}|r|d d j+ | z z
}|j S ) Nscriptstylepr- r r spanTr r r r r
r r z r r r u ">查看图片 \s+c 3 & K | ] }|v
y wN .0sktxts r) z"extract_content.. s >RrSy > divF) recursivec 3 & K | ] }|v
y wr@ rA rB s r) rF z"extract_content.. s "B29"BrG
)r/ r1 r5 r8 listr r lower find_parentr r r
findr r r resubany
SKIP_TEXTSr r )r2 r" r# s table_htmls img_lineschildtnrR r% r
r& r' r r r( resultrE s @r) extract_contentr[ a s E
!
!8W"5
6
)jAK {J7Ik**+ 6* ::
ZZ
9 )C ~~
' c3' Aq3wXX'xx( S 5Aq3wXX%4KCXX_ 7762.D <: >> uRy 0LL%
5[ ~~cU~;
*ll3dl3s"Bz"B B C59$4 S)
*e6*p [[
F&6;;{333&6;;y111 <<>r+ c g }t }| j dd D ] }|d }t j d|t j s.t || }|j
d xs& |j d d j d d
}||vsy|j | |j ||d |S ) Nr T)r
r
z&\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$r /rH ?r )r url)
setr/ rQ searchIr r splitaddr )soupr" attsseenr r
full_urlr s r) extract_attachmentsri s
D 5D
]]3T]
* =y
99>bdd K z40H::D:)NTZZ_R-@-F-Fs-KA-NDt#" T( ;<= Kr+ c t j | t d }d|_ t |j d }d}|j d }|r|j d
}d}|j d }|r7t j d
|j }|r|j d }|j dd } t! || }
| r
t# | | } nd} t% | j' dk rd| d| d} |j) d } g }
| D ]R }|j dd }|st+ | | }|j dd }|
j- |rd| dnd| d T |
r| ddj/ |
z z
} |
r-|
D ]( }d|d d |d d }|| vs| d |d d |d d z
} * ||| |
fS # t $ r1}t d| d| t j d d d g fcY d }~S d }~ww xY w) N headerstimeoututf-8z fetch_detail error : file html.parserr zdiv.newsshowt h3Tr zdiv.newsshowt pu" 时间[：:]\s*(\d{4}-\d{2}-\d{2}) rI newsshowcclass_r r r r r r r rL [r z](r_ )u
附件：[)requestsr HEADERSencoding Exceptionprintsysstderrr r&
select_oner rQ ra grouprP ri r[ r0 r
r/ r r r )r_ rere titleh3date_strp_timemr2 attachmentscontentr7 rW r r r( r r
inline_markers r) fetch_detailr sH $ LLgr :
/D
E
+ ,B $ ' H
__ .
/F
II;V__=N O wwqzH ))E+)6K%dC0K!+s3 7==?b Rwh7}}U# NC''%$C"3,ggeR( TW<z9O!P_kltku vL ^M N N v I 66 6G EA & {"QuXJa 8M G+^AfI;b5
!DD E
(G[ 00g $
%cU"QC0szzBT4##$s #F; ; G5&G0*G50G5c t d| S )Nz&page=)LIST_URL)page_nums r) get_list_urlr s ZvhZ ((r+ c H t | } t j |t d }d|_ |j dk7 r-t
d| d |j t j g S t |j d
}g }|j d d
D ] }|j d }|r|j d s(|d j }|j dd j xs |j d } t! || }
|j dd
} | r| j d nd} | j# dd } | s|
s|j% | |
| f |S # t
$ r-}t
d| d| t j g cY d }~S d }~ww xY w)Nrk rl ro z Error fetching rp rq Page z: HTTP rs rI newslistrv r r
r r Tr r< newslisttime.-)r rz r r{ r| r} r~ r r status_coder r& r/ rP r
r r replacer )
r r_ r r re itemsnewsdivr r
r page_urltime_pr s
r) extract_list_pager s|
x
C LLgr :
}}
z 8szzJ /D E==z=: 6 LL f
y gr"((*Dajjtj.D3% c. 9286???.b##C- X LL%84 5!6$ L;
!#b,3::> s #E+ + F!4"F F! F!c t j t d } | j d | j d | S )N< )rn zPRAGMA journal_mode=WALzPRAGMA busy_timeout=5000)sqlite3connectDB_PATHexecute)conns r) init_dbr $ s1 ??7B /DLL*+LL+, Kr+ c
|r|n|}t j dd| j }|rt j |d nd} | j dt |||j |||t |t f
}|j dkD S # t $ r(} t d| t j Y d } ~ yd } ~ ww xY w)
Nr> r F) ensure_asciiz[]zINSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r z DB error: rq )rQ rR r
jsondumpsr SITE_NAMECATEGORYGROUProwcountr} r~ r r )
r r r publish_dater r summaryatt_jsoncurr s
r) save_articler + s geGffVS'*002G>Itzz+E:tH ll
8 (EKKM<
gx5
:
||a
QC szz2s AB
B> B99B>c
f t } t t j dkD r t t j d } t
dt d| t j t }d}d}|j t
dt t j t d| dz D ]K }t | }|s"t
d | d
t j n |t | z
}|D ] \ }}}|r*|d k r%t
d | d
|d d t j 3|j d|f } | j rWt | \ }
} } }
|
xs |}| xs |}| xs d| d| d}|
xs g }t! |||||| r.|dz
}|dk r$t
d| d
|d d t j t# j$ d t
d | dt | d| dt j N |j |j' t
d| d| t j y # t
$ rb t j d dv rIt t j dkD r- t t j d } n# t
$ r Y nw xY wY ow xY w) Nrt )z --max-pages z Scraping z
, max pages: rq r z Cleared old data for r z: 0 articles (end)z
2020-01-01z Skip old: r ( z)SELECT id FROM gov_raw WHERE page_url = ?r r r z + 2 g333333?rp z found, z totalz
Done! Found: z, New: ) MAX_PAGESr0 r argvint
ValueErrorr~ r r r commitranger r fetchoner r timesleepclose) max_pagesr total_new total_foundpnarticlesr r dater det_titledet_date det_contentdet_atts final_title
final_date
final_content
final_attss r) mainr ? s I
388}q CHHQK(I
Ii[
i[
9
K 9DIK KKM #I;
/cjjAAy1}% Y$R( GB4 12 D s8}$ %- !E8T|+ TF!E#2J<8szzJ ,, JXKXC||~9Eh9O 6Ixh#,uK!)TJ ' [\(2k]RZ+[M! RJD+x]T^_Q >D Ak#2.>-?@szzR JJsO+ . t2c(m_HYKvFSZZX? YB KKMJJL OK= {
;#**Mi xx{ ..3sxx=13D #CHHQK 0I! s5 I :J0 J J0 J)&J0(J))J0/J0__main__)"__doc__rz rQ r r r osr r bs4r urllib.parser r r r r BASE_URLr r r{ rT r* r5 r8 r[ ri r r r r r r __name__rA r+ r) r s
2 1 1 1
6 $ 6 DO0
#< GX 71x)"N(9Nx zF r+