۵j2 d Z ddlZddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
m Z ddl m
Z
dZdZdZd Zd
Zd Zd Zd
dddZg dZd Zd Zd Zd Zd Zd Z d Z d Z d Z d Z d Z!e"d k( r e! yy) uR
Crawler for 苏州独墅湖科教创新区 - 公告
CMS: UCAPCONTENT
List: ul > li > a[href] + span (date)
Pagination: createPageHTML, 4 pages, 15/page, ~57 items
Detail: div.c_d_content > UCAPCONTENT
Usage:
python3 crawl_sipac_dshkjycxq.py # Full crawl
python3 crawl_sipac_dshkjycxq.py --max-pages 1 # Incremental
N)datetime)
BeautifulSoupTag)urljoinz/root/search.dbu% 苏州独墅湖科教创新区-公告u 公告u 江苏zhttp://www.sipac.gov.cnz9http://www.sipac.gov.cn/szdshkjcxq/gonggao/nav_list.shtml zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentAcceptzAccept-Language) u 【打印本页】u 【关闭窗口】u 打印本页u 关闭窗口u 转载分享：u 浏览量：u 相关附件：u 相关稿件：u 扫一扫在手机打开u 附件：u 累计次数：c : g }t | j D ] }t |t r%|j }|s'|j | 9|j dk( rp|j dd j }|j d }|r'|r%t || }|j d| d| d |s|j | |j d k( rU|j d
d }|s|j d d } t || }
|j | r d | d|
dnd
|
d |j dv r|j d =|j dv r=|j d r^|j dd } | su|j | |j dk( r|j dk( r|j d r|j dd } | s|j | dj | j S )z>Extract plain text from a tag (excluding embedded tables).ahref Tstrip[]()imgsrcalt![![]()br
)spanfontbstrongemutable div) listchildren
isinstancestrr appendnamegetget_textr findjoin) p_tag
detail_urlpartselemtr textfullr r full_srctxts */root/gov_crawler/crawl_sipac_dshkjycxq.pyrender_paragraphr6 , s EU^^$ %" dC
A Q
YY#
88FB'--/D==t=,D z40 qba 01 T"
YY%
((5"%Chhub)":s3 r#b
! 44zQRAST
YY'
! LL
YYD
Dyy!--4-0C S! yyG#yyE!dii&8--4-0C S!K%"L 88E? "" c g }| j d D ]L }t | }t |j d dk\ r|j | |j N |S )zWExtract
elements and return as raw HTML strings, then remove them from content.r Tr
)find_allr% lenr) r& decompose) content_divtablesr
table_htmls r5 extract_tables_as_htmlr@ X s_
F %%g. Z
u~~D~) *b 0 MM* %
Mr7 c g }| j d D ]e }|j dd }|st || }|j dd }|j |r d| d| dnd| d |j g |S ) z;Extract ![]()
tags from content_div and return as markdown.r r r r r r r r )r: r( r r& r< )r= r- imgsr r r3 r s r5 extract_imagesrC c s
D##E* ggeR z3/H''%$C KK3"SEH:Q/d8*Az"extract_content.. >RrSy > )r! sectioncenter ucapcontentr F) recursivec 3 & K | ] }|v
y wrI rJ rK s r5 rN z"extract_content.. &FRrSy&FrP r Tr
c 3 & K | ] }|v
y wrI rJ rK s r5 rN z"extract_content.. rW rP c 3 & K | ] }|v
y wrI rJ rK s r5 rN z"extract_content.. rO rP
)r: r< r@ rC r" r# r' lowerr6 any
SKIP_TEXTSr& find_parentr) r+ r ) r= r- r. s table_htmls img_lineschildtnsubresultr4 s @r5 extract_contentrf p s E
!
!8W"5
6
)5K {J7I k**+ & ::
ZZ
9"5*5C3 >: >> 59 4LL%
>
> ~~sFmu~E
.88s?*3
;C3&F:&F#F $59(< !LL -XX's+ ,,s$,7C3&F:&F#F $59(< !LL -
. ..D.1C3 >: >> 59 4LL%A &F [[
F&6;;{333&6;;y111 <<>r7 c g }t }| j dd D ] }|d }t j d|t j s.t || }|j
d xs& |j d d j d d
}||vsy|j | |j ||d |S ) Nr
T)r r z&\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$r
/rQ ?r )r' url)
setr: researchIr r) splitaddr& )soupr- attsseenr
r full_urlr' s r5 extract_attachmentsru s
D 5D
]]3T]
* =y
99>bdd K z40H::D:)NTZZ_R-@-F-Fs-KA-NDt#" T( ;<= Kr7 c t j | t d }d|_ t |j d }d}|j d d
d i }|r$|j d
r|d
j }|sL|j d }|r9|j d }t j dd| j }|}d}|j d d
di }|r'|j d
r|d
j d d }|j dd } | r| j d }
|
r|
} t || } | r
t! | | } nd} t# | j dk rd| d| d} |j% d }
g }|
D ]U }|j d d }|st' | | }|j d d }|j) |r d | d | d nd!| d W |r| d"d"j+ | z z
} | r-| D ]( }d#|d
d |d$ d }|| vs| d%|d
d |d$ d z
} * ||| | fS # t $ r1}t d| d| t j d d d g fcY d }~S d }~ww xY w)&N headerstimeoututf-8z fetch_detail error : file html.parserr metar' ArticleTitle)attrscontenttitleTr
u- \s*[-_—|].*(?:通知公告|高端制造).*$PubDate
r! c_d_content)class_ UCAPCONTENTz z
r r r r r r r rZ r rj u
附件：[)requestsr( HEADERSencoding Exceptionprintsysstderrr r1 r* r r) rl rd ru rf r; r: r r& r+ )rj rerq r mttagr0 date_strr= ucap attachmentsr rB ra r r r3 r r
inline_markers r5 fetch_detailr s $ LLgr :
/D
E
6&.!9 :B bffY 9
##% ii 4 (AGQOUUWAE H
6&)!4 5B bffY i=&&("- ))E-)8K
. K%dC0K!+s3 7==?b Rwh7}}U# YC''%$C"3,ggeR( C2cU"XJa!8tH:UVEWX Y v I 66 6G EA & {"QuXJa 8M G+^AfI;b5
!DD E
(G[ 00w $
%cU"QC0szzBT4##$s #H- - I'6&I" I'"I'c $ | dk( rt S d| dS )N z4http://www.sipac.gov.cn/szdshkjcxq/gonggao/nav_list_z.shtml)LIST_URL)page_nums r5 get_list_urlr s 1}
A(6 RRr7 c F t | } t j |t d }d|_ |j dk7 r-t
d| d |j t j g S t |j d
}|j d }|sg S g }|j d D ] }|j d
}|j d } |r|j d s9|d j }
|j dd j xs |j! d } t# ||
} | r| j! d nd}
| s| s|j% | | |
f |S # t
$ r-}t
d| d| t j g cY d }~S d }~ww xY w)Nrw rx r{ z Error fetching r| r} Page z: HTTP r zdiv.listBox ullir
r r r r Tr
)r r r( r r r r r r status_coder r1
select_oner: r* r r) r r& )r rj r r rq ulitemsr r
r r r page_urlr s r5 extract_list_pager su
x
C LLgr :
}}
z 8szzJ /D
) *B
Ekk$ 6 GGCLwwvf
y gr"((*Dajjtj.D3%044==t=," X LL%84 5 6 L9
!#b,3::> s #E* * F 3"FF F c t j t d } | j d | j d | S )N< )rz zPRAGMA journal_mode=WALzPRAGMA busy_timeout=5000)sqlite3connectDB_PATHexecute)conns r5 init_dbr , s1 ??7B /DLL*+LL+, Kr7 c
|r|n|}t j dd| j }|rt j |d nd} | j dt |||j |||t |t f
}|j dkD S # t $ r(} t d| t j Y d } ~ yd } ~ ww xY w)
Nz\s+r F) ensure_asciiz[]zINSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r z DB error: r} )rl rd r jsondumpsr SITE_NAMECATEGORYGROUProwcountr r r r )
r r r publish_dater r summaryatt_jsoncurr s
r5 save_articler 3 s geGffVS'*002G>Itzz+E:tH ll
8 (EKKM<
gx5
:
||a
QC szz2s AB
B> B99B>c
f t } t t j dkD r t t j d } t
dt d| t j t }d}d}|j t
dt t j t d| dz D ]K }t | }|s"t
d | d
t j n |t | z
}|D ] \ }}}|r*|d k r%t
d | d
|d d t j 3|j d|f } | j rWt | \ }
} } }
|
xs |}| xs |}| xs d| d| d}|
xs g }t! |||||| r.|dz
}|dk r$t
d| d
|d d t j t# j$ d t
d | dt | d| dt j N |j |j' t
d| d| t j y # t
$ rb t j d dv rIt t j dkD r- t t j d } n# t
$ r Y nw xY wY ow xY w) Nr )z --max-pages z Scraping z
, max pages: r} r z Cleared old data for r z: 0 articles (end)z
2020-01-01z Skip old: r ( z)SELECT id FROM gov_raw WHERE page_url = ?r r r z + 2 g333333?r| z found, z totalz
Done! Found: z, New: ) MAX_PAGESr; r argvint
ValueErrorr r r r commitranger r fetchoner r timesleepclose) max_pagesr total_new total_foundpnarticlesr r dater det_titledet_date det_contentdet_atts final_title
final_date
final_content
final_attss r5 mainr G s I
388}q CHHQK(I
Ii[
i[
9
K 9DIK KKM #I;
/cjjAAy1}% Y$R( GB4 12 D s8}$ %- !E8T|+ TF!E#2J<8szzJ ,, JXKXC||~9Eh9O 6Ixh#,uK!)TJ ' IQ{m2hZq+IM! RJD+x]T^_Q >D Ak#2.>-?@szzR JJsO+ . t2c(m_HYKvFSZZX? YB KKMJJL OK= {
;#**Mi xx{ ..3sxx=13D #CHHQK 0I! s5 I :J0 J J0 J)&J0(J))J0/J0__main__)#__doc__r rl r r r osr r bs4r r urllib.parser r r r r BASE_URLr r r r] r6 r@ rC rf ru r r r r r r __name__rJ r7 r5 r s
2 1 1 1 "
3 $ F
DO0
)#X
7x ?1HS !L(9Nx zF r7