۵j3 d Z ddlZddlZddlZddlZddlZddlZddlZddlmZ ddl m
Z
m Z ddl m
Z
dZdZdZd Zd
Zd Zd Zd
dddZg dZd Zd Zd Zd Zd Zd Z d Z d Z d Z d Z d Z!e"d k( r e! yy) uZ
Crawler for 苏州工业园区高端制造与国际贸易区 - 通知公告
CMS: UCAPCONTENT
List: ul > li > a[href] + span (date)
Pagination: createPageHTML, only 1 page, 12 items
Detail: div.c_d_content > UCAPCONTENT
Usage:
python3 crawl_sipac_tzgg.py # Full crawl
python3 crawl_sipac_tzgg.py --max-pages 1 # Incremental
N)datetime)
BeautifulSoupTag)urljoinz/root/search.dbu= 苏州工业园区高端制造与国际贸易区-通知公告u 通知公告u 江苏zhttp://www.sipac.gov.cnz8http://www.sipac.gov.cn/szgyyqgdzzygjmyq/tzgg/list.shtml zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentAcceptzAccept-Language) u 【打印本页】u 【关闭窗口】u 打印本页u 关闭窗口u 转载分享：u 浏览量：u 相关附件：u 相关稿件：u 扫一扫在手机打开u 附件：u 累计次数：c 4 g }t | j D ] }t |t r%|j }|s'|j | 9|j dk( rp|j dd j }|j d }|r'|r%t || }|j d| d| d |s|j | |j d k( rR|j d
d }|s|j d d } t || }
|j | rd|
d nd|
d |j d
v r|j d :|j dv r=|j d r[|j dd } | sr|j | |j dk( r|j dk( r|j d r|j dd } | s|j | dj | j S )z>Extract plain text from a tag (excluding embedded tables).ahref Tstrip
imgsrcalt ">查看图片)br
)spanfontbstrongemutable div) listchildren
isinstancestrr appendnamegetget_textr findjoin) p_tag
detail_urlpartselemtr textfullr r full_srctxts %/root/gov_crawler/crawl_sipac_tzgg.pyrender_paragraphr5 , s EU^^$ %" dC
A Q
YY#
88FB'--/D==t=,D z40 |D6D6 BC T"
YY%
((5"%Chhub)":s3 PS|H:5K L[ghpgq rH ZI J
YY'
! LL
YYD
Dyy!--4-0C S! yyG#yyE!dii&8--4-0C S!K%"L 88E? "" c g }| j d D ]L }t | }t |j d dk\ r|j | |j N |S )zWExtract elements and return as raw HTML strings, then remove them from content.r Tr
)find_allr$ lenr( r% decompose) content_divtablesr
table_htmls r4 extract_tables_as_htmlr? X s_
F %%g. Z
u~~D~) *b 0 MM* %
Mr6 c g }| j d D ]b }|j dd }|st || }|j dd }|j |rd| dnd| d |j d |S )z;Extract ![]()
tags from content_div and return as markdown.r r r r r r )r9 r' r r% r; )r< r, imgsr r r2 r s r4 extract_imagesrB c s
D##E* ggeR z3/H''%$C KK3,xj0FGVbckbl mC UD
E MMO
Kr6 c
g }| j ddg D ] }|j t | }t | | }t | j
D ] }|j |j j }|dk( rDt ||
s?t
fdt D rX|r
|d vsb|j
t|dv r|j ddgd D ] }|j dk( rDt ||
s!t
fd
t D r:|r
|d vsD|j
V|j dk( sf|j d rx|j d d
st
fdt D r|r
|d vs|j
S|j d d
sjt
fdt D r|r
|d vs|j
dj | } |r| ddj | z z
} |r| ddj | z z
} | j S )Nscriptstylepc 3 & K | ] }|v
y wN .0skr3 s r4 z"extract_content.. >RrSy > )r sectioncenter ucapcontentr F) recursivec 3 & K | ] }|v
y wrH rI rJ s r4 rM z"extract_content.. &FRrSy&FrO r Tr
c 3 & K | ] }|v
y wrH rI rJ s r4 rM z"extract_content.. rV rO c 3 & K | ] }|v
y wrH rI rJ s r4 rM z"extract_content.. rN rO
)r9 r; r? rB r! r" r& lowerr5 any
SKIP_TEXTSr% find_parentr( r* r ) r< r, r- s table_htmls img_lineschildtnsubresultr3 s @r4 extract_contentre p s E
!
!8W"5
6
)5K {J7I k**+ & ::
ZZ
9"5*5C3 >: >> 59 4LL%
>
> ~~sFmu~E
.88s?*3
;C3&F:&F#F $59(< !LL -XX's+ ,,s$,7C3&F:&F#F $59(< !LL -
. ..D.1C3 >: >> 59 4LL%A &F [[
F&6;;{333&6;;y111 <<>r6 c g }t }| j dd D ] }|d }t j d|t j s.t || }|j
d xs& |j d d j d d
}||vsy|j | |j ||d |S ) Nr
T)r r z&\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$r
/rP ?r )r& url)
setr9 researchIr r( splitaddr% )soupr, attsseenr
r full_urlr& s r4 extract_attachmentsrt s
D 5D
]]3T]
* =y
99>bdd K z40H::D:)NTZZ_R-@-F-Fs-KA-NDt#" T( ;<= Kr6 c t j | t d }d|_ t |j d }d}|j d d
d i }|r$|j d
r|d
j }|sL|j d }|r9|j d }t j dd| j }|}d}|j d d
di }|r'|j d
r|d
j d d }|j dd } | r| j d }
|
r|
} t || } | r
t! | | } nd} t# | j dk rd| d| d} |j% d }
g }|
D ]R }|j d d }|st' | | }|j d d }|j) |rd| d nd| d T |r| d d j+ | z z
} | r-| D ]( }d |d
d!|d" d#}|| vs| d$|d
d!|d" d#z
} * ||| | fS # t $ r1}t d| d| t j d d d g fcY d }~S d }~ww xY w)%N headerstimeoututf-8z fetch_detail error : file html.parserr metar& ArticleTitle)attrscontenttitleTr
u- \s*[-_—|].*(?:通知公告|高端制造).*$PubDate
r c_d_content)class_ UCAPCONTENTr r r r r r r rY [z](ri )u
附件：[)requestsr' HEADERSencoding Exceptionprintsysstderrr r0 r) r r( rk rc rt re r: r9 r r% r* )ri rerp r mttagr/ date_strr< ucap attachmentsr rA r` r r r2 r r
inline_markers r4 fetch_detailr s $ LLgr :
/D
E
6&.!9 :B bffY 9
##% ii 4 (AGQOUUWAE H
6&)!4 5B bffY i=&&("- ))E-)8K
. K%dC0K!+s3 7==?b Rwh7}}U# NC''%$C"3,ggeR( TW<z9O!P_kltku vL ^M N N v I 66 6G EA & {"QuXJa 8M G+^AfI;b5
!DD E
(G[ 00w $
%cU"QC0szzBT4##$s #H* * I$3&I I$ I$c $ | dk( rt S d| dS )Nr z3http://www.sipac.gov.cn/szgyyqgdzzygjmyq/tzgg/list_z.shtml)LIST_URL)page_nums r4 get_list_urlr s 1}
@
& QQr6 c F t | } t j |t d }d|_ |j dk7 r-t
d| d |j t j g S t |j d
}|j d }|sg S g }|j d D ] }|j d
}|j d } |r|j d s9|d j }
|j dd j xs |j! d } t# ||
} | r| j! d nd}
| s| s|j% | | |
f |S # t
$ r-}t
d| d| t j g cY d }~S d }~ww xY w)Nrv rw rz z Error fetching r{ r| Page z: HTTP r~ zdiv.listBox ullir
r r r r Tr
)r r r' r r r r r r status_coder r0
select_oner9 r) r r( r r% )r ri r r rp ulitemsr r
r r r page_urlr s r4 extract_list_pager su
x
C LLgr :
}}
z 8szzJ /D
) *B
Ekk$ 6 GGCLwwvf
y gr"((*Dajjtj.D3%044==t=," X LL%84 5 6 L9
!#b,3::> s #E* * F 3"FF F c t j t d } | j d | j d | S )N< )ry zPRAGMA journal_mode=WALzPRAGMA busy_timeout=5000)sqlite3connectDB_PATHexecute)conns r4 init_dbr , s1 ??7B /DLL*+LL+, Kr6 c
|r|n|}t j dd| j }|rt j |d nd} | j dt |||j |||t |t f
}|j dkD S # t $ r(} t d| t j Y d } ~ yd } ~ ww xY w)
Nz\s+r F) ensure_asciiz[]zINSERT OR IGNORE INTO gov_raw
(site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r z DB error: r| )rk rc r jsondumpsr SITE_NAMECATEGORYGROUProwcountr r r r )
r r r publish_dater r summaryatt_jsoncurr s
r4 save_articler 3 s geGffVS'*002G>Itzz+E:tH ll
8 (EKKM<
gx5
:
||a
QC szz2s AB
B> B99B>c
f t } t t j dkD r t t j d } t
dt d| t j t }d}d}|j t
dt t j t d| dz D ]K }t | }|s"t
d | d
t j n |t | z
}|D ] \ }}}|r*|d k r%t
d | d
|d d t j 3|j d|f } | j rWt | \ }
} } }
|
xs |}| xs |}| xs d| d| d}|
xs g }t! |||||| r.|dz
}|dk r$t
d| d
|d d t j t# j$ d t
d | dt | d| dt j N |j |j' t
d| d| t j y # t
$ rb t j d dv rIt t j dkD r- t t j d } n# t
$ r Y nw xY wY ow xY w) Nr )z --max-pages z Scraping z
, max pages: r| r z Cleared old data for r z: 0 articles (end)z
2020-01-01z Skip old: r ( z)SELECT id FROM gov_raw WHERE page_url = ?r r r z + 2 g333333?r{ z found, z totalz
Done! Found: z, New: ) MAX_PAGESr: r argvint
ValueErrorr r r r commitranger r fetchoner r timesleepclose) max_pagesr total_new total_foundpnarticlesr r dater det_titledet_date det_contentdet_atts final_title
final_date
final_content
final_attss r4 mainr G s I
388}q CHHQK(I
Ii[
i[
9
K 9DIK KKM #I;
/cjjAAy1}% Y$R( GB4 12 D s8}$ %- !E8T|+ TF!E#2J<8szzJ ,, JXKXC||~9Eh9O 6Ixh#,uK!)TJ ' [\(2k]RZ+[M! RJD+x]T^_Q >D Ak#2.>-?@szzR JJsO+ . t2c(m_HYKvFSZZX? YB KKMJJL OK= {
;#**Mi xx{ ..3sxx=13D #CHHQK 0I! s5 I :J0 J J0 J)&J0(J))J0/J0__main__)#__doc__r rk r r r osr r bs4r r urllib.parser r r r r BASE_URLr r r r\ r5 r? rB re rt r r r r r r __name__rI r6 r4 r s
2 1 1 1 "
K $ E
DO0
)#X
7x ?1HR !L(9Nx zF r6