/
/are recursed into. Standalone becomes a markdown image.
produces an empty part. pr" r$ r( Tr% []()imgsrcaltzr blockquotesectionrF rG )setchildren isinstancestrr& r3 nameidaddlowercontentsr r. r hasattrjoinr- extend_render_node_children)noder processedpartschildtxttagp_partselemahrefatxt full_hrefr@ rA full_srcjoinedrowsrE rF cellss r rW rW D sq E E J" eS !++-C S! zz e9 ! bi jj #:G , dC( **,C s+YY#% MM"T(+ HHVR 0 6 6 8E ==t=4D $+C$7 4&9+Q'?@ t,YY%' MM"T(+ ((5"-C "hhub 1#*3#4 #C58*A'>TRZQ[[\K]^YY$& T:. --4-8C s+5 ,6 XXg&F V$ G^DnnT* A@B TSWL@YZ"S5ZZKKuzz%'8 84 ?@ A TYYt_- E\))E2&Ciir*"3, r#b ! 44zQRAST @ @ LL.uc9E F D[ LL ..D.1C S!UJ"X L3 [s )Oc X t | d }d}|j ddt j dt j i }|r$|j d r|d j }|sZ|j dt j d t j }|r#|j d s|j d }d}|j }t j d| }|r|j d }d} |j dd } | r t | | } dj | } | xs |} g } | j d d D ] }|d j j }t j d| s;t ||d j }| j |j d xs |j! d d |d t# | j dk r$| r"d| d| d} | D ] }| d |d d |d d z } ||| | fS ) u: Parse detail — meta title + div.content-wrapper content.r r( metarO zTitle|ArticleTitle)attrscontentr r! r r" Tr% z(\d{4}-\d{2}-\d{2}) zcontent-wrapperz r# r$ z&\.(pdf|doc|docx|xls|xlsx|zip|rar|txt)$r* )r! r r z 附件列表： z - [r= r z) )r r, recompileIr r& r. searchgrouprW rU r- rR r r3 splitlen)r4 r r5 r! meta_titler7 pub_dater m content_html content_divrZ scope attachmentsr" r` full_urlatts r parse_detailr} s# } -D E6&"**=RTVTXTX2Y)Z[JjnnY/9%++- IIeBJJx,FIG Y^^C 0 &&T&2E H ==?D ($/A771: L))E*;)F sAfIOO$56H $)?)Z8>>RUCVWYCZck l m n < 2%+%cU"UG3PQ DC eCL>CJ >FH % 4 sH 4 ))(J?IF"X 67 4 O J' 4 OOUC 2 3 4s /A%%#B B c < t d dD } t g}t dt dz D ]% }|j t t z d| dz ' g }t | D ][ \ }} t | }t | }t d|dz dt | d t j |j | ] t d t | t j | r2t! | }t dt | t j t# } g } |D ]/ } | d | vs | j% | d | j | 1 t dt | t j g } | D ] \ } }} t | }t' || \ }}}}|xs | }|xs |}| j |||||rt) j* |d ndt, d t d|d d t j d}| D ] }|j/ dd d d }t1 j2 dd|d d j5 xs |d }|j/ d d }t7 |t8 rnt) j* |d } t; j< t> }|jA }|jC d |d |d |d! t, tD ||||d f |jF dkD r|dz }|jI |jK t d$| t j | D ]" }t t) j* |d $ y # t $ r2}t d|dz d | d | t j Y d }~>d }~ww xY w# t $ r,}t d| d| t j Y d }~Rd }~ww xY w# t $ r2}t d"|d d d# d| t j Y d }~d }~ww xY w)%Nc 3 @ K | ] }|t j v y wN)sysargv).0args rzmain. . s \S \s )z --incremental incrementalinc rk z/index_z.htmzPage z: z items)filez error (z): zTotal: z Incremental: zUnique: F) ensure_asciir( )r! page_url publish_daterj rz site_namez OK: 2 z ERR r rj iP z\s+rC r! rz zINSERT OR IGNORE INTO gov_raw (page_url, title, publish_date, site_name, group_name, summary, content, attachments, source_url) VALUES (?,?,?,?,?,?,?,?,?)r r z DB error for z Saved to DB: )&anyLIST_URLrange MAX_PAGESr3 r0 r1 enumerater r9 printrs r stderrrV Exceptionr rK rQ r} jsondumps SITE_NAMEr rm subr& rM rN sqlite3connectDB_PATHcursorexecuteGROUProwcountcommitclose)is_incremental page_urlsi all_itemsidxr r4 r6 eseen unique_itemsitemresultsr! list_date det_titledet_daterj rz final_title final_datesavedr content_textsummaryconncs r mainr s \4[\\N I 1i!m $ CI-'!D0AABC I i( GS G:D t$E E#a%3u:,f5CJJ G U # G GC N# $3::6&y1 c)n -.SZZ@ 5DL & 7$ HHT!W %& HS &' (szz:G!- 8 sI 8:D8DT38O 5Ix+#,uK!.YJ NN$ *"NYtzz+E J_a & F;s+,-CJJ ? 8$ E MuuY+FU3 &&l4C&89??AOQwZeeM2. k3 ' **[uEK M??7+D A II . : ' An,=E7L a m- . zzA~ KKM JJL)M. M% ! 3 1 djj/01} G E#a%S43:: F F GB 8 F3%r!%CJJ 7 7 82 M OAgJsO#4Bqc: L L MsK )AM*A>N(B O * N%3'N N%( O 1!OO P)'PP__main__r )#__doc__requestsrm r r osr r r bs4r urllib.parser r0 r1 r r r getenvr r r HEADERSr Sessionr headersupdater r9 rW r} r r __name__ r r r s , + + ( & i , . - "))K!2 3 DO0 ( w >Wt*6Z L1^ zF r