Ë
    ¡Š£ji#  ã                   ó¼   — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 dZ
g d¢ZdZdZd	d
iZdZdZdZh d£Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uU  çœ‰å±±å¸‚çŽ¯è¯„å…¬ç¤ºçˆ¬è™« - Visual Site Builder 9
URL: http://www.ms.gov.cn/zfxxgk/fdzdgknr/zdmsxx/sthj/hpgs.htm
åˆ—è¡¨: div.govnewslista245277 > table > tr[id^=line245277] > a[title] + td date
åˆ†é¡µ: hpgs.htm â†’ hpgs/3.htm â†’ hpgs/2.htm â†’ hpgs/1.htm (å…±4é¡µ, ~50æ�¡/é¡µ)
è¯¦æƒ…: meta ArticleTitle, meta PubDate, div#vsb_content_100
é    N)ÚBeautifulSoup)Úurljoinz1http://www.ms.gov.cn/zfxxgk/fdzdgknr/zdmsxx/sthj/)úhpgs.htmz
hpgs/3.htmz
hpgs/2.htmz
hpgs/1.htmu%   çœ‰å±±å¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€-çŽ¯è¯„å…¬ç¤ºu   å››å·�z
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36é   g      ø?z/root/search.db>)   ÚpÚddÚdlÚdtÚh1Úh2Úh3Úh4Úh5Úh6ÚhrÚliÚolÚtdÚthÚtrÚulÚdivÚnavÚpreÚformÚmainÚasideÚtableÚtbodyÚtfootÚtheadÚcenterÚdialogÚfigureÚfooterÚheaderÚhgroupÚaddressÚarticleÚcaptionÚdetailsÚsectionÚfieldsetÚ
blockquoteÚ
figcaptionc                 óÒ   ‡‡‡— | €yddl }ddlmŠ g Šˆˆˆfd„Š ‰| «       dj                  ‰«      }|j	                  dd|«      }|j	                  dd	|«      }|j                  «       S )
ul   å�—çº§è¾¹ç•Œå‡ºæ�¢è¡Œã€�è¡Œå†…èŠ‚ç‚¹ç›´æŽ¥æ‹¼æŽ¥ã€�<br> å‡ºæ�¢è¡Œï¼ˆâ‰ˆ æµ�è§ˆå™¨çœ‹åˆ°çš„æ�¢è¡Œç»“æž„ï¼‰ã€‚NÚ r   )ÚNavigableStringc                 óF  •— | j                   D ]‘  }t        |‰«      r‰j                  t        |«      «       Œ*t	        |dd «      dk(  r‰j                  d«       ŒLt	        |dd «      t
        v r+‰j                  d«        ‰|«       ‰j                  d«       ŒŠ ‰|«       Œ“ y )NÚnameÚbrú
)ÚchildrenÚ
isinstanceÚappendÚstrÚgetattrÚ_BLOCK_TAGS)ÚnodeÚchr2   ÚoutÚwalks     €€€ú"/root/gov_crawler/crawl_ms_hpgs.pyr@   zbody_text.<locals>.walk0   s„   ø€ Ø—-‘-ò 
	ˆBÜ˜"˜oÔ.Ø—
‘
œ3˜r›7Õ#Ü˜˜V TÓ*¨dÒ2Ø—
‘
˜4Õ Ü˜˜V TÓ*¬kÑ9Ø—
‘
˜4Ô Ù�R”Ø—
‘
˜4Õ á�R•ñ
	ó    z[ \t\r\f\v]*\n[ \t\r\f\v]*r6   z\n{3,}ú

)ÚreÚbs4r2   ÚjoinÚsubÚstrip)ÚelÚ_reÚtr2   r?   r@   s      @@@rA   Ú	body_textrL   (   sb   ú€ à	€zØÛÝ#Ø
€Cöñ 	ˆ„HØ
�‰�‹€AØ�‰Ð-¨t°QÓ7€AØ�‰�	˜6 1Ó%€AØ�7‰7‹9ÐrB   c                 ó   — t        | «      S )u'   ä¿�ç•™ HTML è¡¨æ ¼ç»“æž„ï¼ˆä¸�è½¬ mdï¼‰©r:   )r   ÚargsÚkwargss      rA   Útable_to_markdownrQ   C   s   € äˆu‹:ÐrB   c                 óô  ‡— | j                   D �]g  }t        |t        «      r$|j                  «       }|r|j	                  |«       Œ8|j
                  r|j
                  j                  «       nd}|dk(  r¢t        |«      }|r|j	                  |«       |j                  d«      D ]n  }|j                  dd«      Št        ˆfd„dD «       «      sŒ*|j	                  |j                  d¬«      xs t        j                  j                  ‰«      ‰d	œ«       Œp �Œ|d
k(  r"t        |«      }|s�Œ|j	                  |«       �Œ.|dv r"t        |«      }|s�ŒA|j	                  |«       �ŒT|dk(  s�Œ[t!        |||«       �Œj y)znRecurse into node children, extracting paragraphs/tables/attachments.
    Avoids double-extracting table text.r1   r   ÚaÚhrefc              3   óB   •K  — | ]  }|‰j                  «       v –— Œ y ­w)N)Úlower)Ú.0ÚextrT   s     €rA   ú	<genexpr>z_walk_node.<locals>.<genexpr>W   s   øè ø€ Òq¨s�s˜dŸj™j›lÔ*Ñqùs   ƒ)z.pdfz.docz.docxz.xlsz.xlsxz.zipz.rarT©rH   )r4   Úurlr   )r   r   r   N)r7   r8   r:   rH   r9   r4   rV   rL   Úfind_allÚgetÚanyÚget_textÚosÚpathÚbasenamerQ   Ú
_walk_node)	r=   ÚpartsÚattachmentsrI   ÚtextÚtagrS   ÚmdrT   s	           @rA   rc   rc   G   s9  ø€ ð �m‰mó /ˆÜ�bœ#ÔØ—8‘8“:ˆDÙØ—‘˜TÔ"ØØ!#§¢ˆb�g‰g�m‰mŒo¨bˆØ�#Š:Ü˜R“=ˆDÙØ—‘˜TÔ"Ø—[‘[ Ó%ò �Ø—u‘u˜V RÓ(�ÜÓqÐ6pÔqÕqØ×&Ñ&Ø !§
¡
° 
Ó 6Ò P¼"¿'¹'×:JÑ:JÈ4Ó:PØ#ñ(õ òð �GŠ^Ü" 2Ó&ˆBÛØ—‘˜RÖ Ø�LÑ Ü˜R“=ˆDÛØ—‘˜TÖ"Ø�EŒ\ä�r˜5 +Ö.ñ9/rB   c                 óH  — t        | d«      }|j                  d¬«      }|s|j                  d¬«      }|sdg fS g }g }t        |||«       |D �cg c]  }|j                  «       sŒ|‘Œ }}dj	                  |«      }|D ]  }t        t        dz   |d   «      |d<   Œ ||fS c c}w )	zEExtract paragraphs and tables from vsb_content_100 or vsb_content divÚlxmlÚvsb_content_100©ÚidÚvsb_contentr1   rC   r   r[   )r   Úfindrc   rH   rF   r   ÚBASE)ÚhtmlÚsoupÚcontent_divrd   re   r   Útext_contentÚatts           rA   Úextract_contentrv   i   sµ   € ä˜˜vÓ&€DØ—)‘)Ð0�)Ó1€KÙØ—i‘i =�iÓ1ˆÙØ�2ˆvˆØ€EØ€KÜˆ{˜E ;Ô/ØÖ+�1 §¡¥ŠQÐ+€EÐ+Ø—;‘;˜uÓ%€LØò <ˆÜœT JÑ.°°E±
Ó;ˆˆEŠ
ð<à˜Ð$Ð$ùò	 ,s   ÁBÁ$Bc                 ó  — t        | d«      }|j                  dd¬«      }|st        dt        j                  ¬«       g S g }|j                  dd„ ¬	«      D ]§  }|j                  d
«      }|sŒ|j                  dd«      xs |j                  d¬«      }|j                  dd«      }|sŒRt        ||«      }	|j                  d«      }
t        |
«      dkD  r|
d   j                  d¬«      nd}|j                  |||	f«       Œ© |S )z9Parse list page, return list of (title, date, detail_url)rj   r   Úgovnewslista245277)Úclass_z$  WARN: govnewslista245277 not found)Úfiler   c                 ó$   — | xr dt        | «      v S )NÚ
line245277rN   )Úxs    rA   ú<lambda>zparse_list.<locals>.<lambda>ƒ   s   € ¨aÒ.J°LÄCÈÃFÐ4J€ rB   rl   rS   Útitler1   TrZ   rT   r   é   )r   ro   ÚprintÚsysÚstderrr\   r]   r_   r   Úlenr9   )rq   Úbase_urlrr   r   Úitemsr   rS   r   rT   Úfull_urlÚtdsÚdate_strs               rA   Ú
parse_listrŠ   {   s÷   € ä˜˜vÓ&€DØ
�)‰)�EÐ"6ˆ)Ó
7€CÙÜÐ4¼C¿J¹JÕGØˆ	Ø€EØ�l‰l˜4Ñ$JˆlÓKò 2ˆØ�G‰G�C‹LˆÙØØ—‘�g˜rÓ"Ò< a§j¡j°t jÓ&<ˆØ�u‰u�V˜RÓ ˆÙØÜ˜8 TÓ*ˆØ�k‰k˜$ÓˆÜ25°c³(¸Q²,�3�q‘6—?‘?¨�?Ô.ÀBˆØ�‰�e˜X xÐ0Õ1ð2ð €LrB   c                 ó  — t        j                  | t        t        ¬«      }d|_        t        |j                  d«      }d}|j                  d«      D ]B  }|j                  dd«      xs d}d|v sŒ|j                  dd«      xs dj                  «       } n d}|j                  d«      D ]E  }|j                  dd«      xs d}d	|v sŒ|j                  dd«      xs dj                  «       d
d } n t        |j                  «      \  }}||||fS )z>Scrape detail page, return (title, date, content, attachments)©ÚheadersÚtimeoutúutf-8rj   r1   Úmetar4   ÚArticleTitleÚcontentÚPubDateNé
   )
Úrequestsr]   ÚHEADERSÚTIMEOUTÚencodingr   rf   r\   rH   rv   )	r[   Úresprr   r   r�   Únr‰   r’   re   s	            rA   Úscrape_detailr›   ’   s  € ä�<‰<˜¤W´gÔ>€DØ€D„MÜ˜Ÿ™ FÓ+€DØ€EØ—‘˜fÓ%ò ˆØ�H‰H�V˜RÓ Ò& BˆØ˜QÒØ—X‘X˜i¨Ó,Ò2°×9Ñ9Ó;ˆEÙð	ð
 €HØ—‘˜fÓ%ò ˆØ�H‰H�V˜RÓ Ò& BˆØ˜Š>ØŸ™ ¨BÓ/Ò5°2×<Ñ<Ó>¸sÀÐCˆHÙð	ô
 +¨4¯9©9Ó5Ñ€Gˆ[Ø�(˜G [Ð0Ð0rB   c                  ó²  — dd l } | j                  d¬«      }|j                  dt        dd¬«       |j                  dd	d
d¬«       |j	                  «       }t        |j                  t        t        «      «      }g }t        |«      D ]¸  }t        t        t        |   «      }t        d|dz   › d|› d|› �«       t        j                  |t        t         ¬«      }d|_        t%        |j&                  |«      }t        dt        |«      › d�«       |j)                  |«       ||dz
  k  sŒ t+        j,                  t.        «       Œº t        dt        |«      › d�«       t        |«      }	d}
d}t1        |«      D �]  \  }\  }}}t        |«      dkD  r|d d n|}t        d|dz   › d|	› d|› d�d¬«       t2        j4                  j7                  «        	 t9        |«      \  }}}}|s|}|rt        |j;                  «       «      dk  rt        d«       |dz  }Œœt        dt        |«      › d �d!¬«       |rt        d"t        |«      › d#�d!¬«       t        «        |j<                  �rdd l}|jA                  tB        «      }|jE                  «       }|jG                  d$|tH        f«       |jK                  «       rt        d%«       |jM                  «        �ŒW|r|d d& jO                  d'd«      nd!}|r*d'jQ                  |D �cg c]  }|d(   › d|d)   › �‘Œ c}«      nd!}|jG                  d*||||tH        |||xs d+tR        f	«       |jU                  «        |jM                  «        |
dz  }
t+        j,                  t.        «       �Œ t        d.«       t        d/|
› d0|› �«       y c c}w # tV        $ r3}t        d,|› �«       |dz  }t+        j,                  d-«       Y d }~�Œ_d }~ww xY w)1Nr   u   çœ‰å±±å¸‚çŽ¯è¯„å…¬ç¤ºçˆ¬è™«)Údescriptionz--pagesé   u   çˆ¬å�–é¡µæ•°)ÚtypeÚdefaultÚhelpz--import-dbÚ
store_trueTu   å…¥åº“)Úactionr    r¡   u
   åˆ—è¡¨é¡µ r€   ú/z: rŒ   r�   u	     æ‰¾åˆ° u    æ�¡u   
å…±é‡‡é›† u    æ�¡åˆ—è¡¨æ•°æ�®é(   z  [z] z...ú )Úendr”   u   è·³è¿‡ï¼ˆæ­£æ–‡è¿‡çŸ­ï¼‰u   âœ… u   å­—r1   z +u   é™„ä»¶z7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?u       å·²å­˜åœ¨ï¼Œè·³è¿‡éÈ   r6   r4   r[   zÜINSERT OR REPLACE INTO gov_raw
                             (page_url, title, content, publish_date, site_name, summary, attachments, date_rank, group_name)
                             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)z
0000-00-00u   â�Œ é   u   
=== å®Œæˆ� ===u   å…¥åº“: u
   , è·³è¿‡: ),ÚargparseÚArgumentParserÚadd_argumentÚintÚ
parse_argsÚminÚpagesr„   Ú	PAGE_URLSÚranger   rp   r�   r•   r]   r–   r—   r˜   rŠ   rf   ÚextendÚtimeÚsleepÚDELAYÚ	enumerater‚   ÚstdoutÚflushr›   rH   Ú	import_dbÚsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚ	SITE_NAMEÚfetchoneÚcloseÚreplacerF   ÚGROUPÚcommitÚ	Exception)rª   ÚparserrO   Úpages_to_crawlÚ	all_itemsÚiÚpage_urlr™   r†   ÚtotalÚinsertedÚskippedÚidxr   r‰   r[   ÚdspÚd_titleÚd_dater’   re   r»   ÚconnÚcÚsummaryrS   Ú
attach_strÚes                               rA   r   r   §   s�  € ÛØ×$Ñ$Ð1NÐ$ÓO€FØ
×Ñ˜	¬°Q¸^ÐÔLØ
×Ñ˜¨lÀDÈxÐÔXØ×ÑÓ€DÜ˜Ÿ™¤S¬£^Ó4€NØ€IÜ�>Ó"ò 	ˆÜœ4¤¨1¡Ó.ˆÜ�
˜1˜Q™3˜%˜q Ð 0°°8°*Ð=Ô>Ü�|‰|˜H¬g¼wÔGˆØˆŒÜ˜4Ÿ9™9 hÓ/ˆÜ�	œ#˜e›*˜ TÐ*Ô+Ø×Ñ˜ÔØˆ~ Ñ!Ó!Ü�J‰J”uÕð	ô 
ˆLœ˜Y›Ð(Ð(8Ð
9Ô:Ü�	‹N€EØ€HØ€GÜ'0°Ó';ó &Ñ#ˆÑ#ˆe�X˜sÜ ›J¨šOˆe�C�R‰j°ˆÜ��C˜‘E�7˜!˜E˜7 " S E¨Ð-°3Õ7Ü�
‰
×ÑÔð"	Ü4AÀ#Ó4FÑ1ˆG�V˜W kÙØ�Ùœc '§-¡-£/Ó2°RÒ7ÜÐ0Ô2Ø˜1‘�ØÜ�Dœ˜W›˜ cÐ*°Õ3ÙÜ˜œ3˜{Ó+Ð,¨FÐ3¸Õ<ÜŒGØ�~‹~ÛØ—‘¤wÓ/�Ø—K‘K“M�Ø—	‘	ÐSÐVYÔ[dÐUeÔfØ—:‘:”<ÜÐ2Ô4Ø—J‘J”LÙÙ>E˜' $ 3˜-×/Ñ/°°cÔ:È2�Ù]h˜TŸY™YÈKÖ'XÀq¨1¨V©9¨+°R¸¸%¹°zÒ(BÒ'XÔYÐnp�
Ø—	‘	ð Cð  ¨°&¼)ÀWÈjÐZ`ÒZpÐdpÔrwÐxôzð —‘”Ø—
‘
”Ø˜A‘�Ü�J‰J”uÖðE&ôN 
ÐÔÜ	ˆH�X�J˜j¨¨	Ð
2Õ3ùò (Yøô ò 	Ü�D˜˜�*ÔØ�q‰LˆGÜ�J‰J�q�MŠMûð	ús9   ÇANÈB9NË	*NË3N
ÌA.NÎNÎ	OÎ#(OÏOÚ__main__)Ú__doc__rD   r‚   r´   r`   r•   rE   r   Úurllib.parser   rp   r±   rÀ   rÄ   r–   r—   r¶   r½   r<   rL   rQ   rc   rv   rŠ   r›   r   Ú__name__© rB   rA   ú<module>rÝ      s�   ðñ÷ × Û Ý Ý  à:€ò€	ð 4€	Ø€ØÐWÐ
X€Ø
€Ø€à
€ò*€òò6ò/òD%ò$ò.1ò*>4ðB ˆzÒÙ…Fð rB   