Ë
    ¸¬ujw  ã                   ó˜   — d Z ddlZddlZddlZddlZddlmZ  ej                  d«       dZdZ	dZ
dd	iZd
„ Zd„ Zd„ Zedk(  r e«        yy)u2   Crawl ddh.hld.gov.cn - ä¸œæˆ´æ²³æ–°åŒºé€šçŸ¥å…¬å‘Šé    N)ÚBeautifulSoupÚignorezhttps://ddh.hld.gov.cnz/root/search.dbu   ä¸œæˆ´æ²³æ–°åŒº-é€šçŸ¥å…¬å‘Šz
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36c                 ó–  — | j                  d«      }|sy|j                  d«      }|s|}t        j                  ddt        |«      «      }t        j                  d|«      }t        j                  dd|«      }t        |d«      }|j                  dd¬	«      }|s|sy|}|r*|r|dz   dj                  |«      z   ndj                  |«      }|S )
z§Extract content from TRS_Editor div.
    Strip inline <span>/<o:p>/<font> tags so dates aren't split.
    Tables are extracted separately as HTML to avoid duplication.zdiv.content_conÚ zdiv.TRS_Editorz;</?(?:span|o:p|font|b|strong|em|i|u|s|strike|sub|sup)[^>]*>z<table[\s\S]*?</table>úhtml.parserú

T©Ústrip)Ú
select_oneÚreÚsubÚstrÚfindallr   Úget_textÚjoin)	ÚsoupÚcontÚeditorÚhtmlÚtablesÚhtml_without_tablesÚ
clean_soupÚtextÚresults	            ú/root/gov_crawler/crawl_ddh.pyÚextract_clean_textr      sÎ   € ð �?‰?Ð,Ó-€DÙØà�_‰_Ð-Ó.€FÙØˆä�6‰6ÐPÐRTÔVYÐZ`ÓVaÓb€Dô �Z‰ZÐ1°4Ó8€FÜŸ&™&Ð!:¸BÀÓEÐäÐ2°MÓB€JØ×Ñ˜v¨TÐÓ2€Dá™Øà€FÙÙ:@�˜&‘ 6§;¡;¨vÓ#6Ò6ÀfÇkÁkÐRXÓFYˆà€Mó    c                 ó^  — | j                  d«      }|syg }|j                  dd¬«      D ]q  }|d   }|j                  d¬«      }|sŒ|sŒ |j                  d«      r)|j                  d	«      rt        d
z   |dd z   nt        |z   }|j                  d|› d|› d�«       Œs dj                  |«      S )zExtract attachment linkszdiv.appendixr   ÚaT©Úhrefr!   r	   ú.ú./ú/xwzx/tzgg/é   Nú[z](ú)ú
)r   Úfind_allr   Ú
startswithÚBASEÚappendr   )r   ÚappendixÚlinksr   r!   r   s         r   Úextract_attachmentsr/   -   s¹   € à�‰˜~Ó.€HÙØà€EØ×Ñ˜s¨ÐÓ.ò .ˆØ�‰yˆØ�z‰z ˆzÓ%ˆÚ’DØ�‰˜sÔ#Ø:>¿/¹/È$Ô:O”t˜mÑ+¨d°1°2¨hÒ6ÔUYÐ\`ÑU`�Ø�L‰L˜1˜T˜F " T F¨!Ð,Õ-ð.ð �9‰9�UÓÐr   c                  óv  — t        j                  t        «      } | j                  d«       | j                  d«       d}d}t	        d|dz   «      D �]  }|dk(  r
t
        › d�}nt
        › d|› d�}	 t        j                  |t        d	d
¬«      }d|_	        t        |j                  d«      }g }|j                  d«      D ]Ÿ  }	|	j                  dd¬«      }
|	j                  dd¬«      }|
sŒ,|sŒ/|
j                  dd«      j!                  d«      sŒQ|
j#                  d¬«      }|
d   }|j#                  d¬«      }|sŒ}t%        |«      dkD  sŒŒ|j'                  |||f«       Œ¡ t        d|› dt%        |«      › d�«       |s�Œ.|D �]Ò  \  }}}|j!                  d«      rt
        › d|dd  › �}n-|j!                  d «      r|nt
        › d!|j)                  d!«      › �}|j+                  d"«      d   }| j                  d#|f«      j-                  «       }|rt        d$|d d	 › �«       Œž	 t        j                  |t        d	d
¬«      }d|_	        t        |j                  d«      }|j/                  d'«      }|r|j#                  d¬«      n|}|}|j/                  d(«      }|rGt1        j2                  d)|j#                  «       «      }|r!|j5                  d«      j7                  d!d*«      }t9        |«      }t;        |«      }|r|r|d+z   |z   n|}|sd,}|d d- }	 | j                  d.t<        |||||f«       | j?                  «        |dz  }t        d/|› d0|d d1 d2›d3|› �«       �ŒÕ �Œ | jE                  «        t        d6|› �«       y # t        $ r}t        d|› d|› �«       Y d }~�ŒHd }~ww xY w# t        $ r}t        d%|d d	 › d&|› �«       Y d }~�ŒCd }~ww xY w# t         j@                  $ r% t        d4|d d	 › �«       | jC                  «        Y �Œ‚t        $ r/}| jC                  «        t        d5|d d	 › d&|› �«       Y d }~�Œ´d }~ww xY w)7NzPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000r   é   é   r$   z/xwzx/tzgg/index_z.htmlé   F)ÚheadersÚtimeoutÚverifyzutf-8zPage z ERROR: r   Úlir   Tr    ÚspanÚdate)Úclass_r!   r   z./20r	   é   z: z itemsr#   r%   Úhttpú/ú?z(SELECT 1 FROM gov_raw WHERE page_url = ?z  SKIP: z  FETCH ERR: z - zdiv.content h1z	span.timez(\d{4}[-/]\d{1,2}[-/]\d{1,2})ú-r   u   (æ— å†…å®¹)iô  zjINSERT INTO gov_raw (site_name, page_url, title, content, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)z  [z] é#   z.35sz | z  DUP: z
  DB ERR: u   
âœ… Done. Total new: )#Úsqlite3ÚconnectÚDB_PATHÚexecuteÚranger+   ÚrequestsÚgetÚHEADERSÚencodingÚ	ExceptionÚprintr   r   r)   Úfindr*   r   Úlenr,   ÚlstripÚsplitÚfetchoner   r   ÚsearchÚgroupÚreplacer   r/   Ú	SITE_NAMEÚcommitÚIntegrityErrorÚrollbackÚclose)ÚdbÚ	total_newÚtotal_pagesÚpageÚlist_urlÚrespÚer   Úitemsr7   r   Ú	date_spanÚtitler!   r9   Ú	list_dateÚpage_urlÚexistsÚdrespÚdsoupÚh1Ú
page_titleÚ	time_spanÚmÚcontentÚattachmentsÚsummarys                              r   Úcrawlro   ?   sK  € Ü	�‰œÓ	!€BØ‡J�JÐ(Ô)Ø‡J�JÐ*Ô+à€IØ€Kä�a˜ q™Ó)ó [<ˆØ�1Š9Ü˜˜{Ð+‰Hä˜Ð0°°°eÐ<ˆHð	Ü—<‘< ´'À2ÈeÔTˆDØ#ˆDŒMô
 ˜TŸY™Y¨Ó6ˆð ˆØ—-‘- Ó%ò 	6ˆBØ—‘˜ $�Ó'ˆAØŸ™ ¨v˜Ó6ˆIÚ’Y 1§5¡5¨°Ó#4×#?Ñ#?ÀÕ#GØŸ
™
¨˜
Ó.�Ø˜‘y�Ø ×)Ñ)°Ð)Ó5�ÚœS ›Z¨!›^Ø—L‘L $¨¨tÐ!4Õ5ð	6ô 	��d�V˜2œc %›j˜\¨Ð0Ô1ÙÙà&+ó <	<Ñ"ˆD�%˜Ø�‰˜tÔ$Ü"˜V ;¨t°A°B¨x¨jÐ9‘à#'§?¡?°6Ô#:™4Ä4À&ÈÈ$Ï+É+ÐVYÓJZÐI[Ð@\�Ø—~‘~ cÓ*¨1Ñ-ˆHð —Z‘ZÐ JÈXÈKÓX×aÑaÓcˆFÙÜ˜  s¨  Ð-Ô.ØðÜ Ÿ™ X´wÈÐSXÔY�Ø!(�”Ü% e§j¡j°-Ó@�ð ×!Ñ!Ð"2Ó3ˆBÙ46˜Ÿ™¨4˜Ô0¸EˆJð ˆDØ×(Ñ(¨Ó5ˆIÙÜ—I‘IÐ>À	×@RÑ@RÓ@TÓU�ÙØŸ7™7 1›:×-Ñ-¨c°3Ó7�Dô )¨Ó/ˆGô .¨eÓ4ˆKÙÙ<C˜' FÑ*¨[Ò8È�áØ'�à˜d˜s�mˆGð<Ø—
‘
ð AÜ ¨*°g¸tÀWÐMôð —	‘	”Ø˜Q‘�	Ü˜˜D˜6  J¨s° O°DÐ#9¸¸T¸FÐCÖDòm<	<ð?[<ðz ‡H�H„JÜ	Ð# I ;Ð
/Õ0øôk ò 	Ü�E˜$˜˜x¨ sÐ+Ô,Ýûð	ûôP ò Ü˜ e¨C¨R j \°°Q°CÐ8Ô9ÝûðûôJ ×)Ñ)ò Ü˜ 
¨3¨B Ð0Ð1Ô2Ø—‘—Üò <Ø—‘”Ü˜
 :¨c¨r ?Ð"3°3°q°cÐ:×;Ò;ûð<úsO   Á/$M7È:NÌ	A	O
Í7	NÎ NÎNÎ	OÎ(OÏOÏ
4P8ÐP8Ð	$P3Ð3P8Ú__main__)Ú__doc__rF   rA   r   ÚwarningsÚbs4r   Úfilterwarningsr+   rC   rT   rH   r   r/   ro   Ú__name__© r   r   ú<module>rw      sf   ðÙ 8Û Û Û 	Û Ý Ø €× Ñ ˜Ô !à€Ø
€Ø*€	ØÐWÐ
X€òò>ò$f1ðP ˆzÒÙ	…Gð r   