Ë
    ¸¬uj  ã                   ó@  — d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
mZ ddlZ ej                  dd«      ZdZedz   Zd	Z ej$                  «        e	d
¬«      z
  j'                  d«      ZdddœZd„ Zdd„Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uD   ç»©æºªåŽ¿äººæ°‘æ”¿åºœ - å»ºè®¾é¡¹ç›®çŽ¯è¯„æ–‡ä»¶å®¡æ‰¹ (cnjx.gov.cn)é    N)ÚdatetimeÚ	timedelta)ÚurljoinÚ	SEARCH_DBz/root/search.dbzhttps://www.cnjx.gov.cnz*/Jczwgk/showList/0/111001001/page_{n}.htmlu'   ç»©æºªåŽ¿å»ºè®¾é¡¹ç›®çŽ¯è¯„æ–‡ä»¶å®¡æ‰¹iG  )Údaysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8)z
User-AgentÚAcceptc                 ól   — t        dt        j                  «       j                  d«      › d| › �d¬«       y )Nú[z%H:%M:%Sz] T)Úflush)Úprintr   ÚnowÚstrftime)Úmsgs    ú/root/gov_crawler/crawl_cnjx.pyÚlogr      s+   € Ü	ˆAŒh�l‰l‹n×%Ñ% jÓ1Ð2°"°S°EÐ
:À$ÖGó    c                 óZ  — t        |«      D ]^  }	 t        j                  | t        d¬«      }d|_        |j
                  dk(  r|j                  c S t        d|j
                  › d| › �«       Œ` y # t        $ r3}t        d|dz   › d	|› �«       t        j                  d
«       Y d }~Œ™d }~ww xY w)Né   )ÚheadersÚtimeoutzutf-8éÈ   z  HTTP z for z
  Attempt é   z	 failed: é   )ÚrangeÚrequestsÚgetÚHEADERSÚencodingÚstatus_codeÚtextr   Ú	ExceptionÚtimeÚsleep)ÚurlÚretriesÚattemptÚrÚes        r   Úsafe_getr)      sž   € Ü˜“>ò 	ˆð	Ü—‘˜S¬'¸2Ô>ˆAØ ˆAŒJØ�}‰} Ò#Ø—v‘v’Ü�'˜!Ÿ-™-˜¨¨c¨UÐ3Õ4ð	ð øô ò 	Ü�*˜W Q™Y˜K y°°Ð4Ô5Ü�J‰J�q�M‰Mûð	ús   �=A.ÁA.Á.	B*Á7)B%Â%B*c                 óÆ  — g }t        j                  d| «      }|D ]Ã  }d|vrŒ|d|j                  d«       }t        j                  d|«      }|sŒ5|j	                  d«      }t        j                  d|«      }|sŒ_|j	                  d«      |j	                  d«      }	}|j                  d«      st        t        |«      }|j                  |	j                  «       ||f«       ŒÅ |S )	z2Parse list page, return list of (title, url, date)z	<li[^>]*>z</li>Nz%<span[^>]*>(\d{4}-\d{2}-\d{2})</span>r   z'<a\s+href="([^"]+)"[^>]*title="([^"]*)"r   Úhttp)
ÚreÚsplitÚindexÚsearchÚgroupÚ
startswithr   ÚBASE_URLÚappendÚstrip)
ÚhtmlÚitemsÚpartsÚpartÚliÚdate_mÚdate_strÚa_mÚhrefÚtitles
             r   Ú
parse_listr?   #   sÓ   € à€Eä�H‰H�\ 4Ó(€EØò 6ˆØ˜$ÑØØÐ&�4—:‘:˜gÓ&Ð'ˆÜ—‘ÐCÀRÓHˆÙØØ—<‘< “?ˆÜ�i‰iÐBÀBÓGˆÙØØ—i‘i “l C§I¡I¨a£LˆeˆØ�‰˜vÔ&Üœ8 TÓ*ˆDØ�‰�e—k‘k“m T¨8Ð4Õ5ð6ð €Lr   c                 ó`  — t        | «      }|syt        j                  d|«      }|r|j                  d«      dd nd}t        j                  d|t        j                  «      }|s%t        j                  d|t        j                  «      }|r|j                  d«      j                  «       nd}||fS )	z2Fetch detail page, return (content_html, date_str))NNzA<meta[^>]*PubDate[^>]*content="(\d{4}-\d{2}-\d{2}\s*\d{2}:\d{2})"r   Né
   z5<div[^>]*id="zoom"[^>]*>(.*?)</div>\s*</div>\s*</div>z#<div[^>]*id="zoom"[^>]*>(.*?)</div>Ú )r)   r,   r/   r0   ÚDOTALLr4   )r$   r5   r:   Úpub_dateÚ	content_mÚcontent_htmls         r   Úfetch_detailrG   9   s–   € ä�C‹=€DÙØä�Y‰YÐ[Ð]aÓb€FÙ'-ˆv�|‰|˜A‹˜s Ñ#°4€Hä—	‘	ÐRÐTXÔZ\×ZcÑZcÓd€IÙÜ—I‘IÐDÀdÌBÏIÉIÓVˆ	Ù1:�9—?‘? 1Ó%×+Ñ+Ô-À€LØ˜Ð!Ð!r   c                 ój   — t        j                  d| «      }|rt        |j                  d«      «      S dS )Nzpagecount="(\d+)"r   )r,   r/   Úintr0   )r5   Úms     r   Úparse_total_pagesrK   H   s,   € Ü
�	‰	Ð&¨Ó-€AÙŒ3ˆq�w‰w�q‹z‹?Ð& QÐ&r   c           
      ó¬   — | r|sy|sy	 |j                  d| ||||| |f«       |j                  dkD  S # t        $ r}t        d|› �«       Y d }~yd }~ww xY w)NFzƒINSERT OR IGNORE INTO gov_raw (page_url, title, content, publish_date, site_name, source_url, summary) VALUES (?, ?, ?, ?, ?, ?, ?)r   z  DB error: )ÚexecuteÚtotal_changesr!   r   )Úpage_urlr>   ÚcontentrD   Ú	site_nameÚconnr(   s          r   Úinsert_recordrS   L   sn   € Ù™5ØÙØðØ�‰ð RØ�u˜g x°¸HÀeÐLô	
ð ×!Ñ! AÑ%Ð%øÜò Üˆl˜1˜#ÐÔÜûðús   Š'2 ²	A»AÁAc            
      ó  — t        dt        › �«       t        dt        › �«       t        j                  t
        «      } t        t        j                  d¬«      «      }|st        d«       y t        |«      }t        d|› �«       d}d}d}d}t        d|dz   «      D �]   }t        j                  |¬«      }t        d|› d	|› d
�«       t        |«      }|s|dz  }Œ?t        |«      }	|	st        d|› �«       Œ[t        dt        |	«      › d�«       d}
|	D ]^  \  }}}|t        k  r|dz  }Œd}
t        |«      \  }}|r|n|}|st        d|d d › �«       |dz  }ŒEt        ||||t        | «      sŒZ|dz  }Œ` |
r|dkD  rt        d«        nt        j                   d«       �Œ | j#                  «        | j%                  «        t        d«       t        d|› �«       t        d|› �«       t        d|› �«       t        d|› �«       t        d«       y )NzStarting crawl: zCutoff date: r   )ÚnzERROR: Cannot fetch page 1zTotal pages: r   zPage ú/z...z  No items on page z  z itemsTFz	  Empty: é(   z!  All items past cutoff, stoppingg      à?z
=== Summary ===zNew entries: zSkipped (old): zSkipped (empty): zErrors: zDone!)r   Ú	SITE_NAMEÚCUTOFFÚsqlite3ÚconnectÚDB_PATHr)   ÚLIST_URLÚformatrK   r   r?   ÚlenrG   rS   r"   r#   ÚcommitÚclose)rR   r5   Útotal_pagesÚ	new_countÚskip_oldÚ
skip_emptyÚerrorsÚpager$   r6   Úall_oldr>   Úlinkr;   rP   rD   Úuse_dates                    r   Úmainrk   [   s   € ÜÐ
œ9˜+Ð&Ô'Üˆ-œ�xÐ Ô!ä�?‰?œ7Ó#€Dô ”H—O‘O a�OÓ(Ó)€DÙÜÐ(Ô)ØÜ# DÓ)€KÜˆ-˜�}Ð%Ô&à€IØ€HØ€JØ€Fä�a˜ q™Ó)ó &ˆÜ�o‰o ˆoÓ%ˆÜˆe�D�6˜˜;˜- sÐ+Ô,ä˜‹}ˆÙØ�a‰KˆFØä˜4Ó ˆÙÜÐ% d VÐ,Ô-Øäˆb”�U“�˜FÐ#Ô$àˆØ%*ò 	Ñ!ˆE�4˜Øœ&Ò Ø˜A‘�ØØˆGä ,¨TÓ 2ÑˆG�XÙ#+‘x°ˆHáÜ�i  c r 
˜|Ð,Ô-Ø˜a‘�
Øä˜T 5¨'°8¼YÈÕMØ˜Q‘‘	ð	ñ" �t˜a’xÜÐ3Ô5Ùä�
‰
�3ŽðM&ðP 	‡K�K„MØ‡J�J„LäÐ
ÔÜˆ-˜	�{Ð#Ô$Üˆ/˜(˜Ð$Ô%ÜÐ
˜J˜<Ð(Ô)Üˆ(�6�(ÐÔÜˆ%…Mr   Ú__main__)é   )Ú__doc__ÚosÚsysr,   Újsonr"   Ú	tracebackrZ   r   r   Úurllib.parser   r   Úgetenvr\   r2   r]   rX   r   r   rY   r   r   r)   r?   rG   rK   rS   rk   Ú__name__© r   r   ú<module>rw      s±   ðá Jß 2× 2× 2Ó 2ß (Ý  Û à
ˆ"�)‰)�KÐ!2Ó
3€Ø$€ØÐBÑB€Ø5€	à
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€ð DØOñ€ò
Hóòò,"ò'òòCðJ ˆzÒÙ…Fð r   