Ë
    †É…ji  ã                   óÌ  — d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
mZ ddlZddlZ ej                  «       Zej#                  dedd¬«       ej'                  «       \  ZZej,                  Z ej0                  dd	«      Zd
Zedz   ZdZ ej:                  «        e	d¬«      z
  j=                  d«      ZdddœZ d„ Z!dd„Z"d„ Z#d„ Z$d„ Z%d„ Z&e'dk(  r e&«        yy)uG   å¹¿å¾·å¸‚äººæ°‘æ”¿åºœ - å»ºè®¾é¡¹ç›®çŽ¯è¯„æ–‡ä»¶å®¡æ‰¹ (guangde.gov.cn)é    N)ÚdatetimeÚ	timedelta)Úurljoinz--pagesu   é™�åˆ¶é¡µæ•°)ÚtypeÚdefaultÚhelpÚ	SEARCH_DBz/root/search.dbzhttps://www.guangde.gov.cnz*/Jczwgk/showList/0/111001001/page_{n}.htmlu'   å¹¿å¾·å¸‚å»ºè®¾é¡¹ç›®çŽ¯è¯„æ–‡ä»¶å®¡æ‰¹iG  )Údaysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8)z
User-AgentÚAcceptc                 ól   — t        dt        j                  «       j                  d«      › d| › �d¬«       y )Nú[z%H:%M:%Sz] T)Úflush)Úprintr   ÚnowÚstrftime)Úmsgs    ú"/root/gov_crawler/crawl_guangde.pyÚlogr      s+   € Ü	ˆAŒh�l‰l‹n×%Ñ% jÓ1Ð2°"°S°EÐ
:À$ÖGó    c                 óZ  — t        |«      D ]^  }	 t        j                  | t        d¬«      }d|_        |j
                  dk(  r|j                  c S t        d|j
                  › d| › �«       Œ` y # t        $ r3}t        d|dz   › d	|› �«       t        j                  d
«       Y d }~Œ™d }~ww xY w)Né   )ÚheadersÚtimeoutzutf-8éÈ   z  HTTP z for z
  Attempt é   z	 failed: é   )ÚrangeÚrequestsÚgetÚHEADERSÚencodingÚstatus_codeÚtextr   Ú	ExceptionÚtimeÚsleep)ÚurlÚretriesÚattemptÚrÚes        r   Úsafe_getr,      sž   € Ü˜“>ò 	ˆð	Ü—‘˜S¬'¸2Ô>ˆAØ ˆAŒJØ�}‰} Ò#Ø—v‘v’Ü�'˜!Ÿ-™-˜¨¨c¨UÐ3Õ4ð	ð øô ò 	Ü�*˜W Q™Y˜K y°°Ð4Ô5Ü�J‰J�q�M‰Mûð	ús   �=A.ÁA.Á.	B*Á7)B%Â%B*c                 óÆ  — g }t        j                  d| «      }|D ]Ã  }d|vrŒ|d|j                  d«       }t        j                  d|«      }|sŒ5|j	                  d«      }t        j                  d|«      }|sŒ_|j	                  d«      |j	                  d«      }	}|j                  d«      st        t        |«      }|j                  |	j                  «       ||f«       ŒÅ |S )	z2Parse list page, return list of (title, url, date)z	<li[^>]*>z</li>Nz%<span[^>]*>(\d{4}-\d{2}-\d{2})</span>r   z'<a\s+href="([^"]+)"[^>]*title="([^"]*)"r   Úhttp)
ÚreÚsplitÚindexÚsearchÚgroupÚ
startswithr   ÚBASE_URLÚappendÚstrip)
ÚhtmlÚitemsÚpartsÚpartÚliÚdate_mÚdate_strÚa_mÚhrefÚtitles
             r   Ú
parse_listrB   )   sÓ   € à€EÜ�H‰H�\ 4Ó(€EØò 6ˆØ˜$ÑØØÐ&�4—:‘:˜gÓ&Ð'ˆÜ—‘ÐCÀRÓHˆÙØØ—<‘< “?ˆÜ�i‰iÐBÀBÓGˆÙØØ—i‘i “l C§I¡I¨a£LˆeˆØ�‰˜vÔ&Üœ8 TÓ*ˆDØ�‰�e—k‘k“m T¨8Ð4Õ5ð6ð €Lr   c                 ó`  — t        | «      }|syt        j                  d|«      }|r|j                  d«      dd nd}t        j                  d|t        j                  «      }|s%t        j                  d|t        j                  «      }|r|j                  d«      j                  «       nd}||fS )	z2Fetch detail page, return (content_html, date_str))NNzA<meta[^>]*PubDate[^>]*content="(\d{4}-\d{2}-\d{2}\s*\d{2}:\d{2})"r   Né
   z5<div[^>]*id="zoom"[^>]*>(.*?)</div>\s*</div>\s*</div>z#<div[^>]*id="zoom"[^>]*>(.*?)</div>Ú )r,   r/   r2   r3   ÚDOTALLr7   )r'   r8   r=   Úpub_dateÚ	content_mÚcontent_htmls         r   Úfetch_detailrJ   >   s–   € ä�C‹=€DÙØÜ�Y‰YÐ[Ð]aÓb€FÙ'-ˆv�|‰|˜A‹˜s Ñ#°4€Hä—	‘	ÐRÐTXÔZ\×ZcÑZcÓd€IÙÜ—I‘IÐDÀdÌBÏIÉIÓVˆ	Ù1:�9—?‘? 1Ó%×+Ñ+Ô-À€LØ˜Ð!Ð!r   c                 ój   — t        j                  d| «      }|rt        |j                  d«      «      S dS )Nzpagecount="(\d+)"r   )r/   r2   Úintr3   )r8   Úms     r   Úparse_total_pagesrN   L   s,   € Ü
�	‰	Ð&¨Ó-€AÙŒ3ˆq�w‰w�q‹z‹?Ð& QÐ&r   c                  óö  — t        dt        › �«       t        dt        › �«       t        j                  t
        d¬«      } t        t        j                  d¬«      «      }|st        d«       y t        |«      }t        dkD  rt        |t        «      }t        d	|› �«       d}d}d}d}t        d|dz   «      D �](  }t        j                  |¬«      }t        d
|› d|› d�«       t        |«      }|s|dz  }Œ?t        |«      }	|	st        d|› �«       Œ[t        dt        |	«      › d�«       d}
|	D ]v  \  }}}|t        k  r|dz  }Œd}
t        |«      \  }}|r|n|}|st        d|d d › �«       |dz  }ŒE	 | j!                  d||||t        ||f«       | j"                  dkD  r|dz  }Œx |
r|dkD  rt        d«        n(| j'                  «        t)        j*                  d«       �Œ+ | j'                  «        | j-                  «        t        d«       t        d|› �«       t        d|› �«       t        d|› �«       t        d|› �«       t        d«       y # t$        $ r}t        d|› �«       |dz  }Y d }~�ŒFd }~ww xY w)NzStarting crawl: zCutoff date: é<   )r   r   )ÚnzERROR: Cannot fetch page 1r   zTotal pages: zPage ú/z...z  No items on page z  z itemsTFz	  Empty: é(   zƒINSERT OR IGNORE INTO gov_raw (page_url, title, content, publish_date, site_name, source_url, summary) VALUES (?, ?, ?, ?, ?, ?, ?)z  DB error: z!  All items past cutoff, stoppingg      à?z
=== Summary ===zNew entries: zSkipped (old): zSkipped (empty): zErrors: zDone!)r   Ú	SITE_NAMEÚCUTOFFÚsqlite3ÚconnectÚDB_PATHr,   ÚLIST_URLÚformatrN   Ú_MAX_PAGES_ARGÚminr   rB   ÚlenrJ   ÚexecuteÚtotal_changesr$   Úcommitr%   r&   Úclose)Úconnr8   Útotal_pagesÚ	new_countÚskip_oldÚ
skip_emptyÚerrorsÚpager'   r9   Úall_oldrA   Úlinkr>   ÚcontentrG   Úuse_dater+   s                     r   Úmainrm   P   s‘  € ÜÐ
œ9˜+Ð&Ô'Üˆ-œ�xÐ Ô!ä�?‰?œ7¨BÔ/€Dä”H—O‘O a�OÓ(Ó)€DÙÜÐ(Ô)ØÜ# DÓ)€KÜ˜ÒÜ˜+¤~Ó6ˆÜˆm˜K˜=Ð)Ô*à€IØ€HØ€JØ€Fä�a˜ q™Ó)ó 1ˆÜ�o‰o ˆoÓ%ˆÜˆe�D�6˜˜;˜- sÐ+Ô,ä˜‹}ˆÙØ�a‰KˆFØä˜4Ó ˆÙÜÐ% d VÐ,Ô-Øäˆb”�U“�˜FÐ#Ô$àˆØ%*ò 	Ñ!ˆE�4˜Øœ&Ò Ø˜A‘�ØØˆGä ,¨TÓ 2ÑˆG�XÙ#+‘x°ˆHáÜ�i  c r 
˜|Ð,Ô-Ø˜a‘�
Øð	Ø—‘ð ZØ˜5 '¨8´YÀÀeÐLôð ×%Ñ%¨Ò)Ø ‘N�Iøð)	ñ2 �t˜a’xÜÐ3Ô5Ùð 	�‰Œä�
‰
�3Žðc1ðf 	‡K�K„MØ‡J�J„LäÐ
ÔÜˆ-˜	�{Ð#Ô$Üˆ/˜(˜Ð$Ô%ÜÐ
˜J˜<Ð(Ô)Üˆ(�6�(ÐÔÜˆ%…Møô+ ò Ü�l 1 #Ð&Ô'Ø˜!‘–ûðús   Å31IÉ	I8ÉI3É3I8Ú__main__)é   )(Ú__doc__ÚosÚsysr/   Újsonr%   Ú	tracebackrV   r   r   Úurllib.parser   r   ÚargparseÚ_APÚArgumentParserÚ
_AP_PARSERÚadd_argumentrL   Úparse_known_argsÚ_AP_ARGSÚ_Úpagesr[   ÚgetenvrX   r5   rY   rT   r   r   rU   r    r   r,   rB   rJ   rN   rm   Ú__name__© r   r   ú<module>r‚      sñ   ðá Mß 2× 2× 2Ó 2ß (Ý  Û ã ØˆS×ÑÓ!€
Ø 
× Ñ ˜	¨°Q¸^Ð Ô LØ×)Ñ)Ó+�€ˆ!Ø—‘€à
ˆ"�)‰)�KÐ!2Ó
3€Ø'€ØÐBÑB€Ø5€	à
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€ð DØOñ€ò
Hóòò*"ò'òOðb ˆzÒÙ…Fð r   