Ë
    ¸¬ujÞ  ã                   óÐ  — d Z ddlZddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlmZmZ d„ ZdZdZ ej                   dd	«      Zd
ZdZdZ ej*                  «        ed¬«      z
  j-                  d«      ZddddœZd„ Zd„ Zd„ Zdd„Zedk(  rXdej<                  v ZdZ ej<                  D ].  Z!e!jE                  d«      sŒ e#e!jI                  d«      d   «      Z Œ0  eee ¬«       yy)zM
crawl_shz_gov.py -- Shihezi city government info crawler (EpointWebBuilder)
é    N)ÚBeautifulSoup)ÚdatetimeÚ	timedeltac                 ó¼   — | syt        | d«      } |ddg«      D ]  }|j                  «        Œ |j                  dd¬«      }t        j                  dd|«      }|d d	 S )
NÚ úhtml.parserÚscriptÚstyleú T)Ú	separatorÚstripz\s+iô  )r   Ú	decomposeÚget_textÚreÚsub)Ú	html_textÚsoupÚtagÚtexts       ú"/root/gov_crawler/crawl_shz_gov.pyÚextract_plain_textr   
   sf   € ÙØÜ˜ MÓ2€DÙ�X˜wÐ'Ó(ò ˆØ�‰�ðà�=‰= 3¨dˆ=Ó3€DÜ�6‰6�&˜#˜tÓ$€DØ��ˆ:Ðó    zFhttps://www.shz.gov.cn/EpointWebBuilder/rest/govenopen/govopeninfolistÚ030Ú	SEARCH_DBz/root/search.dbu%   å…«å¸ˆçŸ³æ²³å­�å¸‚-æ”¿åºœä¿¡æ�¯å…¬å¼€é   é   iG  )Údaysz%Y-%m-%d)ÚnewÚskipÚerrorsc                 óv  — dt         t        | «      t        t        «      dddœ}t        j                  j                  |«      j                  «       }t        j                  j                  t        |ddddœ¬«      }	 t        j                  j                  |d¬	«      }t        j                  |j                  «       j                  d
«      «      }|j                  di «      j                  dg «      |j                  di «      j                  dd«      fS # t         $ r}t#        d|› �«       g dfcY d }~S d }~ww xY w)Nr   )ÚcatenumÚdeptnumÚ	pageindexÚpagesizeÚ
identifierÚtitlez<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36z!application/x-www-form-urlencodedz2https://www.shz.gov.cn/government_dept.html?id=030)ú
User-AgentzContent-TypeÚReferer)ÚdataÚheadersé   ©Útimeoutúutf-8ÚcustomÚlistÚtotalr   z  [ERROR] API error: )ÚDEPT_NUMÚstrÚ	PAGE_SIZEÚurllibÚparseÚ	urlencodeÚencodeÚrequestÚRequestÚAPI_URLÚurlopenÚjsonÚloadsÚreadÚdecodeÚgetÚ	ExceptionÚprint)Úpager*   ÚbodyÚreqÚrespÚjÚes          r   Ú
fetch_listrK      s  € àÜÜ˜“YÜœ	“NØØñ€Dô �<‰<×!Ñ! $Ó'×.Ñ.Ó0€DÜ
�.‰.×
 Ñ
 ¤¨tàXØ?ØKñ
ð !ó €CðÜ�~‰~×%Ñ% c°2Ð%Ó6ˆÜ�J‰J�t—y‘y“{×)Ñ)¨'Ó2Ó3ˆØ�u‰u�X˜rÓ"×&Ñ& v¨rÓ2°A·E±E¸(ÀBÓ4G×4KÑ4KÈGÐUVÓ4WÐWÐWøÜò ÜÐ% a SÐ)Ô*Ø�1ˆu�ûðús   Á<BD Ä	D8ÄD3Ä-D8Ä3D8c                 ód  — |dk(  ry|dk(  ryd| z   }	 t         j                  j                  |ddi¬«      }t         j                  j                  |d¬	«      }|j	                  «       j                  d
«      }t        |d«      }|j                  dd¬«      }|rt        |«      S y# t        $ r Y yw xY w)NÚLinkr   ÚAttachúhttps://www.shz.gov.cn/govxxgkr(   zMozilla/5.0)r+   r,   r-   r/   r   Údivzarticle-cont)Úclass_)
r6   r:   r;   r=   r@   rA   r   Úfindr4   rC   )ÚinfourlÚinfotypeÚurlrG   rH   Úhtmlr   Úcontent_divs           r   Úfetch_detailrX   8   sµ   € Ø�6ÒØØ�8ÒØà
*¨WÑ
4€Cð
Ü�n‰n×$Ñ$ S°<ÀÐ2OÐ$ÓPˆÜ�~‰~×%Ñ% c°2Ð%Ó6ˆØ�y‰y‹{×!Ñ! 'Ó*ˆÜ˜T =Ó1ˆØ—i‘i ¨n�iÓ=ˆÙÜ�{Ó#Ð#ØøÜò Ùðús   “BB# Â#	B/Â.B/c                 ó   — dd l }	 |j                  t        «      }|j                  «       }t	        |«      }d}|j                  |t        | |||||df«       |j                  dkD  rt        dxx   dz  cc<   nt        dxx   dz  cc<   |j                  «        |j                  «        y # t        $ r}	t        dxx   dz  cc<   Y d }	~	y d }	~	ww xY w)Nr   z�INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, source_url, summary, category) VALUES (?, ?, ?, ?, ?, ?, ?, ?)Úshz_govr   é   r   r    )Úsqlite3ÚconnectÚDB_PATHÚcursorr   ÚexecuteÚ	SITE_NAMEÚrowcountÚstatsÚcommitÚcloserC   )
r'   rU   ÚcontentÚdate_strr\   ÚconnÚcÚsummaryÚsqlrJ   s
             r   Ú
store_itemrl   L   s±   € ÛðØ�‰œwÓ'ˆØ�K‰K‹MˆÜ$ WÓ-ˆð aˆØ	�	‰	�#Ü˜e S¨(°G¸SÀ'È9ÐUô	Wà�:‰:˜Š>Ü�%‹L˜AÑŒLä�&‹M˜QÑ‹MØ�‰ŒØ�
‰
�øÜò Üˆh‹˜1Ñ�ˆûðús   †B"B) Â)	CÂ2CÃCc                 óÖ  — t        d«       t        d«       t        d«       t        }t        d|› �«       d}t        d|dz   «      D �]v  }t        d|› d|› �«       t        |«      \  }}|st        d«        �nGt        d	|› d
t	        |«      › �«       t        |«      D �]  \  }}|j                  dd«      }	|j                  dd«      d d }
|j                  dd«      }|j                  dd«      }|	r|sŒW|
r/|
t        k  r&t        d|
› d|	d d › �«       t        dxx   dz  cc<   Œˆ|dk(  rd|z   }n|j                  d«      r|nd|z   }t        d|dz   › d|	d d › d|
› d|› �«       |r||k\  rt        d|› d�«         y d}|dk(  rt        ||«      }t        |	|||
«       |dz  }t        j                  d«       �Œ �Œy t        d t        d!   › d"t        d   › d#t        d$   › �«       y )%Nz<============================================================z$Shihezi city government info crawlerzTotal pages: r   r[   z

--- Page ú/z  [ERROR] No items returnedz  API total: z, Items this page: r'   r   Úinfodateé
   rS   rT   ÚNewsz  [SKIP] Too old: r   r,   r   rO   Úhttpzhttps://www.shz.gov.cnz  [z] é(   z | z
  Test mode: z items doneg      à?z
Done! New: r   z, Skip: z
, Errors: r    )rD   Ú	MAX_PAGESÚrangerK   ÚlenÚ	enumeraterB   ÚTHREE_YEARS_AGOrc   Ú
startswithrX   rl   ÚtimeÚsleep)ÚfullÚtest_nÚ
page_countÚseenrE   Úitemsr2   ÚidxÚitemr'   rg   rS   rT   rU   rf   s                  r   Úcrawlrƒ   _   s  € Ü	ˆ(„OÜ	Ð
0Ô1Ü	ˆ(„Oä€JÜ	ˆM˜*˜Ð
&Ô'à€DÜ�a˜ a™Ó(ó )ˆÜ�˜D˜6  : ,Ð/Ô0Ü! $Ó'‰ˆˆuÙÜÐ/Ô0Úä�˜e˜WÐ$7¼¸E»
°|ÐDÔEä" 5Ó)ó  	‰IˆC�Ø—H‘H˜W bÓ)ˆEØ—x‘x 
¨BÓ/°°Ð4ˆHØ—h‘h˜y¨"Ó-ˆGØ—x‘x 
¨FÓ3ˆHá¡Øá˜H¤Ò6ÜÐ*¨8¨*°A°e¸C¸R°j°\ÐBÔCÜ�f“ Ñ"“Øð ˜6Ò!Ø6¸Ñ@‘à!(×!3Ñ!3°FÔ!;‘gÐAYÐ\cÑAc�ä�C˜˜A™�w˜b  s¨  ¨C°¨z¸¸X¸JÐGÔHá˜$ &š.Ü˜¨ x¨{Ð;Ô<ÚàˆGØ˜6Ò!Ü& w°Ó9�ä�u˜c 7¨HÔ5Ø�A‰IˆDÜ�J‰J�sŽOòA 	ð)ôV 
ˆMœ% ™,˜ x´°f±¨¸jÌÈxÉÐHYÐ
ZÕ[r   Ú__main__z--fullz--test=ú=r[   )r|   r}   )FN)%Ú__doc__Úsysr   rz   Úosr>   Úurllib.requestr6   Úurllib.parseÚbs4r   r   r   r   r<   r3   Úgetenvr^   ra   r5   rt   ÚnowÚstrftimerx   rc   rK   rX   rl   rƒ   Ú__name__Úargvr|   r}   Úargry   ÚintÚsplit© r   r   ú<module>r•      sú   ðñ÷ =× <× <Ó <Ý ß (òð S€Ø€Ø
ˆ"�)‰)�KÐ!2Ó
3€Ø3€	Ø€	Ø€	à�8—<‘<“>¡I°5Ô$9Ñ9×CÑCÀJÓO€Ø˜1¨Ñ*€òò2ò(ó&4\ðn ˆzÒØ�s—x‘xÐ€DØ€FØ�x‰xò ,ˆØ�>‰>˜)Õ$Ù˜Ÿ™ 3›¨Ñ*Ó+‰Fð,ñ 
ˆt˜FÖ#ð r   