Ë
    ^¨£j  ã                   ó�  — d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z	 ej                  dd«      ZdZedz   ZdZdZd	Zd
Z ee	j&                  «      dkD  r-e	j&                  d   j)                  d«      re	j&                  d   Z ee	j&                  «      dkD  re	j&                  d   ZddddœZd„ Zd„ Zd„ Zedk(  r e«        yy)u˜   
Crawler for è�£åŽ¿äººæ°‘æ”¿åºœ - å…¶ä»–æ³•å®šä¿¡æ�¯
https://www.rongzhou.gov.cn/rxrmzf/rxzhxzzfjqtfdx/pc/list.html
API: POST /queryList (æ•°èž�å¹³å�°)
é    NÚ	SEARCH_DBz/root/search.dbzhttps://www.rongzhou.gov.cnz
/queryListu%   è�£åŽ¿äººæ°‘æ”¿åºœ-å…¶ä»–æ³•å®šä¿¡æ�¯ÚrxzhxzzfjqtfdxÚrxrmzfé   é   Úrxé   zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z!application/x-www-form-urlencodedz>https://www.rongzhou.gov.cn/rxrmzf/rxzhxzzfjqtfdx/pc/list.html)z
User-AgentzContent-TypeÚRefererc                 ó¨  — | t         t        t        dddœ}t        j                  j                  t        t        j                  j                  |«      j                  «       t        ¬«      }	 t        j                  j                  |d¬«      }t        j                  |j                  «       «      S # t        $ r}t!        d|› �«       Y d}~yd}~ww xY w)	z,Fetch a single page of results from the API.ÚpubDateÚdesc)ÚcurrentÚpageSizezwebSiteCode[]zchannelCode[]ÚsortÚorder)ÚdataÚheadersé   ©Útimeoutz[error] API request failed: N)Ú	PAGE_SIZEÚWEBSITE_CODEÚCHANNEL_CODEÚurllibÚrequestÚRequestÚAPI_URLÚparseÚ	urlencodeÚencodeÚHEADERSÚurlopenÚjsonÚloadsÚreadÚ	ExceptionÚprint)r   r   ÚreqÚrespÚes        úcrawl_rongzhou.pyÚ
fetch_pager,   '   s©   € ð ÜÜ%Ü%ØØñ€Dô �.‰.×
 Ñ
 ¤¬v¯|©|×/EÑ/EÀdÓ/K×/RÑ/RÓ/TÔ^eÐ
 Ó
f€CðÜ�~‰~×%Ñ% c°2Ð%Ó6ˆÜ�z‰z˜$Ÿ)™)›+Ó&Ð&øÜò ÜÐ,¨Q¨CÐ0Ô1Üûðús   Á,AB0 Â0	CÂ9CÃCc           	      óê   — |r|sy|j                  «       }|sy|r|j                  «       dk(  rd}n|j                  «       }|r|j                  «       dd nd}| j                  d|||||t        f«       y)z*Save or update an article in the database.FÚ Né   zƒINSERT OR REPLACE INTO gov_raw (title, content, publish_date, page_url, source_url, site_name)
           VALUES (?, ?, ?, ?, ?, ?)T)ÚstripÚexecuteÚ	SITE_NAME)ÚconnÚtitleÚcontentÚpub_dateÚurlÚ
source_urls         r+   Úsave_articler9   :   sx   € á™Øà�K‰K‹M€EÙØá�g—m‘m“o¨Ò+Ø‰à—-‘-“/ˆá(0ˆx�~‰~Ó  Ñ$°b€HØ‡L�Lð	(à	�˜ 3¨
´IÐ>ôð
 ó    c            
      óŠ  — t        j                   «       } d}d}t        j                  t        d¬«      }|j	                  d«       t        d«       t        d«      }|s0t        d«       |j                  «        t        j                  d«       |j                  di «      j                  d	d«      }|dk(  r0t        d
«       |j                  «        t        j                  d«       |t        z   dz
  t        z  }t        d|› d|› d�«       t        d|dz   «      D �]Ð  }|dk(  r|}n2t        d|› d|› d�«       t        |«      }t        j                  d«       |st        d|› �«       ŒO|j                  di «      j                  dg «      }	|	D �]8  }
|
j                  di «      }|j                  dd«      }|j                  dd«      }|j                  di «      }|j                  di «      }t        |t        «      r|j                  dd«      }nHt        |t         «      r)	 t#        j$                  |«      }|j                  dd«      }n|rt!        |«      nd}|r|j'                  d«      s	t(        |z   }t        |t        «      r|j                  dd«      }nt        |t         «      r|}nd}|dz  }t+        ||||||«      s�Œ4|dz  }�Œ; |j-                  «        t        d|› d|› d�«       �ŒÓ |j                  «        t        j                   «       | z
  }t        d|› d|› d|d ›d!�«       t        d"t.        › �«       y #  |}Y ŒîxY w)#Nr   é<   r   zPRAGMA busy_timeout=60000z[rongzhou] Fetching page 1...r   z%[rongzhou] Failed to fetch first pager   Útotalz[rongzhou] No results foundz[rongzhou] Total: z articles, z pagesz[rongzhou] Fetching page ú/z...g333333Ó?z [rongzhou] Failed to fetch page ÚresultsÚsourcer4   r.   r   Úurlsr5   ÚpcÚhttpz[rongzhou] Progress: z savedz[rongzhou] Done: z
 saved in z.1fÚsz[rongzhou] Site: )ÚtimeÚsqlite3ÚconnectÚDB_PATHr1   r'   r,   ÚcloseÚsysÚexitÚgetr   ÚrangeÚsleepÚ
isinstanceÚdictÚstrr#   r$   Ú
startswithÚBASE_URLr9   Úcommitr2   )Ú
start_timeÚtotal_savedÚtotal_articlesr3   Úrawr=   Útotal_pagesÚpageÚraw_pager?   ÚrÚsrcr4   r6   Úraw_urlÚcontent_objÚarticle_urlÚraw_url_parsedr5   Úelapseds                       r+   Úmainrc   Q   s  € Ü—‘“€JØ€KØ€NÜ�?‰?œ7¨BÔ/€DØ‡L�LÐ,Ô-ô 
Ð
)Ô*Ü
�Q‹-€CÙÜÐ5Ô6Ø�
‰
ŒÜ�‰�Œà�G‰G�F˜BÓ×#Ñ# G¨QÓ/€EØ�‚zÜÐ+Ô,Ø�
‰
ŒÜ�‰�Œàœ9Ñ$ qÑ(¬YÑ6€KÜ	Ð˜u˜g [°°¸VÐ
DÔEä�a˜ q™Ó)ó 0CˆØ�1Š9Ø‰HäÐ-¨d¨V°1°[°MÀÐEÔFÜ! $Ó'ˆHÜ�J‰J�sŒOáÜÐ4°T°FÐ;Ô<Øà—,‘,˜v rÓ*×.Ñ.¨y¸"Ó=ˆØó  	!ˆAØ—%‘%˜ "Ó%ˆCØ—G‘G˜G RÓ(ˆEØ—w‘w˜y¨"Ó-ˆHØ—g‘g˜f bÓ)ˆGØŸ'™' )¨RÓ0ˆKô ˜'¤4Ô(Ø%Ÿk™k¨$°Ó3‘Ü˜G¤SÔ)ð*Ü%)§Z¡Z°Ó%8�NØ"0×"4Ñ"4°T¸2Ó">‘Kñ /6œc 'œl¸2�á ;×#9Ñ#9¸&Ô#AÜ&¨Ñ4�ô ˜+¤tÔ,Ø%Ÿ/™/¨)°RÓ8‘Ü˜K¬Ô-Ø%‘à�à˜aÑˆNÜ˜D %¨°(¸KÈÖUØ˜qÑ ’ðA 	!ðD 	�‰ŒÜÐ% k ]°!°E°7¸&ÐAÖBða0Cðd 	‡J�J„LÜ�i‰i‹k˜JÑ&€GÜ	Ð˜k˜]¨!¨NÐ+;¸:ÀgÈcÀ]ÐRSÐ
TÔUÜ	Ðœi˜[Ð
)Õ*øð5*Ø")’Kús   È'L<Ì<MÚ__main__)Ú__doc__r#   Úurllib.requestr   Úurllib.parserF   rE   ÚreÚosrJ   ÚgetenvrH   rS   r   r2   r   r   r   ÚlenÚargvrR   r!   r,   r9   rc   Ú__name__© r:   r+   ú<module>ro      sã   ðñó Û Û Û Û Û 	Û 	Û 
à
ˆ"�)‰)�KÐ!2Ó
3€Ø(€Ø
�\Ñ
!€à3€	Ø€Ø€Ø€	ñ €s‡x�xƒ=�1Ò˜Ÿ™ !™×/Ñ/°Ô5Ø—8‘8˜A‘;€LÙ€s‡x�xƒ=�1ÒØ—‘˜‘€Ið DØ7ØOñ€òò&ò.M+ð` ˆzÒÙ…Fð r:   