Ë
    ¸¬ujÅ  ã                   ó°  — d Z ddlZddlZddlZddlZddlZddlmZmZ ej                  j                  dej                  j                  ej                  j                  e«      «      «       ddlmZ  ej                  «        ed¬«      z
  j!                  d«      ZdZg d	¢Zd
„ Zdd„Zdd„Zedk(  r$dej0                  v Z ee¬«      Z ede› d�«       yy)uV   crawl_susong_towns.py - å®¿æ�¾åŽ¿ä¹¡é•‡å…¬ç¤ºå…¬å‘Šï¼ˆé«˜å²­ä¹¡/æ²³å¡Œä¹¡/åŒ—æµ´ä¹¡ï¼‰é    N)ÚdatetimeÚ	timedelta)Úpush_to_searchdbiG  )Údaysz%Y-%m-%dé   ))Ú
2000004191u    å®¿æ�¾åŽ¿-é«˜å²­ä¹¡-å…¬ç¤ºå…¬å‘Š)Ú
2000004081u    å®¿æ�¾åŽ¿-æ²³å¡Œä¹¡-å…¬ç¤ºå…¬å‘Š)Ú
2000004231u    å®¿æ�¾åŽ¿-åŒ—æµ´ä¹¡-å…¬ç¤ºå…¬å‘Šc                 óž   — t        j                  dd| «      j                  «       } dD ]  }| j                  |d«      } Œ | j                  «       S )Nz\s+ú )õ   ã€�æˆ‘è¦�çº é”™ã€‘u   åˆ†äº«åˆ°ï¼šu   æ‰“å�° ä¸‹è½½ æ”¶è—�Ú )ÚreÚsubÚstripÚreplace)ÚtÚps     ú'/root/gov_crawler/crawl_susong_towns.pyÚclean_titler      sH   € Ü
�‰ˆv�s˜AÓ×$Ñ$Ó&€AØKò ˆØ�I‰I�a˜Ó‰ðà�7‰7‹9Ðó    c              ƒ   óp  K  — ddl m}  |«       4 ƒd {  –—† }|j                  j                  ddg d¢¬«      ƒ d {  –—† }|j	                  dd¬	«      ƒ d {  –—† }|j                  d
«      ƒ d {  –—†  |j                  «       ƒ d {  –—† }|j                  ddd¬«      ƒ d {  –—†  |j                  d«      ƒ d {  –—†  g }t        D �]   \  }}t        d|› d�«       g }	t        dt        dz   «      D �]  }
t        t        t        |› d|
› �«      «      «      dd  }d|› d|› d|
› d�}|j!                  d|› d�«      ƒ d {  –—† }d|v sd|vr n¬|j#                  dg «      }|s n–|D ]q  }|j#                  dd«      j%                  d«      d   }|t&        k  rŒ1|	j)                  t+        |j#                  d d«      «      |j#                  d!d«      ||ddd"œ«       Œs | r n|j                  d#«      ƒ d {  –—†  �Œ t        d$t-        |	«      › d%�«       t/        |	«      D ]ñ  \  }}	 |j                  |d&   dd'¬«      ƒ d {  –—†  |j                  d(«      ƒ d {  –—†  |j1                  d)«      ƒ d {  –—† }|rj|j3                  «       ƒ d {  –—† }t5        j6                  d*d|t4        j8                  ¬+«      }t5        j6                  d,d|«      }|j;                  «       |d-<   nd|d-<   |dz   d2z  dk(  sŒÕt        d3|dz   › d/t-        |	«      › �«       Œó |	D �cg c]  }|d-   j;                  «       sŒ|‘Œ }}t        d4t-        |«      › d/t-        |	«      › �«       |j?                  |«       �Œ£ |jA                  «       ƒ d {  –—†  d d d «      ƒd {  –—†  rtC        |d5«       t-        |«      S 7 �Œ›7 �Œv7 �Œ^7 �ŒH7 �Œ37 �Œ7 �Œ7 �Œ‚7 �ŒÒ7 �Œ‰7 �Œs7 �Œ]7 �ŒF# t<        $ r6}t        d.|dz   › d/t-        |	«      › d0|d&   › d1|› �«       d|d-<   Y d }~�Œ'd }~ww xY wc c}w 7 Œ›7 Œ�# 1 ƒd {  –—†7  sw Y   Œ�xY w­w)6Nr   )Úasync_playwrightTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z--no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)ÚheadlessÚexecutable_pathÚargsz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36zzh-CN)Ú
user_agentÚlocalez\
            Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
        zThttps://www.susong.gov.cn/public/column/2000004191?type=4&catId=41335247&action=listÚdomcontentloadedi0u  )Ú
wait_untilÚtimeouti¸  z
=== z ===é   Ú_iïÿÿÿz,https://www.susong.gov.cn/site/label/8888?_=z4&labelName=publicInfoList&siteId=2000005581&organId=z&pageSize=20&pageIndex=z“&isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&isDriving=true&result=&isJson=true&keyWords=&isSetValue=true&catIds=&catId=41335247zt
                    async () => {
                        try {
                            const r = await fetch("z¡");
                            return await r.json();
                        } catch(e) { return { error: e.message }; }
                    }
                ÚerrorÚdataÚpublishDater   r   ÚtitleÚlink)r'   ÚurlÚpub_dateÚ	site_nameÚcontentÚsummaryi,  z  List: z itemsr)   i N  iÜ  z.clearfix.xxgkcontentu   åˆ†äº«åˆ°ï¼š.*?æ”¶è—�)Úflagsr   r,   z    Detail error [ú/z]: z - é   z    Detail: z  With content: Úsusong_towns)"Úplaywright.async_apir   ÚchromiumÚlaunchÚnew_contextÚadd_init_scriptÚnew_pageÚgotoÚwait_for_timeoutÚORGANSÚprintÚrangeÚ	MAX_PAGESÚstrÚabsÚhashÚevaluateÚgetÚsplitÚCUTOFFÚappendr   ÚlenÚ	enumerateÚquery_selectorÚ
inner_htmlr   r   ÚDOTALLr   Ú	ExceptionÚextendÚcloser   )Úincrementalr   r   ÚbrowserÚcontextÚpageÚall_recordsÚoidr+   ÚrecordsÚpnÚrandÚapi_urlr%   ÚitemsÚitemr*   ÚiÚrecÚ
content_elÚhtmlÚeÚrÚvalids                           r   Ú	run_asyncra      s¼  è ø€ Ý5áÓ!÷ Zð Z QØŸ
™
×)Ñ)ØØ\Únð *ó 
÷ 
ˆð
  ×+Ñ+ØtØð ,ó 
÷ 
ˆð ×%Ñ%ð 'ó ÷ 	ð 	ð ×%Ñ%Ó'×'ˆð �i‰iÐnð  |Nð  X]ˆió  ^÷  	^ð  	^Ø×#Ñ# DÓ)×)Ð)àˆä$ó C	&‰NˆC�Ü�F˜9˜+ TÐ*Ô+ØˆGä˜Aœy¨1™}Ó-ó $1�Üœ3œt s e¨1¨R¨D MÓ2Ó3Ó4°S°TÐ:�ØHÈÈð  OCð  DGð  CHð  H_ð  `bð  _cð  cvð  w�à!Ÿ]™]ð 04ð 5<°9ð =ð,ó ÷ �ð ˜d‘? f°DÑ&8ÙàŸ™ ¨Ó,�ÙÙà!ò �DØ#Ÿx™x¨°rÓ:×@Ñ@ÀÓEÀaÑH�HØ¤&Ò(Ø à—N‘NÜ!,¨T¯X©X°g¸rÓ-BÓ!CØ#Ÿx™x¨°Ó3Ø$,Ø%.Ø#%Ø#%ñ$õ ðñ ÙØ×+Ñ+¨CÓ0×0Ò0ðI$1ôL �HœS ›\˜N¨&Ð1Ô2ô $ GÓ,ò ?‘��3ð(ØŸ)™) C¨¡JÐ;MÐW\˜)Ó]×]Ð]Ø×/Ñ/°Ó5×5Ð5à'+×':Ñ':Ð;RÓ'S×!S�JÙ!Ø%/×%:Ñ%:Ó%<×<˜Ü!Ÿv™vÐ&>ÀÀDÔPR×PYÑPYÔZ˜Ü!Ÿv™vÐ&;¸RÀÓF˜Ø)-¯©«˜˜Išà)+˜˜I™ð
 �a‘C˜1‘9 “>Ü˜L¨¨1©¨¨Q¬s°7«|¨nÐ=Õ>ð%?ð( !(Ö@˜1¨1¨Y©<×+=Ñ+=Õ+?’QÐ@ˆEÐ@ÜÐ$¤S¨£Z L°´#°g³,°Ð@ÔAØ×Ñ˜uÖ%ðGC	&ðJ �m‰m‹o×Ð÷uZ÷ Zñx Ü˜ nÔ5äˆ{ÓÐðZùð
ùð

ùð	ùð (ùð 	^ùØ)ùðùð@ 1ùð ^ùØ5ùà!Sùà<úô !ò (ÜÐ.¨q°©s¨e°1´S¸³\°NÀ#ÀcÈ%ÁjÀ\ÐQTÐUVÐTWÐXÔYØ%'�C˜	—Nûð(üò Að 	øðuZø÷ Z÷ Zñ Züsl  ‚P6”N/•P6˜$P!¼N2½P!ÁN5ÁP!Á0N8Á1P!ÂN;Â	P!Â$N>Â%P!Â=OÂ>BP!ÅO
ÅB2P!Ç5O
Ç60P!È'OÉO

ÉOÉO
ÉOÉ3O
É4OÊO
ÊAOË)P!Ë5#P!ÌPÌ1PÌ5AP!ÎPÎP!ÎP6ÎPÎP6Î2P!Î5P!Î8P!Î;P!Î>P!ÏP!ÏP!ÏP!Ï
OÏOÏOÏOÏ	P	Ï+P	Ð
P!ÐP	Ð	P!ÐP6Ð!P3Ð'P*Ð(P3Ð/P6c                 ó@   — t        j                  t        | ¬«      «      S )N©rN   )ÚasyncioÚrunra   rc   s    r   re   re   |   s   € Ü�;‰;”y¨[Ô9Ó:Ð:r   Ú__main__z--incrementalrc   zDone: z records total)F)Ú__doc__ÚsysÚosr   rd   Útimer   r   ÚpathÚinsertÚdirnameÚabspathÚ__file__Úcrawler_libr   ÚnowÚstrftimerD   r=   r:   r   ra   re   Ú__name__ÚargvÚincÚcntr;   © r   r   ú<module>rx      sº   ðá \ß !× !Ó !ß (à ‡�‡���2—7‘7—?‘? 2§7¡7§?¡?°8Ó#<Ó=Ô >Ý (à
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€Ø€	ò
€òóbóH;ð ˆzÒØ
˜SŸX™XÐ
%€CÙ
˜#Ô
€CÙ	ˆF�3�%�~Ð
&Õ'ð r   