Ë
    n}j  ã                   ó¶  — d dl Z 	 d dlZd dl Z d dlZd dlZd dlZd dlmZmZ ej                  j                  d e j                  j                  e j                  j                  e«      «      «       d dlmZ  ej                  «        ed¬«      z
  j                  d«      ZdZdZd	Zd
„ Zdd„Zdd„Zedk(  r$dej0                  v Z ee¬«      Z ede› d�«       yy)é    N)ÚdatetimeÚ	timedelta)Úpush_to_searchdbiG  )Údaysz%Y-%m-%du"   å®¿æ�¾åŽ¿äººæ°‘æ”¿åºœ-è¡Œæ”¿è®¸å�¯é
   a   https://www.susong.gov.cn/site/label/8888?_={rand}&labelName=publicInfoList&siteId=2000005581&organId=2000003861&pageSize=20&pageIndex={page}&isDate=true&dateFormat=yyyy-MM-dd&length=50&type=4&action=list&isDriving=true&result=&isJson=true&keyWords=&isSetValue=true&catIds=&catId=41336664c                 óž   — t        j                  dd| «      j                  «       } dD ]  }| j                  |d«      } Œ | j                  «       S )Nz\s+ú )õ   ã€�æˆ‘è¦�çº é”™ã€‘u   åˆ†äº«åˆ°ï¼šu   æ‰“å�° ä¸‹è½½ æ”¶è—�Ú )ÚreÚsubÚstripÚreplace)ÚtÚps     ú!/root/gov_crawler/crawl_susong.pyÚclean_titler      sH   € Ü
�‰ˆv�s˜AÓ×$Ñ$Ó&€AØKò ˆØ�I‰I�a˜Ó‰ðà�7‰7‹9Ðó    c              ƒ   ó2  K  — ddl m}  |«       4 ƒd {  –—† }|j                  j                  ddg d¢¬«      ƒ d {  –—† }|j	                  dd¬	«      ƒ d {  –—† }|j                  d
«      ƒ d {  –—†  |j                  «       ƒ d {  –—† }d}|j                  |dd¬«      ƒ d {  –—†  |j                  d«      ƒ d {  –—†  g }t        dt        dz   «      D �]  }t        t        t        d|› �«      «      «      dd  }	t        j                  |	|¬«      }
|j!                  d|
› d�«      ƒ d {  –—† }d|v sd|vr n´|j#                  dg «      }|s nž|D ]y  }|j#                  dd«      j%                  d«      d   }|t&        k  rŒ1t)        |j#                  dd«      «      }|j#                  dd«      }|j+                  |||t,        dddœ«       Œ{ | r n|j                  d«      ƒ d {  –—†  �Œ t/        dt1        |«      › d �«       t3        |«      D ]ñ  \  }}	 |j                  |d!   dd"¬«      ƒ d {  –—†  |j                  d#«      ƒ d {  –—†  |j5                  d$«      ƒ d {  –—† }|rj|j7                  «       ƒ d {  –—† }t9        j:                  d%d|t8        j<                  ¬&«      }t9        j:                  d'd|«      }|j?                  «       |d(<   nd|d(<   |dz   d-z  dk(  sŒÕt/        d.|dz   › d*t1        |«      › �«       Œó |jC                  «       ƒ d {  –—†  d d d «      ƒd {  –—†  D �cg c]  }|d(   j?                  «       sŒ|‘Œ }}t/        d/t1        |«      › d*t1        |«      › �«       |rtE        |d0«       t1        |«      S 7 �Œ|7 �ŒW7 �Œ?7 �Œ)7 �Œ7 �Œù7 �Œã7 �Œv7 �Œ¾7 �Œu7 �Œ_7 �ŒI7 �Œ2# t@        $ r6}t/        d)|dz   › d*t1        |«      › d+|d!   › d,|› �«       d|d(<   Y d }~�Œd }~ww xY w7 ŒÜ7 ŒÎ# 1 ƒd {  –—†7  sw Y   ŒÞxY wc c}w ­w)1Nr   )Úasync_playwrightTz>/root/.cache/ms-playwright/chromium-1228/chrome-linux64/chrome)z--no-sandboxz--disable-setuid-sandboxz---disable-blink-features=AutomationControlled)ÚheadlessÚexecutable_pathÚargsz[Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36zzh-CN)Ú
user_agentÚlocalez\
            Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
        zZhttps://www.susong.gov.cn/public/column/2000003861?type=4&catId=41336664&action=list&nav=3Údomcontentloadedi0u  )Ú
wait_untilÚtimeouti¸  é   Úsusong_iïÿÿÿ)ÚrandÚpagezh
                async () => {
                    try {
                        const r = await fetch("z‘");
                        return await r.json();
                    } catch(e) { return { error: e.message }; }
                }
            ÚerrorÚdataÚpublishDater   r	   ÚtitleÚlink)r&   ÚurlÚpub_dateÚ	site_nameÚcontentÚsummaryiô  z  List: z itemsr(   i N  iÜ  z.clearfix.xxgkcontentu   åˆ†äº«åˆ°ï¼š.*?æ”¶è—�)Úflagsr
   r+   z  Detail error [ú/z]: z - é   z
  Detail: z  With content: Úsusong_xzxk)#Úplaywright.async_apir   ÚchromiumÚlaunchÚnew_contextÚadd_init_scriptÚnew_pageÚgotoÚwait_for_timeoutÚrangeÚ	MAX_PAGESÚstrÚabsÚhashÚAPI_TPLÚformatÚevaluateÚgetÚsplitÚCUTOFFr   ÚappendÚ	SITE_NAMEÚprintÚlenÚ	enumerateÚquery_selectorÚ
inner_htmlr   r   ÚDOTALLr   Ú	ExceptionÚcloser   )Úincrementalr   r   ÚbrowserÚcontextr"   Úlist_urlÚrecordsÚpnr!   Úapi_urlr$   ÚitemsÚitemr)   r&   r'   ÚiÚrecÚ
content_elÚhtmlÚeÚrÚvalids                           r   Ú	run_asyncr^      sD  è ø€ Ý5áÓ!÷ Uð U QØŸ
™
×)Ñ)ØØ\Únð *ó 
÷ 
ˆð
  ×+Ñ+ØtØð ,ó 
÷ 
ˆð ×%Ñ%ð 'ó ÷ 	ð 	ð ×%Ñ%Ó'×'ˆð pˆØ�i‰i˜Ð-?ÈˆiÓO×OÐOØ×#Ñ# DÓ)×)Ð)àˆÜ˜œ9 q™=Ó)ó '	-ˆBÜ”sœ4 '¨"¨ Ó/Ó0Ó1°#°$Ð7ˆDÜ—n‘n¨$°R�nÓ8ˆGàŸ™ð ,0ð 18¨yð 9ð(ó ÷ ˆDð ˜$‰ &°Ñ"4Ùà—H‘H˜V RÓ(ˆEÙÙàò �ØŸ8™8 M°2Ó6×<Ñ<¸SÓAÀ!ÑD�ØœfÒ$Øä# D§H¡H¨W°bÓ$9Ó:�Ø—x‘x ¨Ó+�à—‘Ø"ØØ (Ü!*Ø!Ø!ñ õ ðñ" ÙØ×'Ñ'¨Ó,×,Ò,ðO'	-ôR 	�œ˜W›˜ fÐ-Ô.ô   Ó(ò 	9‰FˆAˆsð$Ø—i‘i  E¡
Ð7IÐSX�iÓY×YÐYØ×+Ñ+¨DÓ1×1Ð1à#'×#6Ñ#6Ð7NÓ#O×O�
ÙØ!+×!6Ñ!6Ó!8×8�DÜŸ6™6Ð":¸BÀÌBÏIÉIÔV�DÜŸ6™6Ð"7¸¸TÓB�DØ%)§Z¡Z£\�C˜	’Nà%'�C˜	‘Nð
 �!‘�q‰y˜A‹~Ü˜
 1 Q¡3 % q¬¨W«¨Ð7Õ8ð%	9ð( �m‰m‹o×Ð÷kU÷ Uðn  Ö8�1 1 Y¡<×#5Ñ#5Õ#7ŠQÐ8€EÐ8Ü	ÐœS ›Z˜L¨¬#¨g«,¨Ð
8Ô9áÜ˜ Ô.äˆu‹:Ðð{Uùð
ùð

ùð	ùð (ùð 	PùØ)ùðùðF -ùð ZùØ1ùàOùà8úô ò $ÜÐ(¨¨1©¨¨Q¬s°7«|¨n¸CÀÀEÁ
¸|È3ÈqÈcÐRÔSØ!#��I—ûð$úð 	øðkUø÷ U÷ Uñ Uüòn 9ùsk  ‚P”N•P˜$O=¼N½O=ÁNÁO=Á0NÁ1O=ÂNÂ	O=Â&NÂ'O=Â?N"Ã A/O=Ä/N%Ä0B:O=Ç*N(Ç+0O=ÈN7È6N+È7N7ÉN.ÉN7É(N1É)N7ÊN4ÊAN7ËO=Ë*2O=ÌO9ÌO=Ì!PÌ,O;Ì-PÌ5PÍPÍ?PÎO=ÎO=ÎO=ÎO=ÎO=Î"O=Î%O=Î(O=Î+N7Î.N7Î1N7Î4N7Î7	O6Ï +O1Ï+O=Ï1O6Ï6O=Ï;PÏ=PÐPÐPÐPc                 ó@   — t        j                  t        | ¬«      «      S )N©rN   )ÚasyncioÚrunr^   r`   s    r   rb   rb   w   s   € Ü�;‰;”y¨[Ô9Ó:Ð:r   Ú__main__z--incrementalr`   zDone: z records)F)ÚosÚsysr   ra   Útimer   r   ÚpathÚinsertÚdirnameÚabspathÚ__file__Úcrawler_libr   ÚnowÚstrftimerC   rE   r:   r>   r   r^   rb   Ú__name__ÚargvÚincÚcntrF   © r   r   ú<module>rt      sÃ   ðã 	Ø Pß !× !Ó !ß (à ‡�‡���2—7‘7—?‘? 2§7¡7§?¡?°8Ó#<Ó=Ô >Ý (à
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€Ø0€	Ø€	ð m€òó`óD;ð ˆzÒØ
˜SŸX™XÐ
%€CÙ
˜#Ô
€CÙ	ˆF�3�%�xÐ
 Õ!ð r   