Ë
    u»�j,+  ã                   ó$  — d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
 dZdZdZde› d	�Zd
Z ej                   «        ed¬«      z
  j#                  d«      ZdZdZdZdZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zdd„Ze dk(  r e«        yy)uŠ  å¤§é‚‘åŽ¿äººæ°‘æ”¿åºœ - é€šçŸ¥å…¬å‘Šæ �ç›®çˆ¬è™« (c138361)
ç«™ç‚¹: https://www.day.gov.cn (dayi å¤§é‚‘, é�žå¤§å†¶!)
æ �ç›®: /day/c138361/list.shtml (æ–°é—»ä¸­å¿ƒ > é€šçŸ¥å…¬å‘Š, 1325æ�¡/67é¡µ, es-search API)
WAF: ç‘žæ•°4ä»£ JS æŒ‘æˆ˜ (412, $_ts) â†’ Playwright è¿‡æŒ‘æˆ˜å�Ž fetch
åˆ—è¡¨: es-search API: GET /es-search/search/{lmCode}?_template=zhaofa/day_public&_isAgg=1&_pageSize=20&page=N
   lmCode = 761e9373602340e792d0145a6a6a6478
   æ�¡ç›®: ul.list_text > li > h6 > a[href][title] (å�•/å�Œå¼•å�· title) + span.date (YYYY-MM-DD)
è¯¦æƒ…: fetch â†’ div.details_con h1 æ ‡é¢˜ + div.details_text æ­£æ–‡ + meta ArticleTitle/PubDate/ContentSource
é™„ä»¶: æ­£æ–‡å†… <a href="*.pdf/doc/xls..."> æ­£åˆ™æ��å�– urljoin ç»�å¯¹åŒ–
å…¥åº“: è‡ªåŠ¨æ£€æµ‹ /root/search.db å­˜åœ¨åˆ™ INSERT OR IGNORE ç›´æ�’ gov_raw (FTS è§¦å�‘å™¨è‡ªåŠ¨å�Œæ­¥ gov_search)
è¿�è¡Œ: python3 crawl_dayi_tzgg_c138361.py [--pages=N] [--no-db]
é    N)ÚdatetimeÚ	timedelta)Úsync_playwrightu   å¤§é‚‘åŽ¿-é€šçŸ¥å…¬å‘Šzhttps://www.day.gov.cnÚ 761e9373602340e792d0145a6a6a6478z/es-search/search/z8?_template=zhaofa/day_public&_isAgg=1&_pageSize=20&page=éC   iG  )Údaysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z/tmp/dayi_tzgg_c138361.jsonlz/root/search.dbu   é€šçŸ¥å…¬å‘Šu   å››å·�c                  óÎ   — t         j                  dd D ]O  } | j                  d«      rt        | j	                  d«      d   «      c S | j                  «       sŒDt        | «      c S  y)u   æ”¯æŒ� --pages=N å’Œçº¯æ•°å­—é   Nz--pages=ú=r   )ÚsysÚargvÚ
startswithÚintÚsplitÚisdigit)Úas    ú,/root/gov_crawler/crawl_dayi_tzgg_c138361.pyÚparse_pages_argr       sU   € ä�X‰X�a�bˆ\ò ˆØ�<‰<˜
Ô#Ü�q—w‘w˜s“| A‘Ó'Ò'Ø�9‰9�;Ü�q“6ŠMð	ð
 ó    c                 óœ   — t        j                  dd| xs d«      j                  «       } | j                  dd«      j                  dd«      } | S )Nu   ^[â€¢Â·Â·\s]+Ú u   â€‹u   ï»¿)ÚreÚsubÚstripÚreplace©Úts    r   Úclean_titler   *   sD   € Ü
�‰Ð  Q¢W¨"Ó-×3Ñ3Ó5€AØ	�	‰	�(˜BÓ×'Ñ'¨°"Ó5€AØ€Hr   c                 ó  — ddl m}  || d«      }|j                  d„ «      D ]  }|j                  «        Œ |j                  ddg«      D ]  }|j	                  «        Œ 	 |j                  d«      D �cg c]  }|j                  d«      sŒ|‘Œ }}|sn|d   j	                  «        ŒG|j                  d«      D ]?  }|j                  d«      sŒ|j                  d	«      rd	|j                  d	«      ini |_        ŒA t        |«      S c c}w )
uL   æ­£æ–‡æ¸…æ´—: span/font unwrap + åµŒå¥—pæ‰�å¹³åŒ– + o:pæ¸…ç�† + ä¿�ç•™è¡¨æ ¼r   )ÚBeautifulSoupzhtml.parserc                 ó:   — | j                   xr d| j                   v S )Nú:)Únamer   s    r   ú<lambda>zclean_content.<locals>.<lambda>4   s   €  q§v¡vÒ'?°#¸¿¹°-€ r   ÚspanÚfontÚpÚstyleÚalign)	Úbs4r    Úfind_allÚ	decomposeÚunwrapÚfindÚgetÚattrsÚstr)Úhtmlr    ÚsoupÚtagr'   Únesteds         r   Úclean_contentr6   0   sø   € å!Ù˜˜}Ó-€DØ�}‰}Ñ?Ó@ò ˆØ�‰�ðà�}‰}˜f fÐ-Ó.ò ˆØ�
‰
�ðð Ø!Ÿ]™]¨3Ó/Ö?˜°1·6±6¸#µ;’!Ð?ˆÐ?ÙØØˆq‰	×ÑÔð	 ð
 �]‰]˜3Óò JˆØ�5‰5��>Ø34·5±5¸´>�w §¡ g£Ñ/ÀrˆA�GðJô ˆt‹9Ðùò @s   Á3DÂ
Dc                 ó
  — g }t        j                  d| t         j                  t         j                  z  «      D �]  }|j	                  d«      j                  «       |j	                  d«      }}|r|j                  d«      rŒHt        j                  d|t         j                  «      sŒnt        j                  dd|«      j                  «       dd	 }|j                  d
«      r
t        |z   }n,|j                  d«      s|j                  d
d«      d   d
z   |z   }|j                  |xs |j                  d
«      d   |dœ«       �Œ	 t        «       }g }|D ]/  }	|	d   |vsŒ|j                  |	d   «       |j                  |	«       Œ1 |S )u2   æ­£æ–‡å†…é™„ä»¶é“¾æŽ¥: <a href="*.pdf/doc/xls...">z$<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>r
   é   Ú
javascriptz\.(pdf|docx?|xlsx?|zip|rar|7z)$ú<[^>]+>r   Néd   ú/Úhttpr   éÿÿÿÿ)r#   Úurlr?   )r   ÚfinditerÚIÚSÚgroupr   r   Úsearchr   ÚBASEÚrsplitÚappendr   ÚsetÚadd)
Údetail_htmlÚ
detail_urlÚattsÚmÚhrefÚ	name_htmlr#   ÚseenÚuniqr   s
             r   Úextract_attachmentsrR   E   sO  € à€DÜ�[‰[Ð@À+ÌrÏtÉtÔVX×VZÑVZÉ{Ó[ó 
LˆØŸ'™' !›*×*Ñ*Ó,¨a¯g©g°a«jˆiˆÙ�t—‘ |Ô4ØÜ�9‰9Ð7¸¼r¿t¹tÕDÜ—6‘6˜* b¨)Ó4×:Ñ:Ó<¸T¸cÐBˆDØ�‰˜sÔ#Ü˜d‘{‘Ø—_‘_ VÔ,Ø!×(Ñ(¨¨aÓ0°Ñ3°cÑ9¸DÑ@�Ø�K‰K Ò!<¨¯©°C«¸Ñ)<ÀTÑJÖKð
Lô ‹5€DØ€DØò ˆØˆU‰8˜4ÒØ�H‰H�Q�u‘XÔØ�K‰K˜�Nðð €Kr   c                 ó”   — t        d«      D ]  }	 | j                  d|«      c S  y# t        $ r t        j                  d«       Y Œ:w xY w)u'   fetch è¯¦æƒ…/åˆ—è¡¨, å¤±è´¥é‡�è¯•ä¸€æ¬¡r8   zAasync (u) => { const r = await fetch(u); return await r.text(); }N)ÚrangeÚevaluateÚ	ExceptionÚtimeÚsleep)Úpager?   Ú_s      r   Úfetch_okr[   \   sN   € ä�1‹Xò ˆð	Ø—=‘=Ð!hÐjmÓnÒnðð
 øô ò 	Ü�J‰J�qŽMð	ús   �&¦AÁAc                 ó¼  — | syt         j                  j                  t        «      st	        dt        › d�d¬«       yt        j                  t        d¬«      }|j                  d«       d	x}}|j                  «       }| D ]“  }	 |j                  d
t        |d   |d   |d   |d   |d   |d   xs dd d t        d|d   |j                  dt        «      |j                  dd«      f«       |j                  d	kD  r|dz  }n|dz  }|j                  «        Œ• |j!                  «        ||fS # t
        j                  $ r}t	        d|› �d¬«       |dz  }Y d }~ŒSd }~ww xY w)N)r   r   u     æœªå�‘çŽ° u   , è·³è¿‡å…¥åº“ (ä»…å†™ JSONL)T©Úflushi"  )ÚtimeoutzPRAGMA busy_timeout=290000r   zçINSERT OR IGNORE INTO gov_raw
                   (site_name, source_url, page_url, title, publish_date, content, summary, category, script_name, attachments, group_name, industry)
                   VALUES (?,?,?,?,?,?,?,?,?,?,?,?)r?   ÚtitleÚpub_dateÚcontentr   iô  zcrawl_dayi_tzgg_c138361.pyÚattachmentsÚ
group_nameÚindustryÚotherr
   z
  DB err: )ÚosÚpathÚexistsÚDB_PATHÚprintÚsqlite3ÚconnectÚexecuteÚcursorÚ	SITE_NAMEÚCATEGORYr/   Ú
GROUP_NAMEÚrowcountÚErrorÚcommitÚclose)ÚresultsÚconnÚaddedÚskippedÚcurÚrÚes          r   Ú
push_to_dbr~   f   s[  € ÙØÜ�7‰7�>‰>œ'Ô"Ü�œW˜IÐ%BÐCÈ4ÕPØÜ�?‰?œ7¨CÔ0€DØ‡L�LÐ-Ô.ØÐ€EˆGØ
�+‰+‹-€CØò ˆð	Ø�K‰Kð7ô ˜A˜e™H a¨¡h°°'±
¸A¸j¹MÈ1ÈYÉ<Ø�I‘,Ò$ " d sÐ+¬XÐ7SÐUVÐWdÑUeØ—‘�|¤ZÓ0°!·%±%¸
ÀGÓ2LðNô	ð �|‰|˜aÒØ˜‘
‘à˜1‘�ð 	�‰�ð#ð$ 	‡J�J„LØ�'ˆ>Ðøô �}‰}ò 	Ü�J˜q˜cÐ"¨$Õ/Ø�q‰L�Gûð	ús   ÂB D)Ä)EÄ<EÅEc                  óÊ
  ‡$— t        «       } dt        j                  v }t        dt        › d| xs d› dt
        › �d¬«       g }dddd	œ}t        «       5 }|j                  j                  dg d
¢¬«      }|j                  t        ddddœ¬«      }|j                  «       }	 |j                  t        dz   dd¬«       d}t        d«      D ]O  }	|j!                  d«       	 |j#                  «       }
d|
vsŒ*t%        |
«      dkD  sŒ9t        d|	dz  › d�d¬«       d} n |s't        dd¬«       |j'                  «        	 d d d «       y d}d}t        dt(        dz   «      D �]f  }| r|| kD  r �n\t        t*        z   t-        |«      z   }t/        ||«      }
|
st        d|› d �d¬«       ŒGt1        j2                  d!|
t0        j4                  «      }|s?t1        j2                  d"|
t0        j4                  «      }|r|D ��cg c]  \  }}|||d#f‘Œ }}}|D ����cg c]\  \  }}}}||||xsM t1        j6                  d$|«      r5t1        j6                  d$|«      j9                  d«      j;                  d%d&«      nd#f‘Œ^ }}}}}|st        d|› d'�d¬«       �Œ/t        d|› d%t(        › d(t%        |«      › d)�d¬«       |D �]õ  \  }}}}t=        |xs t1        j>                  d*d#|«      «      }|d d+ }|d,xx   dz  cc<   |r|t
        k  rt        d-|› d.�d¬«       d} �n™|jA                  d/«      r|nt        |z   Š$t/        |‰$«      }|st        d0|d d1 › �d¬«       Œž|d2xx   dz  cc<   t1        j6                  d3|t0        j4                  «      }|r|j9                  d«      }n=t1        j6                  d4|t0        j4                  «      }|r|j9                  d«      d d5 nd#}|rtC        |«      nd#}t1        j>                  d*d#|«      jE                  «       }t%        |«      d+k  r&d6|vr"t        d7|d d1 › �d¬«       |d8xx   dz  cc<   �Œ‰d6|v rt1        j>                  d9ˆ$fd:„|«      }tG        |‰$«      } t        |||‰$‰$tH        d;| rtK        jL                  | d¬<«      nd#d=œ	}!|jO                  |!«       |dz  }�Œø |r ntQ        jR                  d>«       �Œi |j'                  «        d d d «       tU        ||«       |s"tW        |«      \  }"}#t        d?|"› d@|#› dA�d¬«       t        dB|d,   › dC|d2   › dD|d8   › dA�d¬«       y # t        $ r Y �Œ~w xY w# t        $ r Y �Œ�w xY wc c}}w c c}}}}w # 1 sw Y   ŒˆxY w)ENz--no-dbu   ç«™ç‚¹: u    æœ€å¤§é¡µ: u   å…¨é‡�z	 CUTOFF: Tr]   r   )ÚlistÚ	detail_okÚempty)z--no-sandboxz---disable-blink-features=AutomationControlledz--disable-dev-shm-usage)ÚheadlessÚargszzh-CNiV  i„  )ÚwidthÚheight)Ú
user_agentÚlocaleÚviewportz/day/c138361/list.shtmli`ê  ru   )r_   Ú
wait_untilFé   i¸  z$_tsiˆ  u   ç‘žæ•°æŒ‘æˆ˜é€šè¿‡ é   Úsu   æŒ‘æˆ˜å¤±è´¥!r
   u     é¡µu    åˆ—è¡¨å¤±è´¥z‰<li>\s*<h6><a href="([^"]+)"[^>]*?title=[\'"]([^\'"]+)[\'"][^>]*?>(.*?)</a></h6>\s*<p>[^<]*</p>\s*(?:<span class="date">([\d-]+)</span>)?z;<li>\s*<h6><a href="([^"]+)"[^>]*?title=[\'"]([^\'"]+)[\'"]r   z/(\d{4}-\d{2}/\d{2})/r<   ú-u    æ— æ�¡ç›® (å�¯èƒ½é�žåˆ—è¡¨é¡µ)u    åˆ—è¡¨ u    æ�¡r:   é
   r€   u       æ—¥æœŸ u    < CUTOFF å�œæ­¢r=   u       è¯¦æƒ…å¤±è´¥: é   r�   z7<div[^>]*class="details_text"[^>]*>(.*?)</div>\s*</div>z$class="details_con[^"]*"[^>]*>(.*?)$i@  z<imgu       ç©ºæ­£æ–‡è·³è¿‡: r‚   z(<img[^>]+src=")([^"]+)(")c                 ór  •— | j                  d«      | j                  d«      j                  d«      r| j                  d«      na| j                  d«      j                  d«      rt        | j                  d«      z   n)‰j                  dd«      d   dz   | j                  d«      z   z   | j                  d«      z   S )Nr
   r8   r=   r<   r   rŒ   )rC   r   rE   rF   )ÚmmrK   s    €r   r$   zmain.<locals>.<lambda>á   s  ø€  2§8¡8¨A£;ÀÇÁÈ!Ã×AWÑAWÐX^ÔA_°"·(±(¸1´+Ð|~÷  }Eñ  }Eð  FGó  }H÷  }Sñ  }Sð  TWô  }XÔfjÐmo×muÑmuÐvwÓmxÒfxð  ^h÷  ^oñ  ^oð  psð  uvó  ^wð  xyñ  ^zð  }@ñ  ^@ð  CE÷  CKñ  CKð  LMó  CNñ  ^Nñ  $Pð  SU÷  S[ñ  S[ð  \]ó  S^ñ  $^€ r   rf   ©Úensure_ascii)	Ú	site_namer`   ra   rb   Ú
source_urlr?   rd   re   rc   gš™™™™™é?u   === å…¥åº“: æ–°å¢žu    è·³è¿‡z ===u   === å®Œæˆ�: åˆ—è¡¨u	    è¯¦æƒ…OKu    ç©º),r   r   r   rk   rp   ÚCUTOFFr   ÚchromiumÚlaunchÚnew_contextÚUAÚnew_pageÚgotorE   rV   rT   Úwait_for_timeoutrb   Úlenrv   ÚTOTAL_PAGESÚLIST_APIr1   r[   r   ÚfindallrB   rD   rC   r   r   r   r   r6   r   rR   rr   ÚjsonÚdumpsrG   rW   rX   Úsave_resultsr~   )%Ú	max_pagesÚno_dbrw   Ústatsr'   ÚbrowserÚctxrY   ÚpassedÚir2   Útotal_fetchedÚ
cutoff_hitÚpnr?   ÚitemsÚhr   ÚthÚdrN   Út_titleÚt_htmlÚdater`   ra   rJ   rM   Úcontent_htmlÚm2rb   ÚtextrL   Úrecry   rz   rK   s%                                       @r   Úmainr»   †   sì  ø€ ÜÓ!€IØœŸ™Ð!€EÜ	ˆH”Y�K˜|¨IÒ,A¸Ð+BÀ)ÌFÈ8Ð
TÐ\`Õaà€GØ Q°Ñ3€EÜ	Ó	ð h˜aØ—*‘*×#Ñ#ØÚmð $ó oˆð ×!Ñ!¬R¸ÐTXÐdgÑJhÐ!ÓiˆØ�|‰|‹~ˆð	Ø�I‰I”dÐ6Ñ6ÀÐRZˆIÔ[ð ˆÜ�r“ò 		ˆAØ×!Ñ! $Ô'ðØ—|‘|“~�ð ˜TÒ!¤c¨$£i°$Ó&6ÜÐ+¨A¨a©C¨5°Ð2¸$Õ?Ø�Ùð		ñ Ü�/¨Õ.Ø�M‰MŒOØ÷5hð hð: ˆØˆ
Ü˜œ;¨™?Ó+ó G	ˆBÙ˜R )š^ÚÜœ‘/¤C¨£GÑ+ˆCÜ˜D #Ó&ˆDÙÜ˜˜b˜T Ð/°tÕ<ØÜ—J‘Jð   lð  nrô  tv÷  txñ  txó  yˆEÙäŸ
™
Ð#aÐcgÔik×imÑimÓn�ÙØ7<×=©t¨q°!˜a  A rš]Ð=�EÑ=ð pu÷  vñ  vñ  alð  abð  deð  gið  kl�a˜˜B ò  ![Ôkm×ktÑktð  vNð  PQô  lR¤b§i¡iÐ0HÈ!Ó&L×&RÑ&RÐSTÓ&U×&]Ñ&]Ð^aÐcfÔ&gð  XZò  \ð  vˆEó  vÙÜ˜˜b˜TÐ!@ÐAÈÕNÙÜ�E˜"˜˜Qœ{˜m¨8´C¸³J°<¸tÐDÈDÕQØ/4ó 0#Ñ+��g˜v tÜ# GÒ$M¬r¯v©v°jÀ"ÀfÓ/MÓN�Ø  ˜9�Ø�f“ Ñ"“Ù ¬6Ò 1Ü˜K¨ zÐ1AÐBÈ$ÕOØ!%�JÚà%)§_¡_°VÔ%<™TÄ$ÈÁ+�
Ü& t¨ZÓ8�Ù"ÜÐ.¨u°S°b¨z¨lÐ;À4ÕHØØ�kÓ" aÑ'Ó"ä—I‘IÐXÐZeÔgi×gkÑgkÓl�ÙØ#$§7¡7¨1£:‘Lô Ÿ™Ð#JÈKÔY[×Y]ÑY]Ó^�BÙ9; 2§8¡8¨A£;¨u°Ñ#5À�LÙ9Eœ-¨Ô5È2�Ü—v‘v˜j¨"¨gÓ6×<Ñ<Ó>�Ü�t“9˜r’> f°GÑ&;ÜÐ1°%¸¸°*°Ð>ÀdÕKØ˜'“N aÑ'“NÙà˜WÑ$Ü Ÿf™fØ5ó ^Øó!�Gô +¨;¸
ÓC�ä!*Ø"Ø (Ø&Ø",Ø%Ü",Ø 'ÙKO¤4§:¡:¨dÀÕ#GÐUWñ
�ð —‘˜sÔ#Ø Ñ"’ða0#ñb ÙÜ�J‰J�sŽOðOG	ðR 	�‰Œ÷QhôT �˜%Ô ÙÜ# GÓ,‰ˆˆwÜÐ" 5 '¨°°	¸Ð>ÀdÕKÜ	Ð˜u V™}˜o¨Y°u¸[Ñ7IÐ6JÈ$ÈuÐU\É~ÐN^Ð^bÐ
cÐkoÖpøôI ò 	Úð	ûô ò Úðüó6 >ùõ v÷]hð hús‚   ÁAUÂT+Â9!UÃT;Ã+UÃ0UÃ?9UÅB1UÇ2U
È
UÈA!UÉ-I%UÔ+	T8Ô4UÔ7T8Ô8UÔ;	UÕUÕUÕUÕU"c                 óô   — t        t        dd¬«      5 }| D ]+  }|j                  t        j                  |d¬«      dz   «       Œ- 	 d d d «       t        dt        | «      › dt        › �d	¬
«       y # 1 sw Y   Œ*xY w)NÚwzutf-8)ÚencodingFr“   ú
u   å·²å†™ u    æ�¡åˆ° Tr]   )ÚopenÚOUTÚwriter£   r¤   rk   rŸ   )rw   r¨   Úfr|   s       r   r¥   r¥   þ   sm   € Ü	Œc�3 Ô	)ð >¨QØò 	>ˆAØ�G‰G”D—J‘J˜q¨uÔ5¸Ñ<Õ=ñ	>÷>ô 
ˆG”C˜“L�> ¬#¨Ð
/°tÖ<÷>ð >ús   “1A.Á.A7Ú__main__)N)!Ú__doc__r£   rg   r   r   rW   rl   r   r   Úplaywright.sync_apir   rp   rE   ÚLMCODEr¡   r    ÚnowÚstrftimer—   r›   rÁ   rj   rq   rr   r   r   r6   rR   r[   r~   r»   r¥   Ú__name__© r   r   ú<module>rÌ      s¹   ðñ÷ (× '× 'ß (Ý /à$€	Ø€Ø	+€Ø ˜xÐ'_Ð`€Ø€Ø
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€Øv€Ø$€Ø
€Ø€Ø€
òòòò*ò.òò@uqóp=ð ˆzÒÙ…Fð r   