Ë
     ñ‚j  ã                   óF  — d Z ddlZddlZddlZddlZddlZddlZddlZej                  j                  dej                  j                  ej                  j                  e«      «      «       ddlmZ dZdZedz   ZdZddlmZ dd	lmZ dd
„Zdd„Zd„ Zedk(  r e«        yy)uˆ  
crawl_ahtxq_qypd.py - é»„å±±å¸‚å±¯æºªåŒº-è�¥å•†çŽ¯å¢ƒä¼�ä¸šé¢‘é�“ (ahtxq.gov.cn)
ç«™ç‚¹: å¸�è�”NWAF + ç‘žæ•°4ä»£å�ŒWAF (1eeb27c + ctct_bundle) -> å¿…é¡» playwright stealth
åˆ—è¡¨: /zxzx/ztzl/rdzt/hsjldtjjyq/qypd/index.html (é�™æ€�, æ— åˆ†é¡µ, ~9æ�¡)
è¯¦æƒ…: /zxzx/ztzl/rdzt/hsjldtjjyq/qypd/{id}.html
  æ ‡é¢˜: h1 / meta ArticleTitle | æ—¥æœŸ: meta PubDate / é¡µé�¢æ–‡æœ¬ | æ­£æ–‡: å¾…ç¡®è®¤
é    N)Úpush_to_searchdbu"   å±¯æºªåŒº-è�¥å•†çŽ¯å¢ƒä¼�ä¸šé¢‘é�“zhttps://www.ahtxq.gov.cnz*/zxzx/ztzl/rdzt/hsjldtjjyq/qypd/index.htmlzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36)Úasync_playwright)ÚStealthc              ƒ   ó  K  — t        |«      D ];  }	 | j                  |d|¬«      ƒ d {  –—† }| j                  d«      ƒ d {  –—†  |c S  y 7 Œ#7 Œ# t        $ r% ||dz
  k  r| j                  d«      ƒ d {  –—†7   Y Œpw xY w­w)NÚdomcontentloaded)Ú
wait_untilÚtimeoutiˆ  é   é¸  )ÚrangeÚgotoÚwait_for_timeoutÚ	Exception)ÚpageÚurlr	   ÚretriesÚattemptÚresps         ú%/root/gov_crawler/crawl_ahtxq_qypd.pyÚgoto_with_retryr      s�   è ø€ Ü˜“>ò 2ˆð	2ØŸ™ 3Ð3EÈw˜ÓW×WˆDØ×'Ñ'¨Ó-×-Ð-ØŠKð	2ð ð XøØ-ùäò 	2Ø˜ 1™Ò$Ø×+Ñ+¨DÓ1×1Ñ1ùð	2üsV   ‚B’A©AªAÁAÁAÁBÁAÁAÁ%A?Á6A9Á7A?Á<BÁ>A?Á?Bc              ƒ   ó@
  K  — t        «       4 ƒd {  –—† }|j                  j                  dddg¬«      ƒ d {  –—† }|j                  t        ddddœ¬«      ƒ d {  –—† }t        «       }|j                  |«      ƒ d {  –—†  |j                  «       ƒ d {  –—† }t        |t        «      ƒ d {  –—† }t        d	|r|j                  nd
› d|j                  «       ƒ d {  –—† › �«       |j                  d«      ƒ d {  –—† }t        «       }g }	|D ]a  }
|
d   }|j                  d«      r
t         |z   }n|j                  d«      sŒ5||v rŒ:|j#                  |«       |	j%                  |
d   |dœ«       Œc t        dt'        |	«      › d�«       |	s*|j)                  «       ƒ d {  –—†  	 d d d «      ƒd {  –—†  yg }t+        |	d«      D �]Þ  \  }}
|
d   }t        ||«      ƒ d {  –—† }|j-                  «       ƒ d {  –—† }|
d   }t/        j0                  d|«      }|st/        j0                  d|«      }|r1|j3                  d«      j5                  «       }|rt'        |«      dkD  r|}d}t/        j0                  d|«      }|r"|j3                  d«      j5                  «       d d }|s)t/        j0                  d|«      }|r|j3                  d«      }d}dD ]<  }t/        j0                  ||t.        j6                  «      }|sŒ+|j3                  d«      } n |s;t/        j0                  d |t.        j6                  «      }|r|j3                  d«      d d! }|r½t/        j8                  d"d|t.        j6                  ¬#«      }t/        j8                  d$d|t.        j6                  ¬#«      }t/        j8                  d%d&t         z   dz   |«      }t/        j8                  d'd(t         z   dz   |«      }|j5                  «       }|j%                  t:        |||||d)œ«       nAt        d*|› dt'        |	«      › d+|d d, › d-�«       |j%                  t:        ||d.|› d/�||d)œ«       t        d*|› dt'        |	«      › d0|d d, › d1t'        |«      › d2�	«       |j=                  t?        j@                  d3d4«      «      ƒ d {  –—†  �Œá |j)                  «       ƒ d {  –—†  tC        |d5¬6«      }|cd d d «      ƒd {  –—†  S 7 �ŒÎ7 �Œª7 �ŒŠ7 �Œj7 �ŒU7 �Œ@7 �Œ7 �Œø7 �ŒW7 �ŒI7 �Œ7 �Œ7 Œc7 ŒJ7 Œ-# 1 ƒd {  –—†7  sw Y   y xY w­w)7NTz---disable-blink-features=AutomationControlledz--no-sandbox)ÚheadlessÚargsiV  i„  )ÚwidthÚheight)Ú
user_agentÚignore_https_errorsÚviewportz[LOAD] status=zN/Az title=a   () => {
            const out = [];
            document.querySelectorAll('a[href*="/qypd/"].html, a[href*="/qypd/"]').forEach(a => {
                const href = a.getAttribute('href') || '';
                const t = (a.innerText || '').trim().replace(/\s+/g, ' ');
                if (href.includes('/qypd/') && /\d+\.html$/.test(href) && t.length > 5) {
                    if (!out.find(x => x.href === href)) out.push({t: t, href: href});
                }
            });
            return out;
        }Úhrefú/ÚhttpÚt)Útitler   z[LIST] u    æ�¡r   r
   r   r#   z'name="ArticleTitle"\s+content="([^"]+)"z<h1[^>]*>([^<]+)</h1>é   Ú z"name="PubDate"\s+content="([^"]+)"é
   z(\d{4}-\d{2}-\d{2}))zT<div[^>]*class="[^"]*wzcon[^"]*"[^>]*>(.*?)</div>\s*<div[^>]*class="[^"]*clear[^"]*"zV<div[^>]*class="[^"]*content[^"]*"[^>]*>(.*?)</div>\s*<div[^>]*class="[^"]*clear[^"]*"z#<div[^>]*id="zoom"[^>]*>(.*?)</div>zD<div[^>]*class="[^"]*(?:article|news|detail|content)[^"]*"[^>]*>(.*)i@  z<script.*?</script>)Úflagsz<style.*?</style>zhref="/zhref="zsrc="/zsrc=")Ú	site_namer#   Úpub_dateÚcontentÚ
source_urlr   z  [z] [SKIP-EMPTY] é2   z...z<p>z</p>z] z
... (body=ú)iÜ  r   Ú
ahtxq_qypd)Úbatch_label)"r   ÚchromiumÚlaunchÚnew_contextÚUAr   Úapply_stealth_asyncÚnew_pager   ÚLIST_URLÚprintÚstatusr#   ÚevaluateÚsetÚ
startswithÚBASEÚaddÚappendÚlenÚcloseÚ	enumerater*   ÚreÚsearchÚgroupÚstripÚSÚsubÚ	SITE_NAMEr   ÚrandomÚrandintr   )ÚpagesÚpÚbrowserÚctxÚstealthr   r   ÚitemsÚseenÚitems2Úitr   ÚrecordsÚir   Úhtmlr#   ÚmÚt2r)   ÚbodyÚsel_patÚnew_cs                          r   Ú	run_asyncr\   #   sé  è ø€ ÜÓ!÷ nð n QØŸ
™
×)Ñ)°4Ø;Øð?
Ð)ó ÷ ˆð ×'Ñ'ÜØ $Ø#¨sÑ3ð (ó 
÷ 
ˆô
 “)ˆØ×)Ñ)¨#Ó.×.Ð.Ø—\‘\“^×#ˆä$ T¬8Ó4×4ˆÜ�©d˜tŸ{š{¸Ð>¸gÈDÏJÉJËL×FXÐEYÐZÔ[ð —m‘mð 
%ó 
÷ 
ˆô ‹uˆØˆØò 		;ˆBØ�f‘:ˆDØ�‰˜sÔ#Ü˜d‘{‘Ø—_‘_ VÔ,ØØ�t‰|ØØ�H‰H�TŒNØ�M‰M B s¡G°DÑ9Õ:ð		;ô 	�œ˜F›�} DÐ)Ô*ÙØ—-‘-“/×!Ð!Ø÷[n÷ nð nð` ˆÜ˜v qÓ)ó 9	D‰EˆAˆrØ�U‘)ˆCÜ(¨¨sÓ3×3ˆDØŸ™›×'ˆDð �w‘KˆEÜ—	‘	ÐDÀdÓKˆAÙÜ—I‘IÐ6¸Ó=�ÙØ—W‘W˜Q“Z×%Ñ%Ó'�Ùœ#˜b›' Aš+Ø�Eð ˆHÜ—	‘	Ð?ÀÓFˆAÙØŸ7™7 1›:×+Ñ+Ó-¨c¨rÐ2�ÙÜ—I‘IÐ4°dÓ;�ÙØ Ÿw™w q›z�Hð ˆDðDò �ô —I‘I˜g t¬R¯T©TÓ2�ÚØŸ7™7 1›:�DÙðñ ä—I‘IÐeÐgkÔmo×mqÑmqÓr�ÙØŸ7™7 1›: e tÐ,�DáÜ—v‘vÐ4°b¸$ÄbÇdÁdÔK�Ü—v‘vÐ2°B¸ÄBÇDÁDÔI�Ü—v‘v˜j¨(´T©/¸CÑ*?ÀÓF�Ü—v‘v˜i¨´4©¸#Ñ)=¸tÓD�Ø—z‘z“|�Ø—‘Ü!*°UÈØ#°3¸sñ õ ô
 ˜˜A˜3˜a¤ F£˜}¨O¸EÀ#À2¸J¸<ÀsÐKÔLØ—‘Ü!*°UÈØ!$ U G¨4Ð0ÀÈCñ ô ô �C˜�s˜!œC ›K˜=¨¨5°°"¨:¨,°jÄÀTÃÀÈ1ÐMÔNØ×'Ñ'¬¯©°t¸TÓ(BÓC×CÒCðs9	Dðv �m‰m‹o×ÐÜ  °lÔCˆØ÷]n÷ nò nùðùð
ùð 	/ùØ#ùà4ùØFXùð
ùð4 "ùðYnùðf 4ùØ'ùðl DøàøðYnø÷ n÷ nñ nüsA  ‚T‘S’T•#T	¸S"¹"T	ÁS%Á"T	Á>S(Á?T	ÂS+ÂT	Â.S.Â/.T	ÃS1ÃT	Ã<S4Ã=B#T	Æ S7Æ!T	Æ&TÆ1S:Æ2TÆ7)T	Ç S=Ç!T	Ç8T Ç9C?T	Ë9F&T	ÒTÒ T	Ò:TÒ;T	ÓTÓTÓTÓ"T	Ó%T	Ó(T	Ó+T	Ó.T	Ó1T	Ó4T	Ó7T	Ó:TÓ=T	Ô T	ÔT	ÔT	ÔTÔ	TÔTÔTÔTc                  ó  — d} t        t        j                  «      D ]4  \  }}|j                  d«      sŒt	        |j                  d«      d   «      } Œ6 t        j                  t        | ¬«      «      }t        d|› d�«       y )Né   z--pages=ú=r
   )rK   zDone: z records)
rA   ÚsysÚargvr;   ÚintÚsplitÚasyncioÚrunr\   r7   )rK   rU   ÚaÚns       r   Úmainrh   •   sk   € Ø€EÜœ#Ÿ(™(Ó#ò )‰ˆˆ1Ø�<‰<˜
Õ#Ü˜Ÿ™ › Q™Ó(‰Eð)ô 	�‰”I EÔ*Ó+€AÜ	ˆF�1�#�XÐ
Õó    Ú__main__)iÈ¯  r$   )r^   )Ú__doc__r`   ÚosrB   rd   ÚjsonrI   ÚtimeÚpathÚinsertÚdirnameÚabspathÚ__file__Úcrawler_libr   rH   r<   r6   r3   Úplaywright.async_apir   Úplaywright_stealthr   r   r\   rh   Ú__name__© ri   r   ú<module>ry      sŒ   ðñ÷ 0× /× /Ó /Ø ‡�‡���2—7‘7—?‘? 2§7¡7§?¡?°8Ó#<Ó=Ô >Ý (à0€	Ø!€ØÐ>Ñ>€Øv€å 1Ý &ó	óoòd ð ˆzÒÙ…Fð ri   