Ë
    M�†jA  ã                   óœ  — d Z ddlZddlZddlZddlmZ ddlZddlmZ  ej                  dd«      Z
dZdZed	z   Z eej                  «      d
kD  r2ej                  d
   j!                  «       r eej                  d
   «      ndZddiZ ej(                  «       Zej,                  j/                  e«       d„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u–   
å¾·å®‰åŽ¿äººæ°‘æ”¿åºœ-å…¬å‘Šå…¬ç¤º
æ•°æ�®æº�: XML files (data_2025.xml, data_2026.xml) - via Playwright
è¯¦æƒ…: div#detailCont (zhengwen TRS_Editor)
é    N)ÚBeautifulSoup)Úsync_playwrightÚ	SEARCH_DBz/root/search.dbu   å¾·å®‰åŽ¿-å…¬å‘Šå…¬ç¤ºzhttps://www.dean.gov.cnz/xwzx/zdgg/é   im  z
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36c                 ó4  — t        «       5 }|j                  j                  d¬«      }|j                  «       }|j	                  t
        dd¬«       |j                  d«       |j                  d| «      }|j                  «        |cddd«       S # 1 sw Y   yxY w)	z=Fetch XML data via Playwright (bypasses anti-curl protection)T)ÚheadlessÚdomcontentloadedi˜:  )Ú
wait_untilÚtimeoutiÐ  z©
            async (yr) => {
                const resp = await fetch('/xwzx/zdgg/data_' + yr + '.xml');
                return await resp.text();
            }
        N)	r   ÚchromiumÚlaunchÚnew_pageÚgotoÚLIST_URLÚwait_for_timeoutÚevaluateÚclose)ÚyearÚpÚbrowserÚpageÚxmls        ú/root/gov_crawler/crawl_dean.pyÚfetch_xml_via_playwrightr      s‹   € ä	Ó	ð ˜aØ—*‘*×#Ñ#¨TÐ#Ó2ˆØ×ÑÓ!ˆØ�	‰	”(Ð'9À5ˆ	ÔIØ×Ñ˜dÔ#Ø�m‰mð ð
 óˆð 	�‰ŒØ÷÷ ò ús   ‹A9BÂBc                 óâ  — t        | d«      }g }|j                  d«      D ]Ì  }|j                  d«      }|j                  d«      }|j                  d«      }|r|j                  d¬«      nd}|r|j                  d¬«      nd}|r|j                  d¬«      nd}	|r|sŒ}|j	                  d	«      r|}
n'|j	                  d
«      rt
        |dd z   }
n	t
        |z   }
|j                  |
||	dœ«       ŒÎ |S )zParse XML to extract itemsr   ÚITEMÚTITLEÚPUBURLÚRELTIMET)ÚstripÚ Úhttpz./é   N)Úpage_urlÚtitleÚpublish_date)r   Úfind_allÚfindÚget_textÚ
startswithr   Úappend)Úxml_textÚsoupÚitemsÚitemÚtitle_elÚurl_elÚdate_elr%   ÚpuburlÚdate_strr$   s              r   Úparse_xml_itemsr5   *   sø   € ä˜ 5Ó)€DØ€EØ—‘˜fÓ%ò WˆØ—9‘9˜WÓ%ˆØ—‘˜8Ó$ˆØ—)‘)˜IÓ&ˆá19�×!Ñ!¨Ð!Ô-¸rˆÙ06�—‘ t�Ô,¸BˆÙ3:�7×#Ñ#¨$Ð#Ô/Àˆá™FØð ×Ñ˜VÔ$Ø‰HØ×Ñ˜tÔ$Ü &¨¨ *Ñ,‰Hä &Ñ(ˆHà�‰ (°UÈHÑUÕVð)Wð, €Ló    c                 ó€  — 	 t         j                  | d¬«      }d|_        t        |j                  d«      }d}dD ]R  }|j                  |d   |d   «      }|sŒ|j                  d	d
g«      D ]  }|j                  «        Œ t        |«      } |S  |S # t        $ r#}t        dt        |«      z   d¬«       Y d }~yd }~ww xY w)Né   ©r   zutf-8zhtml.parserr!   ))ÚidÚ
detailCont)Úclass_Ú
TRS_Editor)r<   Útrs_editor_view)r<   Úzhengwenr   r   ÚscriptÚstylez
  [ERROR] T©Úflush)ÚsessionÚgetÚencodingr   Útextr(   r'   Ú	decomposeÚstrÚ	ExceptionÚprint)r$   Úrr-   ÚcontentÚselÚdÚtagÚes           r   Úfetch_detailrR   G   sÇ   € ðÜ�K‰K˜¨"ˆKÓ-ˆØˆŒ
Ü˜QŸV™V ]Ó3ˆàˆð
ò 	ˆCð —	‘	˜#˜a™& # a¡&Ó)ˆAÚØŸ:™: x°Ð&9Ó:ò $�CØ—M‘M•Oð$ä˜a›&�Øàˆð	ð ˆøÜò ÜˆlœS ›VÑ#¨4Õ0Üûðús$   ‚AB Á5B ÂB Â	B=ÂB8Â8B=c                  ó  — t        dj                  t        t        «      d¬«       t	        j
                  t        d¬«      } | j                  «       }t        dd¬«       t        d«      }t        |«      }t        dj                  t        |«      «      d¬«       t        d	d¬«       t        d
«      }t        |«      }t        dj                  t        |«      «      d¬«       ||z   }t        dj                  t        |«      «      d¬«       |j                  dt        f«       t        d„ |j                  «       D «       «      }d}t        |«      D ]¦  \  }	}
|
d   |v rŒt        |
d   «      }||
d<   |j                  dt        t         |
d   |
d   |
d   |
d   f«       |j"                  dkD  r|dz  }| j%                  «        |	dz   dz  dk(  sŒ}t        dj                  |	dz   t        |«      |«      d¬«       Œ¨ | j'                  «        t	        j
                  t        d¬«      j                  dt        f«      j)                  «       d   }t        dj                  ||«      d¬«       t        dj                  t        «      d¬«       y )Nz[{}] Starting crawl (days={})TrB   é<   r9   z  Fetching data_2026.xml...Ú2026z  2026 items: {}z  Fetching data_2025.xml...Ú2025z  2025 items: {}z  Total items: {}z0SELECT page_url FROM gov_raw WHERE site_name = ?c              3   ó&   K  — | ]	  }|d    –— Œ y­w)r   N© )Ú.0Úrows     r   ú	<genexpr>zmain.<locals>.<genexpr>x   s   è ø€ Ò2˜c�3�q•6Ñ2ùs   ‚r   r$   rM   zwINSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, content) VALUES (?, ?, ?, ?, ?, ?)r%   r&   r   é   z#  Progress: {}/{} processed, {} newz.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z  New: {}, Total in DB: {}z
[{}] Done!)rK   ÚformatÚ	SITE_NAMEÚDAYSÚsqlite3Úconnectr   Úcursorr   r5   ÚlenÚexecuteÚsetÚfetchallÚ	enumeraterR   r   ÚrowcountÚcommitr   Úfetchone)ÚconnÚcÚxml_2026Ú
items_2026Úxml_2025Ú
items_2025Ú	all_itemsÚexistingÚ	new_countÚir/   rM   Útotals                r   Úmainrv   b   s1  € Ü	Ð
)×
0Ñ
0´¼DÓ
AÈÕNä�?‰?œ9¨bÔ1€DØ�‰‹€Aô 
Ð
'¨tÕ4Ü'¨Ó/€HÜ  Ó*€JÜ	Ð
×
#Ñ
#¤C¨
£OÓ
4¸DÕAä	Ð
'¨tÕ4Ü'¨Ó/€HÜ  Ó*€JÜ	Ð
×
#Ñ
#¤C¨
£OÓ
4¸DÕAà˜ZÑ'€IÜ	Ð
×
$Ñ
$¤S¨£^Ó
4¸DÕAð ‡I�IÐ@Ä9À,ÔOÜÑ2 Q§Z¡Z£\Ô2Ó2€Hà€IÜ˜YÓ'ò n‰ˆˆ4Ø�
Ñ˜xÑ'Øä˜t JÑ/Ó0ˆØ!ˆˆY‰à	�	‰	ð FÜœ $ zÑ"2°D¸±MÀ4ÈÑCWÐY]Ð^gÑYhÐiô	
ð �:‰:˜Š>Ø˜‰NˆIØ�‰Œà�‰E�R‰<˜1ÓÜÐ7×>Ñ>¸qÀ1¹uÄcÈ)ÃnÐV_Ó`ÐhlÖmð!nð$ 	‡J�J„LÜ�O‰OœI¨rÔ2×:Ñ:Ø8¼9¸,óç�hƒj�ñ€Eô 
Ð
&×
-Ñ
-¨i¸Ó
?ÀtÕLÜ	ˆ,×
Ñ
œiÓ
(°Ö5r6   Ú__main__)Ú__doc__ÚosÚsysr`   Úbs4r   ÚrequestsÚplaywright.sync_apir   Úgetenvr   r^   ÚBASE_URLr   rc   ÚargvÚisdigitÚintr_   ÚHEADERSÚSessionrD   ÚheadersÚupdater   r5   rR   rv   Ú__name__rX   r6   r   ú<module>rˆ      sÑ   ðñó
 
Û 
Û Ý Û Ý /àˆB�I‰I�kÐ#4Ó5€	Ø$€	Ø$€Ø�mÑ#€Ù˜sŸx™x›=¨1Ò,°·±¸!±×1DÑ1DÔ1F�sˆ3�8‰8�A‰;ÔÈC€àÐWÐ
X€Ø
ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ òò"ò:ò606ðf ˆzÒÙ…Fð r6   