Ë
    ¸¬uj’%  ã                   óà  — d dl Z 	 d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	 dZ
dZdZ e j                  dd«      Zd	Ze› d
�ZddddddddœZdeez   dœZ ej&                  «        e	d¬«      z
  j)                  d«      Zd„ Zd„ Zd„ Zd#d„Zd„ Zd$d„Zedk(  rRdej:                  v ZdZej:                  d d D ]%  Z e jC                  «       r
 e"e «      Z ne d!k(  sŒ#d Z n  eee¬"«       yy)%é    N)Úurljoin)ÚdatetimeÚ	timedeltau"   æŸ¯åŸŽåŒºäººæ°‘æ”¿åºœ-å…¬å‘Šå…¬ç¤ºzhttps://www.kecheng.gov.cnz/col/col1229068908/index.htmlÚ	SEARCH_DBz/root/search.dbé   z7/api-gateway/jpaas-publish-server/front/page/build/unitÚ3017Ú
1229068908ÚbulidstaticÚcolumnu   å½“å‰�æ �ç›®_listÚnullÚ8LeihoYLP84FLuUsPt4wo)ÚwebIdÚpageIdÚ	parseTypeÚpageTypeÚtagIdÚeditTypeÚtplSetIdz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36)z
User-AgentÚRefereriG  )Údaysz%Y-%m-%dc                 óÂ  — t        t        «      }t        j                  |ddœd¬«      |d<   	 | j	                  t
        |t        d¬«      }|j                  dk7  ry	|j                  «       }|j	                  d
«      sy	|d   d   }t        j                  d|«      }|rt        |j                  d«      «      nd}||fS # t        $ r}t        d|› �«       Y d}~y	d}~ww xY w)z=Fetch one page of list via GET API with paramJson pagination.é   )ÚpageNoÚpageSizeF)Úensure_asciiÚ	paramJsoné   )ÚparamsÚheadersÚtimeoutéÈ   )Nr   ÚsuccessÚdataÚhtmlzcount="(\d+)"é   r   z    API error: N)ÚdictÚ
API_PARAMSÚjsonÚdumpsÚgetÚAPI_URLÚHEADERSÚstatus_codeÚreÚsearchÚintÚgroupÚ	ExceptionÚprint)	ÚsessionÚpager   Úrr#   r$   ÚmÚtotalÚes	            ú"/root/gov_crawler/crawl_kecheng.pyÚ
fetch_listr;   '   sË   € ä”*Ó€FÜŸ*™*°À"Ñ%EÐTYÔZ€Fˆ;ÑðØ�K‰Kœ¨¼ÈˆKÓLˆØ�=‰=˜CÒØØ�v‰v‹xˆØ�x‰x˜	Ô"ØØ�F‰|˜FÑ#ˆä�I‰IÐ&¨Ó-ˆÙ#$”�A—G‘G˜A“J”¨!ˆØ�Uˆ{ÐøÜò Ü� ˜sÐ#Ô$Üûðús#   ®,B= Á!B= Á=?B= Â=	CÃCÃCc                 óˆ  — g }t        j                  dt         j                  «      }|j                  | «      D ]‡  }|j	                  d«      }|j                  d«      st        t        |«      }|j	                  d«      j                  «       }|j	                  d«      j                  «       }|j                  |||dœ«       Œ‰ |S )zzExtract article items from list HTML.
    Format: <li><a href="..." title="...">title</a><span>2026-06-12</span></li>
    zG<li>.*?<a[^>]*href="([^"]+)"[^>]*title="([^"]*)".*?<span>([^<]+)</span>r%   Úhttpé   é   )ÚurlÚtitleÚdate)
r.   ÚcompileÚDOTALLÚfinditerr1   Ú
startswithr   ÚBASEÚstripÚappend)r$   ÚitemsÚpatternr7   r@   rA   Údate_strs          r:   Ú
parse_listrM   <   sª   € ð €EÜ�j‰jØRÜ
�	‰	ó€Gð ×Ñ˜dÓ#ò EˆØ�g‰g�a‹jˆØ�~‰~˜fÔ%Üœ$ Ó$ˆCØ—‘˜“
× Ñ Ó"ˆØ—7‘7˜1“:×#Ñ#Ó%ˆØ�‰˜S¨5¸(ÑCÕDðEð €Ló    c                 óh   — 	 | j                  |t        d¬«      }d|_        |j                  S #  Y y xY w)Nr   )r   r    zutf-8)r*   r,   ÚencodingÚtext)r4   r@   r6   s      r:   Ú
fetch_pagerR   O   s5   € ðØ�K‰K˜¤W°bˆKÓ9ˆØˆŒ
Ø�v‰vˆøðÙús   ‚*- ­1c                 ó  — d}t        j                  d| «      }|r|j                  d«      j                  «       }|s[t        j                  d| t         j                  «      }|r4t        j
                  dd|j                  d«      «      j                  «       }|s|}d}t        j                  d| «      }|r|j                  d«      }|s)t        j                  d| «      }|r|j                  d«      }d}| j                  d«      }|d	k\  rÎ| j                  d|d
z   «      }|d	k\  r|t        d«      z   }n|t        d«      z   }d	|}
}	|
t        | «      k  r…| |
|
dz    dk(  r%| j                  d|
dz   «      }||
kD  r|dz   n|
dz   }
Œ>| |
|
dz    dk(  r|	d	k(  r| ||
 }n?|	dz  }	|
dz  }
Œ_| |
   dk(  r| |
dz   |
dz    dv r|	dz  }	|
dz  }
Œ|
dz  }
|
t        | «      k  rŒ…|r®t        j
                  dd|t         j                  t         j                  z  ¬«      }t        j
                  dd|t         j                  t         j                  z  ¬«      }t        j
                  dd|«      }t        j
                  dd|«      }|j                  «       }|||fS )z.Extract title, date, content from detail page.Ú z3<meta[^>]*name="ArticleTitle"[^>]*content="([^"]*)"r%   z,<div[^>]*class="main_title"[^>]*>(.*?)</div>z<[^>]+>z=<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})u,   å�‘å¸ƒæ—¶é—´[ï¼š:]\s*(\d{4}-\d{1,2}-\d{1,2})z<div class="main_section">r   r   é   z<!--z-->r?   é   z</div>ú<)ÚdivÚsecz<script[^>]*>.*?</script>)Úflagsz<style[^>]*>.*?</style>z\s*style="[^"]*"z
<!--.*?-->)	r.   r/   r1   rH   rD   ÚsubÚfindÚlenÚI)r$   Úfallback_titlerA   r7   rL   ÚcontentÚidxÚidx2ÚstartÚdepthÚir9   s               r:   Úparse_detailrf   X   s†  € ð €EÜ
�	‰	ÐHÈ$ÓO€AÙØ—‘˜“
× Ñ Ó"ˆÙä�I‰IÐEÀtÌRÏYÉYÓWˆÙÜ—F‘F˜: r¨1¯7©7°1«:Ó6×<Ñ<Ó>ˆEÙØˆð €HÜ
�	‰	ÐRÐTXÓY€AÙØ—7‘7˜1“:ˆÙÜ�I‰IÐEÀtÓLˆÙØ—w‘w˜q“zˆHð €GØ
�)‰)Ð0Ó
1€CØ
ˆa‚xà�y‰yÐ5°s¸R±xÓ@ˆØ�1Š9Øœ3Ð;Ó<Ñ<‰Eàœ#Ð:Ó;Ñ;ˆEà�eˆqˆØ”#�d“)ŠmØ�A�a˜‘cˆ{˜fÒ$Ø—I‘I˜e Q q¡SÓ)�Ø˜qš5�A�a’C a¨¡c�ØØ�A�a˜‘cˆ{˜hÒ&Ø˜A’:Ø" 5¨˜m�GØØ˜‘
�Ø�Q‘�ØØ�A‰w˜#Š~ $ q¨¡s¨1¨Q©3 -°>Ñ"AØ˜‘
�Ø�Q‘�ØØ�‰FˆAð! ”#�d“)‹mñ$ Ü—&‘&Ð5°r¸7Ì"Ï)É)ÔVX×VZÑVZÑJZÔ[ˆÜ—&‘&Ð3°R¸ÌÏ	É	ÔTV×TXÑTXÑHXÔYˆÜ—&‘&Ð,¨b°'Ó:ˆÜ—&‘&˜¨¨GÓ4ˆØ—-‘-“/ˆà�(˜GÐ#Ð#rN   c                 óè   — d}d}|D ]Z  }	 | j                  d|d   d d |d   |d   |d   |d   |d	   |d
   |d   |d   |d   f
«       | j                  dkD  r|dz  }n|dz  }Œ\ ||fS #  |dz  }Y ŒjxY w)Nr   z»INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags)
                VALUES (?,?,?,?,?,?,?,?,?,?)Ú	site_namer!   Ú
source_urlÚpage_urlrA   Úpublish_dateÚsummaryr`   ÚstatusÚcategoryÚtagsr%   )ÚexecuteÚtotal_changes)ÚdbÚrecordsÚinsertedÚskippedÚitems        r:   Ú
push_to_dbrw   š   sÊ   € Ø€HØ€GØò ˆð	Ø�J‰Jð0ð ˜Ñ% d sÐ+Ø˜Ñ&Ø˜Ñ$Ø˜‘MØ˜Ñ(Ø˜‘OØ˜‘OØ˜‘NØ˜Ñ$Ø˜‘Lðô	ð" ×Ñ !Ò#Ø˜A‘‘à˜1‘�øð-ð2 �WÐÐøð	Ø�q‰LŠGús   ‹AA(Á(A1c                 ó:  ‡— | €t         } t        j                  t        «      }|j	                  d«       t        d„ |j	                  dt        f«      j                  «       D «       «      Št        dt        ‰«      › �«       t        j                  «       }|j                  j                  t        «       t        dd¬«       t        |d«      \  }}|st        d	«       y t!        |«      }t#        | |r|d
z   dz  n| «      }t        dt        |«      › d|› d|› d�«       g }|r|d   d   nd}	|r |	‰v rt        d«       |j%                  «        y |j'                  |«       t)        d|dz   «      D ]º  }
t        d|
› d�d¬«       t        ||
«      \  }}|st        d	«        n‹t!        |«      }t        t        |«      › d�«       |s ne|rPt+        ˆfd„|D «       «      }|rt        d«        n@|D �cg c]  }|d   ‰vsŒ|‘Œ }}|s n%|j'                  |«       Œª|j'                  |«       Œ¼ |D �cg c]  }|d   t,        k\  s|d   rŒ|‘Œ }}|st        d«       |j%                  «        y t        dt        |«      › d�«       g }t/        |«      D ]›  \  }}t1        ||d   «      }|sŒt3        ||d   «      \  }}}|j5                  t        |d   d d |d   |xs |d   d d |xs |d   d d  |xs |d   d d |d!ddd"œ
«       |dz   d#z  dk(  sŒ~t        d$|dz   › d%t        |«      › d&�«       Œ� t7        ||«      \  }}|j9                  «        |j	                  d't        f«       |j9                  «        |j%                  «        t        d(|› d)|› �«       y c c}w c c}w )*NzPRAGMA journal_mode=WALc              3   ó&   K  — | ]	  }|d    –— Œ y­w)r   N© )Ú.0r6   s     r:   ú	<genexpr>zrun.<locals>.<genexpr>À   s   è ø€ ò àð 	
ˆ!�ñùs   ‚z.SELECT page_url FROM gov_raw WHERE site_name=?z  Known in DB: z  Fetching page 1...ú )Úendr%   ÚFAILé   r   zOK - z items, z pages (total: ú)r   r@   rT   z(  No new data (newest URL already in DB)r>   z  Page z...z itemsc              3   ó,   •K  — | ]  }|d    ‰v –— Œ y­w)r@   Nrz   )r{   rv   Úknowns     €r:   r|   zrun.<locals>.<genexpr>ì   s   øè ø€ ÒH°T˜D ™K¨5Ô0ÑHùs   ƒz    All known, stoppingrB   z  No items to processz  Fetching z details...rA   iô  é
   Úactive)
rh   ri   rj   rA   rk   rl   r`   rm   rn   ro   é   z    [ú/ú]zÔINSERT INTO gov_search(rowid,title,site_name,summary)
        SELECT r.id, r.title, r.site_name, r.summary
        FROM gov_raw r
        WHERE r.id NOT IN (SELECT rowid FROM gov_search)
        AND r.site_name=?z  Done: new=z, skip=)Ú	MAX_PAGESÚsqlite3Úconnectr   rp   ÚsetÚ	SITE_NAMEÚfetchallr3   r]   ÚrequestsÚSessionr   Úupdater,   r;   rM   ÚminÚcloseÚextendÚrangeÚallÚCUTOFFÚ	enumeraterR   rf   rI   rw   Úcommit)Ú	max_pagesÚincrementalrr   r4   r$   r8   rJ   Útotal_pagesÚ	all_itemsÚ
newest_urlr5   Ú_Ú
page_itemsÚ	all_knownre   Ú	new_itemsÚresultsrv   rA   rL   r`   rt   ru   rƒ   s                          @r:   Úrunr¤   ¹   s»  ø€ ØÐÜˆ	ä	�‰œÓ	#€BØ‡J�JÐ(Ô)äñ à—‘Ø<¼y¸ló
ç
‰(‹*ô	ó €Eô 
ˆOœC ›J˜<Ð
(Ô)ä×ÑÓ €GØ‡O�O×Ñœ7Ô#ô 
Ð  sÕ+Ü˜W aÓ(�K€Dˆ%ÙÜˆfŒØÜ�tÓ€EÜ�i±u %¨"¡*°Ò!3À)ÓL€KÜ	ˆE”#�e“*�˜X k ]°/À%ÀÈÐ
JÔKð €IÙ$)��q‘˜%’¨r€JÙ�z UÑ*ÜÐ8Ô:Ø
�‰Œ
Øà×Ñ�UÔô �a˜ q™Ó)ò )ˆÜ�˜�v˜SÐ! sÕ+Ü˜W dÓ+‰ˆˆaÙÜ�&ŒMÙÜ Ó%ˆ
Ü”�Z“Ð! Ð(Ô)ÙÙáÜÓH¸ZÔHÓHˆIÙÜÐ/Ô1ÙØ$.ÖH˜q°!°E±(À%Ò2GšÐHˆIÐHÙÙØ×Ñ˜ZÕ(à×Ñ˜ZÕ(ð+)ð0 &ÖN�q¨¨6©´fÒ)<ÀAÀfÃI’ÐN€IÐNáÜÐ%Ô&Ø
�‰Œ
Øä	ˆKœ˜I›Ð' {Ð
3Ô4à€GÜ˜YÓ'ò 3‰ˆˆ4Ü˜' 4¨¡;Ó/ˆÙØÜ#/°°d¸7±mÓ#DÑ ˆˆx˜Ø�‰ä&Ø" 5™k¨$¨3Ð/Ø  ™KØÒ0 4¨¡=°$°3Ð7Ø!)Ò!9¨T°&©\¸3¸BÐ ?Ø!Ò2 T¨'¡]°D°SÐ9Ø"Ø"ØØñô	
ð �‰E�R‰<˜1ÓÜ�E˜!˜A™#˜˜a¤ I£Ð/¨qÐ1Õ2ð)3ô, # 2 wÓ/Ñ€HˆgØ‡I�I„Kð ‡J�Jð	ô
 
ˆôð ‡I�I„KØ‡H�H„Jä	ˆL˜˜
 '¨'¨Ð
3Õ4ùòo Iùò Os   Ç-NÇ:NÈ-NÉNÚ__main__z--incrementalr%   z--test)rš   r›   )rT   )NF)#ÚosÚsysr.   rŠ   r(   r�   Úurllib.parser   r   r   r�   rG   ÚCOLUMNÚgetenvr   r‰   r+   r'   r,   ÚnowÚstrftimer—   r;   rM   rR   rf   rw   r¤   Ú__name__Úargvr›   ÚmpÚaÚisdigitr0   rz   rN   r:   ú<module>r²      s2  ðã 	ð÷ × Û Ý  ß (à0€	Ø#€Ø	(€ØˆB�I‰I�kÐ#4Ó5€	Ø€	àˆFÐIÐ
J€àØØØØ ØØ'ñ€
ð QØ�f‰}ñ€ð
 ˆ(�,‰,‹.™9¨'Ô2Ñ
2×	<Ñ	<¸ZÓ	H€òò*ò&ó?$òDó>n5ðb ˆzÒØ! S§X¡XÐ-€KØ	€BØ�X‰X�a�bˆ\ò ˆØ�9‰9Œ;Ù�Q“ˆBÙØ�‹=ØˆBÙðñ �" +Ö.ð rN   