Ë
    kŸj�  ã                   óÂ   — d Z ddlZddlmZ ddlZddlZddlZddlmc m	Z
 ddlmZ dZdZdZdZd	Z ej"                  d
d«      ZdedœZd„ Zd„ Zd„ Zedk(  r e«        yy)u`   çˆ¬å�– xiushui.gov.cn - åŸŽæŠ•é›†å›¢é€šçŸ¥å…¬å‘Š (2026-09-08 æ”¹ä¸º data_<year>.xml + x-api-key)é    N)ÚBeautifulSoup)Údatetimezhttps://www.xiushui.gov.cnz/xsctgg/ctgg/z********^^^^^^^^u'   xiushui.gov.cn-åŸŽæŠ•é›†å›¢é€šçŸ¥å…¬å‘Šz
2023-06-16Ú	SEARCH_DBz/root/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36)z
User-Agentz	x-api-keyc           	      óä  ‡— t         › t        › d| › d�}	 t        j                  |t        d¬«      }d|_        |j                  }	 t        j                  |«      }g }|j                  d«      D ]n  Šˆfd„} |d«       |d«       |d«      }
}	}|r|	sŒ%|	r0|	j                  d«      st         t        z   |	j                  d«      z   }	|j!                  ||	|
dd dœ«       Œp |S # t        $ r}t        d| › d|› �«       g cY d}~S d}~ww xY w# t        $ r] 	 t        j                  t        j                  d	d
|d¬«      «      }n+# t        $ r}t        d| › d|› �«       g cY d}~cY S d}~ww xY wY �Œw xY w)u#   æ‹‰ data_<year>.xml çš„ ITEM åˆ—è¡¨Údata_z.xmlé   ©ÚheadersÚtimeoutúutf-8z	  [WARN] z fetch: Nz^[^<]*<ú<é   )Úcountz parse: ÚITEMc                 óˆ   •— ‰j                  | «      }|�-dj                  |j                  «       «      j                  «       S dS )NÚ )ÚfindÚjoinÚitertextÚstrip)ÚtagÚeÚits     €ú'/root/gov_crawler/crawl_xiushui_ctgg.pyÚgzget_items_xml.<locals>.g,   s7   ø€ Ø—‘˜“ˆAØ45°M�2—7‘7˜1Ÿ:™:›<Ó(×.Ñ.Ó0ÐIÀrÐIó    ÚTITLEÚPUBURLÚRELTIMEÚhttpz./é
   )ÚtitleÚurlÚpublish_date)ÚBASE_URLÚ	LIST_PATHÚrequestsÚgetÚHEADERSÚencodingÚtextÚ	ExceptionÚprintÚETÚ
fromstringÚreÚsubÚiterÚ
startswithÚlstripÚappend)Úyearr#   ÚrespÚrawr   ÚrootÚitemsr   r"   ÚpubÚrelr   s              @r   Úget_items_xmlr=      sp  ø€ äˆJ”y�k  t f¨DÐ
1€CðÜ�|‰|˜C¬¸"Ô=ˆØˆŒØ�i‰iˆðÜ�}‰}˜SÓ!ˆð €EØ�i‰i˜Óò 	Mˆô	Jñ ˜G›*¡a¨£k±1°Y³<�CˆsˆÙ™CØÙ�s—~‘~ fÔ-ÜœYÑ&¨¯©°DÓ)9Ñ9ˆCØ�‰˜u¨SÀ#ÀcÀrÀ(ÑKÕLð	Mð €Løô- ò Ü�	˜$˜˜x¨ sÐ+Ô,Ø�	ûðûô
 ò ð	Ü—=‘=¤§¡¨
°C¸ÀAÔ!FÓG‰DøÜò 	Ü�I˜d˜V 8¨A¨3Ð/Ô0Ø�Iûð	úò ðús_   •/C  ÁD	 Ã 	DÃ)DÃ;DÄDÄ		E/Ä,E Ä?E/Å 	E(Å	E#ÅE(ÅE/Å#E(Å(E/Å.E/c                 óž  — t        j                  | t        d¬«      }d|_        t	        |j
                  d«      }g }dD ]z  }|j                  |«      }|sŒ|j                  d«      D ]5  }|j                  dd«      }|sŒ|j                  d	«      rŒ*t        |z   |d<   Œ7 |j                  t        |«      «        n |rdj                  |«      S y)
u   èŽ·å�–è¯¦æƒ…é¡µçš„æ­£æ–‡HTMLr   r	   r   zhtml.parser)zdiv.view.TRS_UEDITORzdiv.Article_zwzdiv#article-boxÚaÚhrefr   r    )r'   r(   r)   r*   r   r+   Ú
select_oneÚfind_allr3   r%   r5   Ústrr   )r#   r7   ÚsoupÚcontent_partsÚselÚdivr?   r@   s           r   Ú
get_detailrH   8   sÁ   € ä�<‰<˜¤W°bÔ9€DØ€D„MÜ˜Ÿ™ MÓ2€Dà€MØLò ˆØ�o‰o˜cÓ"ˆÚØ—\‘\ #Ó&ò 0�Ø—u‘u˜V RÓ(�Ú §¡°Õ 7Ü (¨4¡�A�f’Ið0ð × Ñ ¤ S£Ô*Ùðñ Ø�w‰w�}Ó%Ð%Ør   c            	      ó°  — t        j                  t        «      } | j                  «       }|j	                  d«       | j                  «        t        j                  «       j                  }t        t        d d «      }t        t        ||dz
  d«      «      }t        d|› dt        › d�«       g }|D ]9  }t        |«      }t        d|› d	t        |«      › d
�«       |j!                  |«       Œ; |j#                  d„ d¬«       d}d}	|D ]®  }
|
d   }|r
|t        k  rŒ|j	                  d|
d   f«       |j%                  «       r|	dz  }	Œ@t'        |
d   «      }|st        d|
d   d d › �«       |	dz  }	Œj|j	                  d|
d   |
d   ||t(        f«       |j*                  dkD  r|dz  }| j                  «        Œ° | j-                  «        t        d|› d|	› d�«       y )Nad  
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            title TEXT,
            page_url TEXT UNIQUE,
            content TEXT,
            publish_date TEXT,
            site_name TEXT DEFAULT '',
            crawl_time TEXT DEFAULT (datetime('now', '+8 hours')),
            similar TEXT
        )
    é   r   éÿÿÿÿu   å¹´ä»½: u	    (æˆªæ­¢ ú)z  z: z items from xmlc                 ó   — | d   xs dS )Nr$   r   © )Úxs    r   ú<lambda>zcrawl.<locals>.<lambda>j   s   €   >Ñ!2Ò!8°b€ r   T)ÚkeyÚreverser   r$   z)SELECT id FROM gov_raw WHERE page_url = ?r#   u       è­¦å‘Š: ç©ºæ­£æ–‡ r"   r   zhINSERT OR IGNORE INTO gov_raw (title, page_url, content, publish_date, site_name) VALUES (?, ?, ?, ?, ?)u   å®Œæˆ�: æ–°å¢ž u    æ�¡, è·³è¿‡ u    æ�¡)Úsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚcommitr   Únowr6   ÚintÚCUTOFF_DATEÚlistÚranger-   r=   ÚlenÚextendÚsortÚfetchonerH   Ú	SITE_NAMEÚrowcountÚclose)ÚconnÚcÚnow_yearÚcutoff_yearÚyearsÚ	items_allÚyÚrecsÚtotal_addedÚtotal_skippedÚitemr;   Úcontents                r   Úcrawlrq   N   sÜ  € Ü�?‰?œ7Ó#€DØ�‰‹€Aà‡I�Ið ô 	ð 	‡K�K„Mä�|‰|‹~×"Ñ"€HÜ”k " 1�oÓ&€KÜ”�x ¨q¡°"Ó5Ó6€EÜ	ˆH�U�G˜9¤[ M°Ð
3Ô4à€IØò ˆÜ˜QÓˆÜ��1�#�Rœ˜D›	�{ /Ð2Ô3Ø×Ñ˜Õðð ‡N�NÑ8À$€NÔGà€KØ€Màò ˆØ�>Ñ"ˆÙ�3œÒ$Øà	�	‰	Ð=ÀÀUÁ¸~ÔNØ�:‰:Œ<Ø˜QÑˆMØä˜T %™[Ó)ˆÙÜÐ*¨4°©=¸¸"Ð+=Ð*>Ð?Ô@Ø˜QÑˆMØà	�	‰	ØvØ�'‰]˜D ™K¨°#´yÐAô	
ð �:‰:˜Š>Ø˜1ÑˆKØ�‰�ð-ð0 	‡J�J„LÜ	ˆO˜K˜=¨°m°_ÀDÐ
IÕJr   Ú__main__)Ú__doc__r'   Úbs4r   rS   Úosr0   Úxml.etree.ElementTreeÚetreeÚElementTreer.   r   r%   r&   ÚXML_API_KEYrb   r[   ÚgetenvrU   r)   r=   rH   rq   Ú__name__rN   r   r   ú<module>r|      s‚   ðá fÛ Ý Û Û 	Û 	ß "Ð "Ý à'€Ø€	Ø €Ø5€	Ø€Ø
ˆ"�)‰)�KÐ!2Ó
3€ð DØñ€òò@ò,:Kðz ˆzÒÙ	…Gð r   