Ë
    ¸¬ujL  ã                   ó²  — d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
mZ ddlmZ ddlZ ej                  ej                   j"                  «       dZedz   ZdZdZd	Zd
Z ej0                  dd«      Zdedz   dœZ e	j6                  «       Zej:                  j=                  e«       de_        d„ Z dd„Z!d„ Z"efd„Z#d„ Z$d„ Z%e&dk(  r e%«        yy)uÿ   
å¥‡å�°åŽ¿äººæ°‘æ”¿åºœ - é€šçŸ¥å…¬å‘Š
URL: https://www.xjqt.gov.cn/p232/tzgg.html
CMS: BZRK CMS (REST API)
List: GET /content/page?channelIds=9086&page=N&size=15&orderBy=27
Detail: Content (txt) embedded in API response - no separate detail fetch needed!
é    N)ÚBeautifulSoup)Úurljoinzhttps://www.xjqt.gov.cnz/content/pageu"   å¥‡å�°åŽ¿äººæ°‘æ”¿åºœ-é€šçŸ¥å…¬å‘Šu   é€šçŸ¥å…¬å‘Šu   æ–°ç–†é   Ú	SEARCH_DBz/root/search.dbz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36z/p232/tzgg.html)z
User-AgentÚRefererFc                 ó<   — t        | t        j                  d¬«       y )NT)ÚfileÚflush)ÚprintÚsysÚstderr)Úmsgs    ú/root/gov_crawler/crawl_xjqt.pyÚlogr   !   s   € Ü	ˆ#”C—J‘J dÖ+ó    c                 óH  — d| |ddœ}	 t         j                  t        |d¬«      }d|_        |j	                  «       }|j                  d«      dk(  r|j                  d	i «      S t        d
|j                  d«      › �«       y# t        $ r}t        d|› �«       Y d}~yd}~ww xY w)uB   è°ƒç”¨APIèŽ·å�–åˆ—è¡¨æ•°æ�®ï¼Œæ­£æ–‡å†…å®¹å·²åŒ…å�«åœ¨txtå­—æ®µä¸­Ú9086é   )Ú
channelIdsÚpageÚsizeÚorderByé   )ÚparamsÚtimeoutzutf-8ÚcodeéÈ   Údatau     [WARN] APIè¿”å›žç �: Nu     [ERR] APIè¯·æ±‚å¤±è´¥: )ÚsessionÚgetÚAPI_URLÚencodingÚjsonr   Ú	Exception)r   r   r   Úrr   Úes         r   Ú	fetch_apir'   %   s�   € à"¨D¸$È2ÑN€Fð
Ü�K‰Kœ¨¸ˆKÓ;ˆØˆŒ
Ø�v‰v‹xˆØ�8‰8�FÓ˜sÒ"Ø—8‘8˜F BÓ'Ð'ÜÐ% d§h¡h¨vÓ&6Ð%7Ð8Ô9ØøÜò ÜÐ'¨ sÐ+Ô,Üûðús   ‰AB  Á"B  Â 	B!Â	BÂB!c                 ó´  — | syt        | d«      }g }|j                  d«      D ]V  }|j                  dd¬«      }|sŒt        |j	                  d«      «      dkD  sŒ7|j                  |j	                  d«      «       ŒX |j                  d«      D ]"  }t        |«      }|j                  d	|› d
�«       Œ$ |rdj                  |«      S |j                  dd¬«      }|S )u7   ä»ŽHTMLæ­£æ–‡ä¸­æ��å�–çº¯æ–‡æœ¬ï¼Œä¿�ç•™æ®µè�½å’Œè¡¨æ ¼Ú úhtml.parserÚpT©Ústripu   Â  é   Útableu
   
[è¡¨æ ¼]
u
   
[/è¡¨æ ¼]z

ú )r   Úfind_allÚget_textÚlenr-   ÚappendÚstrÚjoin)Ú	html_textÚsoupÚpartsr+   Úp_textÚtblÚtbl_htmlÚtexts           r   Úextract_text_from_htmlr>   5   sÓ   € áØÜ˜ MÓ2€Dà€Eà�]‰]˜3Óò 2ˆØ—‘˜B d�Ó+ˆâ”c˜&Ÿ,™, yÓ1Ó2°QÓ6Ø�L‰L˜Ÿ™ iÓ0Õ1ð	2ð �}‰}˜WÓ%ò ;ˆÜ�s“8ˆØ�‰�| H :¨[Ð9Õ:ð;ñ Ø�{‰{˜5Ó!Ð!à�=‰=˜ Dˆ=Ó)€DØ€Kr   c                 óü  — | sg S t        | d«      }g }|j                  dd¬«      D ]Ó  }|d   }|j                  d¬«      }t        j                  d|t        j
                  «      r%t        ||«      }|j                  |xs d|d	œ«       Œdt        j                  d
|t        j
                  «      sŒŠt        j                  d|t        j
                  «      sŒ°t        ||«      }|j                  |xs d|d	œ«       ŒÕ |S )u"   ä»ŽHTMLæ­£æ–‡ä¸­æ��å�–é™„ä»¶é“¾æŽ¥r*   ÚaT)ÚhrefrA   r,   z&\.(pdf|doc|docx|xls|xlsx|zip|rar|txt)$u   é™„ä»¶©ÚnameÚurlz/upload|/cms/z"\.(pdf|doc|docx|xls|xlsx|zip|rar)$)r   r1   r2   ÚreÚsearchÚIr   r4   )r7   Úbase_urlr8   Úattachmentsr@   rA   ÚfnameÚfull_urls           r   Úextract_attachmentsrL   O   sä   € áØˆ	Ü˜ MÓ2€DØ€KØ�]‰]˜3 Tˆ]Ó*ò MˆØ�‰yˆØ—
‘
 �
Ó&ˆÜ�9‰9Ð>ÀÄbÇdÁdÔKÜ˜x¨Ó.ˆHØ×Ñ¨Ò(9°À(ÑKÕLÜ�Y‰YÐ'¨¬r¯t©tÕ4¼¿¹ÐChÐjnÔpr×ptÑptÕ9uÜ˜x¨Ó.ˆHØ×Ñ¨Ò(9°À(ÑKÕLðMð Ðr   c           
      óv  — 	 t        j                  t        d¬«      }| j                  d«      xs dd d }| j                  d«      xs dd d }| j                  d«      xs d}| j                  d	«      xs dd d
 }| j                  d«      xs dd d }t	        j
                  | j                  d«      xs g d¬«      }|j                  d|f«      j                  «       }|r)|j                  d|d   f«       |j                  d|f«       |j                  d|||||||f«       |j                  d«      j                  «       d   }	|r|d d n|d d }
|j                  d|	|||
f«       |j                  «        |j                  «        t        d|d d › �«       y# t        $ r}t        d|› �«       Y d }~yd }~ww xY w)Né
   )r   Útitler)   iô  Úpage_urliè  ÚcontentÚpublish_dateé   Ú	site_nameÚunknownéd   rI   F)Úensure_asciiz,SELECT rowid FROM gov_raw WHERE page_url = ?z&DELETE FROM gov_search WHERE rowid = ?r   z&DELETE FROM gov_raw WHERE page_url = ?z�INSERT INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?)zSELECT last_insert_rowid()zWINSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)u     âœ… r   Tz  [ERR] DB import failed: )Úsqlite3ÚconnectÚDB_PATHr    r#   ÚdumpsÚexecuteÚfetchoneÚcommitÚcloser   r$   )ÚrecordÚdbrO   rP   rQ   rR   rT   Úattachments_strÚoldÚ	new_rowidÚsummaryr&   s               r   Úimport_to_dbrf   a   s¶  € ðÜ�_‰_œW¨bÔ1ˆØ—‘˜GÓ$Ò*¨¨D¨SÐ1ˆØ—J‘J˜zÓ*Ò0¨b°%°4Ð8ˆØ—*‘*˜YÓ'Ò-¨2ˆØŸ
™
 >Ó2Ò8°b¸#¸2Ð>ˆØ—Z‘Z Ó,Ò9°	¸4¸CÐ@ˆ	ÜŸ*™* V§Z¡Z°Ó%>Ò%DÀ"ÐSXÔYˆà�j‰jÐGÈ(ÈÓU×^Ñ^Ó`ˆÙØ�J‰JÐ?À#ÀaÁ&ÀÔKØ�J‰JÐ?À(ÀÔMà
�
‰
ð ^Ø�H˜g |°YÀÈ/ÐZô	
ð —J‘JÐ;Ó<×EÑEÓGÈÑJˆ	Ù#*�'˜$˜3‘-°°d°s°ˆØ
�
‰
ÐlØ˜u i°Ð9ô	;à
�	‰	ŒØ
�‰Œ
Üˆf�U˜3˜B�Z�LÐ!Ô"ØøÜò ÜÐ(¨¨Ð,Ô-Üûðús   ‚FF Æ	F8Æ F3Æ3F8c                  óÎ  — dd l } | j                  «       }|j                  dt        dd¬«       |j	                  «       }d}t        d|j                  dz   «      D �]ó  }t        d|› d�«       t        |t        ¬	«      }|st        d
«        �nÃ|j                  dg «      }|st        d«        �n¡t        dt        |«      › d�«       t        |d«      D �]v  \  }}|j                  dd«      j                  «       }	|j                  dd«      xs |j                  dd«      }
|j                  d«      xs dd d }|j                  d«      xs d}t        d|› dt        |«      › d|	d d › �«       t        |«      }t        |«      }|s€|r~t!        |d«      }|j#                  d«      }|r_dt        |«      › d�}|D ]K  }|j                  dd«      }|sŒt%        t&        |«      }|j)                  |j                  dd «      |d!œ«       ŒM |sd"|
› �}|	|
|||t*        t,        t.        d#œ}t1        |«      }|r|dz  }t3        j4                  d$«       �Œy �Œö t        d%t*        › d&|› d�«       y )'Nr   z--pagesé   u1   çˆ¬å�–å‰�Né¡µï¼ˆæ¯�é¡µ15æ�¡ï¼Œé»˜è®¤5é¡µ=75æ�¡ï¼‰)ÚtypeÚdefaultÚhelpé   u   ðŸ“„ åˆ—è¡¨[page=ú])r   r   u     â†’ æ— æ•°æ�®ï¼Œç»“æ�ŸrQ   u     â†’ æ— æ›´å¤šæ•°æ�®u     â†’ u    æ�¡rO   r)   ÚurlWholerD   ÚreleaseTimerN   Útxtz  (ú/z) é(   r*   Úimgu   [æœ¬å…¬å‘Šä¸ºå›¾ç‰‡æ ¼å¼�ï¼Œå…±u
   å¼ é™„å›¾]ÚsrcÚaltu   æ­£æ–‡é™„å›¾rB   u!   [æ— æ­£æ–‡å†…å®¹] åŽŸæ–‡é“¾æŽ¥ï¼š)rO   rP   rR   rQ   rI   rT   ÚcolumnÚprovinceg333333Ó?u   
âœ… u    çˆ¬å�–å®Œæˆ�ï¼Œå…±å…¥åº“ )ÚargparseÚArgumentParserÚadd_argumentÚintÚ
parse_argsÚrangeÚpagesr   r'   ÚPER_PAGEr    r3   Ú	enumerater-   r>   rL   r   r1   r   ÚBASEr4   ÚSITEÚCOLUMNÚPROVINCErf   ÚtimeÚsleep)rx   ÚparserÚargsÚcountr   r   ÚitemsÚidxÚitemrO   rD   Úrelease_timeÚtxt_htmlrQ   rI   r8   Úimgsrs   rt   Úfull_srcr`   Úoks                         r   Úmainr’   �   st  € ÛØ×$Ñ$Ó&€FØ
×Ñ˜	¬°QØPð ô Rà×ÑÓ€Dà€Eä�a˜Ÿ™ a™Ó(ó 7ˆÜÐ ˜v QÐ'Ô(Ü˜d¬Ô2ˆÙÜÐ*Ô,Úà—‘˜ BÓ'ˆÙÜÐ'Ô)ÚÜˆf”S˜“Z�L Ð%Ô&ä" 5¨!Ó,ó *	‰IˆC�Ø—H‘H˜W bÓ)×/Ñ/Ó1ˆEØ—(‘(˜: rÓ*ÒA¨d¯h©h°u¸bÓ.AˆCØ ŸH™H ]Ó3Ò9°r¸3¸BÐ?ˆLØ—x‘x “Ò,¨"ˆHä�#�c�U˜!œC ›J˜< r¨%°°¨*¨Ð6Ô7ô -¨XÓ6ˆGô .¨hÓ7ˆKñ ™xÜ$ X¨}Ó=�Ø—}‘} UÓ+�ÙØ ?ÄÀDÃ	¸{È*ÐU�GØ#ò j˜Ø!Ÿg™g e¨RÓ0˜ÚÜ'.¬t°SÓ'9˜HØ'×.Ñ.¸¿¹ÀÀ~Ó8VÐ_gÑ/hÕið	jñ Ø=¸c¸UÐC�ð ØØ ,Ø"Ø*Ü!Ü Ü$ñ	ˆFô ˜fÓ%ˆBÙØ˜‘
�Ü�J‰J�sŽOòU*	ð7ôr ˆ&”�Ð0°°°tÐ<Õ=r   Ú__main__)rl   r   )'Ú__doc__r   ÚosrE   r#   r…   rX   ÚhtmlÚhtml_modÚrequestsÚbs4r   Úurllib.parser   Úurllib3Údisable_warningsÚ
exceptionsÚInsecureRequestWarningr�   r!   r‚   rƒ   r„   r   ÚgetenvrZ   ÚHEADERSÚSessionr   ÚheadersÚupdateÚverifyr   r'   r>   rL   rf   r’   Ú__name__© r   r   ú<module>r§      sñ   ðñ÷ :× 9× 9Ó 9Û Ý Ý  Û Ø €× Ñ ˜×+Ñ+×BÑBÔ Cà €Ø
�Ñ
 €Ø+€Ø	€Ø€Ø€Ø
ˆ"�)‰)�KÐ!2Ó
3€ð QØÐ'Ñ'ñ€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ Ø€„ò,óò ð4 -1ó ò$ò@B>ðJ ˆzÒÙ…Fð r   