Ë
    ˜ÅzjÐ  ã                   óö   — d Z ddlZddlZddlZddlZddlZddlZddlmZ  ej                  ej                  j                  «       ddlmZ dZdZedz   ZdZd	Zd
ddddœZd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u×  
æ·®åŒ—å¸‚æ”¿åºœé—¨æˆ· - ä¿¡æ�¯å…¬å¼€ï¼ˆæ¿‰æºªåŽ¿æ �ç›®ï¼Œç›‘ç�£ä¿�éšœ/è§„èŒƒæ€§æ–‡ä»¶ï¼‰
https://www.huaibei.gov.cn/public/column/1981?type=4

ç«™ç‚¹ç‰¹å¾�ï¼ˆ2026-08-11 æŽ¢æµ‹ï¼‰:
- é¾™è®¯ Lonsun CMSï¼›é¡µé�¢ HTML æ˜¯å¯¼èˆªå£³ï¼Œåˆ—è¡¨ç»� Ls.ajax å¼‚æ­¥åŠ è½½
- çœŸæ•°æ�®æº�: GET /zwgk/site/label/8888?labelName=publicInfoList&type=6&siteId=4704141&catIds=4751791&pageIndex=N&pageSize=15&isJson=true
- JSON ç›´ä¾›å®Œæ•´å­—æ®µ: title / publishDate / content(å®Œæ•´æ­£æ–‡) / attachPdfName+attachPdfTitle / attachDocName+attachDocTitle / link(è¯¦æƒ…URL) / fileNum(æ–‡å�·)
- é™„ä»¶è·¯å¾„ /group1/M00/... ç›¸å¯¹ â†’ urljoin(BASE_URL) ç»�å¯¹åŒ–ï¼ˆhuaibei/sxx å�ŒåŸŸå�‡å�¯ä¸‹è½½ï¼‰
- é¡µé�¢ title=ã€Œä¿¡æ�¯å…¬å¼€_æ¿‰æºªåŽ¿äººæ°‘æ”¿åºœä¿¡æ�¯å…¬å¼€ç½‘ã€�ï¼›link æŒ‡å�‘ www.sxx.gov.cnï¼ˆæ¿‰æºªåŽ¿ç‹¬ç«‹ç«™ï¼‰
- total=22, pageCount=2ï¼ˆ2026-08-11 æŽ¢æµ‹ï¼‰ï¼Œæ•°æ�® 2021~2025

ç”¨æ³•:
  python3 crawl_huaibei_xxgk.py --pages=2    # å…¨é‡�
  python3 crawl_huaibei_xxgk.py --pages=1    # æµ‹è¯•
é    N)Úurljoin)Úpush_to_searchdbu+   æ¿‰æºªåŽ¿æ”¿åºœä¿¡æ�¯å…¬å¼€-è§„èŒƒæ€§æ–‡ä»¶zhttps://www.huaibei.gov.cnz/zwgk/site/label/8888z
2015-01-01u   æ”¿åºœå…¬å¼€zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z4https://www.huaibei.gov.cn/public/column/1981?type=4ÚXMLHttpRequestz.application/json, text/javascript, */*; q=0.01)z
User-AgentÚRefererzX-Requested-WithÚAcceptc                 óÆ   — t        j                  | xs d«      } t        j                  dd| «      } | j	                  dd«      j	                  dd«      } | j                  «       S )NÚ u   ^(?:&middot;|&nbsp;|\s|â€¢|Â·)+u   â€‹u   ï»¿)ÚhtmlÚunescapeÚreÚsubÚreplaceÚstrip)Úts    ú'/root/gov_crawler/crawl_huaibei_xxgk.pyÚclean_titler   ,   sP   € Ü�‰�a’g˜2Ó€AÜ
�‰Ð1°2°qÓ9€AØ	�	‰	�(˜BÓ×'Ñ'¨°"Ó5€AØ�7‰7‹9Ðó    c                 ó  — dddd| dddœ}t        j                  t        |t        dd	¬
«      }|j                  dk7  rg ddfS |j
                  xs d|_        	 t        j                  |j                  «      }|j                  dd«      }|j                  dd«      }g }|j                  dg «      xs g D ]4  }t        |j                  dd«      «      }|sŒ!|j                  ||dœ«       Œ6 |||fS # t        $ r g ddfcY S w xY w)u   API æ‹‰åˆ—è¡¨ï¼Œè¿”å›ž itemsÚpublicInfoListé   i�ÇG Ú4751791é   Útrue)Ú	labelNameÚtypeÚsiteIdÚcatIdsÚ	pageIndexÚpageSizeÚisJsoné   F)ÚparamsÚheadersÚtimeoutÚverifyéÈ   r   zutf-8ÚtotalÚ	pageCountÚdataÚtitler	   )r*   Úrow)ÚrequestsÚgetÚAPI_URLÚHEADERSÚstatus_codeÚapparent_encodingÚencodingÚjsonÚloadsÚtextÚ	Exceptionr   Úappend)	Úpager"   ÚrÚjr'   Ú
page_countÚitemsr+   r*   s	            r   Úfetch_list_pager=   2   s  € ð &ØØØØØØñ€Fô 	�‰”W V´WÀbÐQVÔW€AØ‡}�}˜ÒØ�1�aˆxˆØ×$Ñ$Ò/¨€A„JðÜ�J‰J�q—v‘vÓˆð �E‰E�'˜1Ó€EØ—‘�{ AÓ&€JØ€EØ�u‰u�V˜RÓ Ò& Bò 3ˆÜ˜CŸG™G G¨RÓ0Ó1ˆÙØØ�‰˜u¨SÑ1Õ2ð	3ð
 �%˜Ð#Ð#øô ò Ø�1�aˆxŠðús   ÁC. Ã.C?Ã>C?c                 ó  — | j                  dd«      xs d}g }dD ]‚  \  }}| j                  |d«      }|sŒ|j                  d«      r|nt        t        |«      }t	        | j                  |d«      «      xs |j                  d«      d   }|j                  ||dœ«       Œ„ t        «       }g }	|D ]/  }
|
d   |vsŒ|j                  |
d   «       |	j                  |
«       Œ1 |rxd	|vr/d
|vr+t        j                  dd|j                  «       «      }d|› d�}t        j                  dd|«      }t        j                  dd|«      }t        j                  dd|«      }|}|	D ]  }
|
d   |vsŒ|d|
d   › d|
d   › d�z  }Œ ||	fS )u(   æ­£æ–‡: content å®Œæ•´æ­£æ–‡ + é™„ä»¶æ®µÚcontentr	   ))ÚattachPdfNameÚattachPdfTitle)ÚattachDocNameÚattachDocTitleÚhttpú/éÿÿÿÿ)ÚnameÚurlrH   z<pz<tablez\n{2,}z</p>
<p>z<p>z</p>z<span[^>]*>z</span>z	<br\s*/?>z<p><a href="z">rG   z</a></p>)r-   Ú
startswithr   ÚBASE_URLr   Úsplitr7   ÚsetÚaddr   r   r   )r+   r?   ÚattsÚkey_nameÚ	key_titleÚpÚabs_urlr   ÚseenÚatts_uÚaÚbodys               r   Úbuild_contentrW   O   sœ  € à�g‰g�i Ó$Ò*¨€Gà€DØiò 1Ñˆ�)Ø�G‰G�H˜bÓ!ˆÙØØ—|‘| FÔ+‘!´¼À1Ó1EˆÜ˜Ÿ™ 	¨2Ó.Ó/ÒI°7·=±=ÀÓ3EÀbÑ3IˆØ�‰˜Q wÑ/Õ0ð1ô ‹5€DØ€FØò ˆØˆU‰8˜4ÒØ�H‰H�Q�u‘XÔØ�M‰M˜!Õðñ
 Ø�wÑ 8°7Ñ#:Ü—f‘f˜Y¨°W·]±]³_ÓEˆGØ˜G˜9 DÐ)ˆGä—&‘&˜¨¨WÓ5ˆÜ—&‘&˜ R¨Ó1ˆÜ—&‘&˜ r¨7Ó3ˆØ€DØò CˆØˆU‰8˜4ÒØ�l 1 U¡8 *¨B¨q°©y¨k¸ÐBÑB‰DðCð �ˆ<Ðr   c                  óx  — d } t         j                  dd  D ]M  }|j                  d«      rt        |j	                  d«      d   «      } Œ2|j                  «       sŒCt        |«      } ŒO g }t        «       }d}d}	 | r|| kD  rnít        |«      \  }}}|st        d|› d�«       nÌd}|D ]Œ  }	|	d   j                  d«      xs |	d	   }
|
|v rŒ#|	d   j                  d
«      xs dd d }|r&|t        k  rt        d|› d|› dt        › d�«       d} n)|j                  |
«       |j                  |	«       |dz  }ŒŽ t        d|› dt        |«      › d|› dt        |«      › d|› d�«       |dk(  rn|dz  }Œõt        d| › dt        |«      › �«       g }t        |«      D ]~  \  }}	|	d   }t        |«      \  }}|j                  d
«      xs dd d }|j                  d«      xs d}|j                  t         |	d	   |||||rt#        j$                  |d¬«      nddœ«       Œ€ t        dt        |«      › �«       t'        |t(        «       y )Né   z--pages=ú=r   z[Pz] empty, stopr+   Úlinkr*   ÚpublishDater	   é
   z] z
 < CUTOFF z, stopi?B z] got z items, new z, total z (api total=ú)z[AutoPg] max_pages=z, collected F)Úensure_asciiz[])Ú	site_namer*   Úpub_dater?   Ú
source_urlrH   Úattachmentsz[Detail] done )ÚsysÚargvrI   ÚintrK   ÚisdigitrL   r=   Úprintr-   ÚCUTOFFrM   r7   ÚlenÚ	enumeraterW   Ú	SITE_NAMEr3   Údumpsr   ÚCATEGORY)Ú	max_pagesrU   Ú	all_itemsrS   r8   r'   r<   r;   ÚnewÚitÚkeyÚpubÚvalidÚir+   r?   rN   r[   s                     r   Úmainrw   q   sp  € Ø€IÜ�X‰X�a�bˆ\ò ˆØ�<‰<˜
Ô#Ü˜AŸG™G C›L¨™OÓ,‰IØ�Y‰Y�[Ü˜A›‰Ið	ð
 €IÜ‹5€DØ€DØ€EØ
Ù˜ 	Ò)ØÜ#2°4Ó#8Ñ ˆˆu�jÙÜ�B�t�f˜MÐ*Ô+ØØˆØò 	ˆBà�U‘)—-‘- Ó'Ò6¨2¨g©;ˆCØ�d‰{ØØ�e‘9—=‘= Ó/Ò5°2°s¸Ð;ˆCÙ�sœV’|Ü˜˜4˜&  3 % z´&°¸Ð@ÔAØ�ÙØ�H‰H�SŒMØ×Ñ˜RÔ Ø�1‰H‰Cð	ô 	��4�&˜œs 5›z˜l¨,°s°e¸8ÄCÈ	ÃNÐCSÐS_Ð`eÐ_fÐfgÐhÔiØ�6Š>ØØ�‰	ˆð1 ô2 
Ð 	˜{¨,´s¸9³~Ð6FÐ
GÔHà€EÜ˜9Ó%ò ‰ˆˆ2Ø�‰iˆÜ% cÓ*‰ˆ�Ø�w‰w�}Ó%Ò+¨¨S¨bÐ1ˆØ�w‰w�v‹Ò$ "ˆØ�‰Ü"Ø˜‘[ØØØØÙCGœ4Ÿ:™: d¸Õ?ÈTñ
õ 	ðô 
ˆNœ3˜u›:˜,Ð
'Ô(Ü�UœHÕ%r   Ú__main__)Ú__doc__r   rd   r3   r
   Úurllib3r,   Úurllib.parser   Údisable_warningsÚ
exceptionsÚInsecureRequestWarningÚcrawler_libr   rl   rJ   r.   ri   rn   r/   r   r=   rW   rw   Ú__name__© r   r   ú<module>r‚      sŸ   ðñó  
Û 
Û Û Û Û Ý  à €× Ñ ˜×+Ñ+×BÑBÔ Cå (à9€	Ø'€Ø
Ð,Ñ
,€Ø	€Ø€ð @ØEØ(Ø>ñ	€òò$ò: òD6&ðp ˆzÒÙ…Fð r   