Ë
    ¡Š£jo&  ã                   ó`  — d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl	m
Z
 dZdZdZdd	d
dœZ ej                  ej                   j#                  e«      d¬«        ej$                  «       Zej(                  j+                  e«       h d£Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  �rXdej<                  v Ze�r? e d«       eZ!ejE                  e!d¬«      Z#de#_$         ee#jJ                  «      Z& e d e'e&«      › d�«        e(edd¬«      5 Z)e&D ]Ê  Z*	 ejE                  e*d   d¬«      Z#de#_$         ee*d   e#jJ                  «      Z+e+jE                  d «      xs e*d!   Z,e,e*d   e+d"   xs e*d#   e+d$   e+d%   re+d%   dd& nd'd(d)e+d*   r ejZ                  e+d*   d+¬,«      nd'd-œZ.e)j_                   ejZ                  e.d+¬,«      d.z   «        ejd                  d1«       ŒÌ 	 ddd«        e d2 e'e&«      › d3�«       y e«        yy# e0$ rZ1 e d/e*d   › d0e1› �«       Y dZ1[1ŒTdZ1[1ww xY w# 1 sw Y   ŒHxY w)4u¥  
Crawl ç›˜é”¦è¾½æ»¨æ²¿æµ·ç»�æµŽæŠ€æœ¯å¼€å�‘åŒº - é€šçŸ¥å…¬å‘Š
https://ldwxq.panjin.gov.cn/13775/
Pagination: list-2.html ... list-37.html
List: <ul class="Q25_ListContlist"><li><a href="..." title="å®Œæ•´æ ‡é¢˜"><span>æ ‡é¢˜</span><span>æ—¥æœŸ</span></a></li>
Detail: <div class="Q25_artcont">content HTML</div>
Detail title: <p class="Q25_articleconttitle">æ ‡é¢˜</p>
Detail date: <span>å�‘å¸ƒæ—¥æœŸï¼š2026-07-03</span>
é    N)ÚBeautifulSoup)Úurljoinz"https://ldwxq.panjin.gov.cn/13775/zhttps://ldwxq.panjin.gov.cnz,/root/gov_crawler/output/panjin_notice.jsonlzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-LanguageT)Úexist_ok>)   ÚpÚddÚdlÚdtÚh1Úh2Úh3Úh4Úh5Úh6ÚhrÚliÚolÚtdÚthÚtrÚulÚdivÚnavÚpreÚformÚmainÚasideÚtableÚtbodyÚtfootÚtheadÚcenterÚdialogÚfigureÚfooterÚheaderÚhgroupÚaddressÚarticleÚcaptionÚdetailsÚsectionÚfieldsetÚ
blockquoteÚ
figcaptionc                 óÒ   ‡‡‡— | €yddl }ddlmŠ g Šˆˆˆfd„Š ‰| «       dj                  ‰«      }|j	                  dd|«      }|j	                  dd	|«      }|j                  «       S )
ul   å�—çº§è¾¹ç•Œå‡ºæ�¢è¡Œã€�è¡Œå†…èŠ‚ç‚¹ç›´æŽ¥æ‹¼æŽ¥ã€�<br> å‡ºæ�¢è¡Œï¼ˆâ‰ˆ æµ�è§ˆå™¨çœ‹åˆ°çš„æ�¢è¡Œç»“æž„ï¼‰ã€‚NÚ r   )ÚNavigableStringc                 óF  •— | j                   D ]‘  }t        |‰«      r‰j                  t        |«      «       Œ*t	        |dd «      dk(  r‰j                  d«       ŒLt	        |dd «      t
        v r+‰j                  d«        ‰|«       ‰j                  d«       ŒŠ ‰|«       Œ“ y )NÚnameÚbrú
)ÚchildrenÚ
isinstanceÚappendÚstrÚgetattrÚ_BLOCK_TAGS)ÚnodeÚchr2   ÚoutÚwalks     €€€ú!/root/gov_crawler/crawl_panjin.pyr@   zbody_text.<locals>.walk2   s„   ø€ Ø—-‘-ò 
	ˆBÜ˜"˜oÔ.Ø—
‘
œ3˜r›7Õ#Ü˜˜V TÓ*¨dÒ2Ø—
‘
˜4Õ Ü˜˜V TÓ*¬kÑ9Ø—
‘
˜4Ô Ù�R”Ø—
‘
˜4Õ á�R•ñ
	ó    z[ \t\r\f\v]*\n[ \t\r\f\v]*r6   z\n{3,}z

)ÚreÚbs4r2   ÚjoinÚsubÚstrip)ÚelÚ_reÚtr2   r?   r@   s      @@@rA   Ú	body_textrK   *   sb   ú€ à	€zØÛÝ#Ø
€Cöñ 	ˆ„HØ
�‰�‹€AØ�‰Ð-¨t°QÓ7€AØ�‰�	˜6 1Ó%€AØ�7‰7‹9ÐrB   c                 ó$   — | dk(  rt         S d| › d�S )Né   z'https://ldwxq.panjin.gov.cn/13775/list-z.html)ÚBASE_URL)Úpage_nums    rA   Úget_page_urlrP   E   s   € Ø�1‚}ÜˆØ4°X°J¸eÐDÐDrB   c                 ó
  — t        | d«      }g }|j                  dd¬«      }|st        d«       |S |j                  dd¬«      D ]¼  }|j                  d	«      }|sŒ|j	                  d
d«      }|j	                  dd«      }|s|j                  d¬«      }|j                  d«      }d}	t        |«      dk\  r|d   j                  d¬«      }	|r!|j                  d«      st        t        |«      }|j                  |||	dœ«       Œ¾ |S )Núhtml.parserr   ÚQ25_ListContlist©Úclass_z(  [WARN] Could not find Q25_ListContlistr   F)Ú	recursiveÚaÚhrefr1   ÚtitleT©rG   Úspané   éÿÿÿÿÚhttp)ÚurlrY   Údate)r   ÚfindÚprintÚfind_allÚgetÚget_textÚlenÚ
startswithr   rN   r9   )
ÚhtmlÚsoupÚitemsr   r   rW   rX   rY   Úspansr`   s
             rA   Ú
parse_listrl   J   s  € Ü˜˜}Ó-€DØ€EØ	�‰�4Ð 2ˆÓ	3€BÙÜÐ8Ô:ØˆØ�k‰k˜$¨%ˆkÓ0ò BˆØ�G‰G�C‹LˆÙØØ�u‰u�V˜RÓ ˆØ—‘�g˜rÓ"ˆÙØ—J‘J T�JÓ*ˆEØ—
‘
˜6Ó"ˆØˆÜˆu‹:˜Š?Ø˜‘9×%Ñ%¨DÐ%Ó1ˆDÙ˜Ÿ™¨Ô/Üœ8 TÓ*ˆDØ�‰˜T¨E¸4Ñ@ÕAðBð €LrB   c                 ód  — t        |d«      }|j                  dd¬«      }|r|j                  d¬«      nd}|j                  dt        j                  d	«      ¬
«      }d}|r;|j                  d¬«      }t        j
                  d|«      }|r|j                  d«      }|j                  dd¬«      }	d}
|	rG|	j                  d«      D ](  }g d¢}t        |j                  «      D ]
  }||vsŒ||= Œ Œ* t        |	«      }
|	rt        |	«      nd}g }|	rž|	j                  dd¬«      D ]ˆ  }|j                  dd«      }|j                  d«      sŒ'|j                  |j                  d«      st!        | |«      n||j                  d¬«      xs t"        j$                  j'                  |«      dœ«       ŒŠ |||
||dœS )NrR   r   ÚQ25_articleconttitlerT   TrZ   r1   r[   u   å�‘å¸ƒæ—¥æœŸ)Ústringz(\d{4}-\d{1,2}-\d{1,2})rM   r   ÚQ25_artcont)rX   ÚsrcÚaltÚtargetrW   ©rX   rX   )z.pdfz.docz.docxz.xlsz.xlsxz.zipz.rarr^   )r_   Útext)Ú
list_titleÚpub_dateÚcontent_htmlÚsummaryÚattachments)r   ra   re   rC   ÚcompileÚsearchÚgrouprc   ÚlistÚattrsr:   rK   rd   Úendswithr9   rg   r   ÚosÚpathÚbasename)r_   rh   ri   Útitle_elrv   Údate_elrw   ÚtxtÚmÚcontent_divrx   ÚtagÚattrs_to_keepÚattrru   rz   rW   Úahrefs                     rA   Úparse_detailr�   b   sÀ  € Ü˜˜}Ó-€Dð �y‰y˜Ð%;ˆyÓ<€HÙ2:�×"Ñ"¨Ð"Ô.À€Jð �i‰i˜¤r§z¡z°/Ó'BˆiÓC€GØ€HÙØ×Ñ TÐÓ*ˆÜ�I‰IÐ0°#Ó6ˆÙØ—w‘w˜q“zˆHð —)‘)˜E¨-�)Ó8€KØ€LÙà×'Ñ'¨Ó-ò 	"ˆCâ<ˆMÜ˜SŸY™Y›ò "�Ø˜}Ò,Ø˜D™	ñ"ð	"ô ˜;Ó'ˆñ &1Œ9�[Ô!°b€Dð €KÙØ×%Ñ% c°Ð%Ó5ò 	ˆAØ—E‘E˜& "Ó%ˆEØ�~‰~ÐXÕYØ×"Ñ"Ø6;×6FÑ6FÀvÔ6Nœ7 3¨Ô.ÐTYØŸJ™J¨T˜JÓ2ÒM´b·g±g×6FÑ6FÀuÓ6Mñ$õ ð	ð !ØØ$ØØ"ñð rB   c                  óv  — t        d«       t        j                  t        d¬«      } d| _        d}t        | j                  d«      }|j                  dd¬	«      D ]O  }|j                  d
d«      }t        j                  d|«      }|sŒ.t        |j                  d«      «      }||kD  sŒN|}ŒQ t        d|› �«       |S )Nz  Checking total pages...é   ©Útimeoutúutf-8rM   rR   rW   Trt   rX   r1   z
list-(\d+)z  Total pages: )rb   Úsessionrd   rN   Úencodingr   ru   rc   rC   r|   Úintr}   )ÚrespÚmax_pageri   rW   rX   r‡   Úpns          rA   Úget_total_pagesr™   •   s«   € Ü	Ð
%Ô&Ü�;‰;”x¨ˆ;Ó,€DØ€D„Mà€Hä˜Ÿ™ MÓ2€DØ�]‰]˜3 Tˆ]Ó*ò ˆØ�u‰u�V˜RÓ ˆÜ�I‰I�m TÓ*ˆÚÜ�Q—W‘W˜Q“Z“ˆBØ�H‹}Ø‘ðô 
ˆO˜H˜:Ð
&Ô'Ø€OrB   c                  ó„  — t        «       } g }t        d| dz   «      D ]”  }t        |«      }t        d|› d| › d|› d�«       	 t        j                  |d¬«      }d|_        t        |j                  «      }t        d	t        |«      › d
�«       |j                  |«       t        j                  d«       Œ– t        dt        |«      › �«       t        «       }g }|D ]/  }	|	d   |vsŒ|j                  |	d   «       |j!                  |	«       Œ1 t        dt        |«      › �«       d}
t#        t$        dd¬«      5 }t'        |«      D �]  \  }}	|	d   }t        d|dz   › dt        |«      › d|	d   d d › d�«       	 t        j                  |d¬«      }d|_        t)        ||j                  «      }|j                  d«      xs |	d   }|||d   xs |	d   |d   |d   r|d   d d nddd |d!   rt+        j,                  |d!   d"¬#«      ndd$œ}|j/                  t+        j,                  |d"¬#«      d%z   «       |
dz  }
t        j                  d'«       �Œ	 	 d d d «       t        d(|
› d)t$        › �«       y # t        $ r}t        d|› �«       Y d }~�Œëd }~ww xY w# t        $ r}t        d|› d&|› �«       Y d }~Œ{d }~ww xY w# 1 sw Y   ŒkxY w)*NrM   z  Page ú/z (ú)r�   r�   r’   z
    Found z itemsú    [ERROR] g      à?z
Total list items: r_   zUnique items: r   Úw©r”   z  [z] rY   é(   z...rv   rw   r`   rx   ry   éô  r1   õ'   ç›˜é”¦è¾½æ»¨æ²¿æµ·ç»�æµŽæŠ€æœ¯å¼€å�‘åŒºõ   ç›˜é”¦è¾½æ»¨rz   F©Úensure_ascii©rY   r_   r`   Úcontentry   Ú	site_namer}   rz   r6   ú: ç333333Ó?z
Done! z records written to )r™   ÚrangerP   rb   r“   rd   r”   rl   ru   rf   ÚextendÚ	ExceptionÚtimeÚsleepÚsetÚaddr9   ÚopenÚOUTPUT_FILEÚ	enumerater�   ÚjsonÚdumpsÚwrite)Útotal_pagesÚ	all_itemsÚpager_   r–   rj   ÚeÚ	seen_urlsÚunique_itemsÚitemÚrecord_countÚfÚiÚdetailrY   Úrecords                   rA   Ú	crawl_allrÄ   §   sæ  € Ü!Ó#€Kà€IÜ�a˜ q™Ó)ò ˆÜ˜4Ó ˆÜ�˜�v˜Q˜{˜m¨2¨c¨U°!Ð4Ô5ð	&Ü—;‘;˜s¨B�;Ó/ˆDØ#ˆDŒMÜ˜tŸy™yÓ)ˆEÜ�Jœs 5›z˜l¨&Ð1Ô2Ø×Ñ˜UÔ#ô 	�
‰
�3�ðô 
Ð ¤ Y£Ð 0Ð
1Ô2ô “€IØ€LØò &ˆØ�‰;˜iÒ'Ø�M‰M˜$˜u™+Ô&Ø×Ñ Õ%ð&ô 
ˆNœ3˜|Ó,Ð-Ð
.Ô/ð €LÜ	Œk˜3¨Ô	1ð °QÜ  Ó.ó 	‰GˆAˆtØ�u‘+ˆCÜ�C˜˜!™�u˜Aœc ,Ó/Ð0°°4¸±=ÀÀ"Ð3EÐ2FÀcÐJÔKð1Ü—{‘{ 3°�{Ó3�Ø '�”Ü% c¨4¯9©9Ó5�àŸ
™
 <Ó0ÒA°D¸±M�ð #ØØ" :Ñ.Ò>°$°v±,Ø% nÑ5Ø:@ÀÒ:K˜v iÑ0°°#Ñ6ÐQSØ!JØ+Ø\bÐcpÒ\q¤4§:¡:¨f°]Ñ.CÐRWÕ#XÐwyñ	�ð —‘œŸ
™
 6¸Ô>ÀÑEÔFØ Ñ!�ô �J‰J�sŽOñ1	÷ô6 
ˆH�\�NÐ"6´{°mÐ
DÕEøôY ò 	&Ü�L  Ð$×%Ò%ûð	&ûôP ò 1Ü˜ S E¨¨A¨3Ð/×0Ñ0ûð1ú÷/ð úsP   Á AI*Ä5?J6Å5B>JÈ3J6É*	JÉ3JÊJÊ	J3ÊJ.Ê)J6Ê.J3Ê3J6Ê6J?Ú__main__z--incrementalz&Incremental mode: crawling page 1 onlyr�   r�   r’   z  Found z items on page 1rž   rŸ   r_   rv   rY   rw   r`   rx   ry   r¡   r1   r¢   r£   rz   Fr¤   r¦   r6   r�   r©   rª   zIncremental done: z records)3Ú__doc__Úrequestsrµ   rC   r�   Úsysr®   rD   r   Úurllib.parser   rN   ÚBASE_DOMAINr³   ÚHEADERSÚmakedirsr‚   ÚdirnameÚSessionr“   ÚheadersÚupdater<   rK   rP   rl   r�   r™   rÄ   Ú__name__ÚargvÚis_incrementalrb   r_   rd   r–   r”   ru   rj   rf   r²   rÀ   r¾   rÂ   rY   r¶   rÃ   r·   r­   r»   r¯   © rB   rA   ú<module>rÕ      sv  ðñ÷ )× (× (Ý Ý  à/€Ø+€Ø<€ð DØOØ0ñ€ð €‡�ˆB�G‰G�O‰O˜KÓ(°4Õ 8à
ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ ò*€òò6Eò
ò01òfò$9Fðv ˆzÓØ$¨¯©Ð0€NÚáÐ6Ô7ØˆØ�{‰{˜3¨ˆ{Ó+ˆØˆŒÙ˜4Ÿ9™9Ó%ˆÙ�™˜U›˜Ð$4Ð5Ô6á�+˜s¨WÔ5ð 	 ¸Øò  �ð=Ø"Ÿ;™; t¨E¡{¸B˜;Ó?�DØ$+�D”MÙ)¨$¨u©+°t·y±yÓA�FØ"ŸJ™J |Ó4ÒE¸¸W¹�Eà!&Ø# E™{Ø & zÑ 2Ò B°d¸6±lØ#)¨.Ñ#9Ø>DÀYÒ>O 6¨)Ñ#4°T°cÑ#:ÐUWØ%NØ!/Ø`fÐgtÒ`u z t§z¡z°&¸Ñ2GÐV[Õ'\Ð{}ñ	�Fð —G‘G˜J˜DŸJ™J v¸EÔBÀTÑIÔJð �—
‘
˜3•ñ' ÷	 ñ* 	Ð"¡3 u£: ,¨hÐ7Õ8á�ðE øð: !ò =Ù˜L¨¨e©¨°R¸°sÐ;×<Ñ<ûð=ú÷%	 ð 	 ús7   ÄH$ÄB5HÇH$ÈH!ÈHÈH$ÈH!È!H$È$H-