Ë
    £,jS  ã                   ó(  — d Z ddlZddlZddlZddlZddlZddlZddlZ ej                  d«       ej                  j                  dej                  j                  ej                  j                  e«      «      «       ddlmZ ddlZddlmZ dZdZdZd	e› d
�Zd	e› �ZddddddddœZdZeddœZd d„Zd„ Zd!d„Zd„ Zd"d„Z e!dk(  rKdejD                  v r1 e#ejD                  ejD                  jI                  d«      dz      «      ndZ% e e%¬«       yy)#u)  
é¾™å�£å¸‚äººæ°‘æ”¿åºœ â€” å»ºè®¾é¡¹ç›®çŽ¯è¯„ (col/col48329)
=============================================
JCMS (Hanweb) with DBApp WAF (self-signed cert).
åˆ—è¡¨: API GET /api-gateway/jpaas-publish-server/front/page/build/unit
è¯¦æƒ…: /col/col48329/art/2026/art_{hash}.html (content in div#zoom)
é    NÚignore)Úpush_to_searchdb©ÚBeautifulSoupu   é¾™å�£å¸‚-å»ºè®¾é¡¹ç›®çŽ¯è¯„zwww.longkou.gov.cnz101.206.185.117zhttps://z7/api-gateway/jpaas-publish-server/front/page/build/unitÚbulidstaticÚ153ÚU1S5IC8QNYd71CkF9ENILÚcolumnu   æ–‡ç« åˆ—è¡¨ÚnullÚ48329)Ú	parseTypeÚwebIdÚtplSetIdÚpageTypeÚtagIdÚeditTypeÚpageIdé   zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36)ÚHostz
User-Agentc                 óD  — t        |«      D ]4  }	 t        j                  | t        dd¬«      }d|_        |j
                  c S  y # t        $ rR}||dz
  k  rt        j                  d«       n*t        d| d d › d	|› �t        j                  ¬
«       Y d }~ y Y d }~ŒŽd }~ww xY w)NFé   )ÚheadersÚverifyÚtimeoutzutf-8é   é   z
  [ERROR] éP   ú: )Úfile)ÚrangeÚrequestsÚgetÚHEADERSÚencodingÚtextÚ	ExceptionÚtimeÚsleepÚprintÚsysÚstderr)ÚurlÚretriesÚattemptÚrÚes        ú"/root/gov_crawler/crawl_longkou.pyÚfetchr2   (   s�   € Ü˜“>ò 
ˆð		Ü—‘˜S¬'¸%ÈÔLˆAØ ˆAŒJØ—6‘6ŠMñ	
øô
 ò 	Ø˜ 1™Ò$Ü—
‘
˜1•ä˜
 3 s¨ 8 *¨B¨q¨cÐ2¼¿¹ÕDÝô ûð	ús   �/AÁ	BÁABÂBc                 óL  — t        t        «      }t        j                  | t        dœd¬«      |d<   t
        dz   t        j                  j                  |«      z   }t        |«      }|sg dfS 	 t        j                  |«      }|j                  d«      sg dfS |d   d	   }t        j                  d
|«      }|rt        |j!                  d«      «      nd}g }t        j"                  d«      }	|	j%                  |«      D ]ˆ  }
t        j                  j'                  t(        |
j!                  d«      «      }|
j!                  d«      j+                  «       }|
j!                  d«      j+                  «       }|j-                  |||dœ«       ŒŠ ||fS # t        j                  $ r g dfcY S w xY w)z+Fetch one list page from JCMS unitbuild API)ÚpageNoÚpageSizeF)Úensure_asciiÚ	paramJsonú?r   ÚsuccessÚdataÚhtmlzcount="(\d+)"r   zM<a\s+href="([^"]+)"[^>]*title="([^"]*)"[^>]*>(.*?)</a>\s*<span>([^<]*)</span>r   é   )Útitler,   Údate)ÚdictÚ
API_PARAMSÚjsonÚdumpsÚ	PAGE_SIZEÚLIST_URLÚurllibÚparseÚ	urlencoder2   ÚloadsÚJSONDecodeErrorr"   ÚreÚsearchÚintÚgroupÚcompileÚfinditerÚurljoinÚDETAIL_BASEÚstripÚappend)Úpage_noÚparamsr,   r%   r:   r;   Úcount_mÚtotalÚitemsÚpatternÚmÚ
detail_urlr=   Údate_strs                 r1   Úfetch_list_pager]   6   sv  € ä”*Ó€FÜŸ*™*°ÄYÑ%OÐ^cÔd€Fˆ;ÑÜ
�S‰.œ6Ÿ<™<×1Ñ1°&Ó9Ñ
9€CÜ�‹:€DÙØ�1ˆuˆðÜ�z‰z˜$Óˆð �8‰8�IÔØ�1ˆuˆØ�‰<˜Ñ€DÜ�i‰iÐ(¨$Ó/€GÙ%,ŒC�—‘˜aÓ Ô!°!€Eà€EÜ�j‰jØXó€Gð ×Ñ˜dÓ#ò LˆÜ—\‘\×)Ñ)¬+°q·w±w¸q³zÓBˆ
Ø—‘˜“
× Ñ Ó"ˆØ—7‘7˜1“:×#Ñ#Ó%ˆØ�‰˜u¨ZÀÑJÕKð	Lð
 �%ˆ<Ðøô# ×Ñò Ø�1ˆuŠðús   Á,F	 Æ	F#Æ"F#c                 óØ  — ddl m}  |t        | «      d«      }|j                  d«      D ]P  }|j	                  dd«      }|sŒ|j                  d«      rŒ*|r t        j                  j                  ||«      n||d<   ŒR |j                  d«      D ]P  }|j	                  d	d«      }|sŒ|j                  d
«      rŒ*|r t        j                  j                  ||«      n||d	<   ŒR t        |«      S )uA   ä¿�ç•™ HTML è¡¨æ ¼ç»“æž„ï¼Œä»…å°†ç›¸å¯¹é“¾æŽ¥/å›¾ç‰‡è½¬ç»�å¯¹ URLr   r   úhtml.parserÚaÚhrefÚ )ÚhttpÚ
javascriptú#ÚimgÚsrc)rc   z//zdata:)	Úbs4r   ÚstrÚfind_allr"   Ú
startswithrE   rF   rP   )ÚtableÚbase_urlr   Útblr`   ra   rf   rg   s           r1   Úhtml_table_to_htmlro   T   sÐ   € å!Ù
œ˜E›
 MÓ
2€CØ�\‰\˜#Óò SˆØ�u‰u�V˜RÓ ˆÚ˜Ÿ™Ð(CÕDÙ@HœŸ™×,Ñ,¨X°tÔ<ÈdˆAˆfŠIðSð �|‰|˜EÓ"ò RˆØ�g‰g�e˜RÓ ˆÚ�s—~‘~Ð&=Õ>Ù@HœŸ™×-Ñ-¨h¸Ô<ÈcˆC�ŠJðRô ˆs‹8€Oó    c                 óâ  — t        | «      }|sdddg fS t        |d«      }|j                  d«      }|r|j                  d¬«      nd}t	        j
                  dd|«      }|j                  d¬	«      }|s|ddg fS d}t	        j                  d
|«      }|r|j                  d«      }g }g }	|j                  g d¢d¬«      D ]Û  }
|
j                  dk(  r(|
j                  dd¬«      }|sŒ(|j                  |«       Œ:|
j                  dk(  r t        |
«      }|sŒW|j                  |«       Œi|
j                  dk(  sŒy|
j                  dd«      }|
j                  dd«      }|sŒ t        j                  j                  t         |«      }|j                  d|› d|› d�«       ŒÝ dj#                  |«      }t%        |«      dk  �r9g }|j                  g d¢«      D �]  }
|
j'                  d«      r|
j                  dk7  rŒ%|
j                  dk(  r:|
j'                  d«      rŒF|
j                  dd¬«      }|sŒ\|j                  |«       Œn|
j                  dk(  r t        |
«      }|sŒ‹|j                  |«       Œ�|
j                  dk(  sŒ­|
j                  dd«      }|
j                  dd«      }|sŒÔt        j                  j                  t         |«      }|j                  d|› d|› d�«       �Œ dj#                  |«      }|j                  d«      D ]`  }|j                  d«      }|sŒ|j                  d¬«      }t        j                  j                  t         |«      }|	j                  ||dœ«       Œb |||xs d|	fS )z Extract content from detail pageNr_   r=   T)rR   rb   z^[^ ]+ Úzoom)Úidu,   å�‘å¸ƒæ—¥æœŸ[ï¼š:]\s*(\d{4}-\d{1,2}-\d{1,2})r   )Úprl   rf   F)Ú	recursivert   ú rl   rf   rg   Úaltz![z](ú)z

éd   r`   z	data-link)Únamer,   )r2   r   ÚfindÚget_textrJ   ÚsubrK   rM   rj   rz   rS   ro   r"   rE   rF   rP   rQ   ÚjoinÚlenÚfind_parent)r[   r%   ÚsoupÚ	title_tagr=   rr   r\   Údate_mÚ
body_partsÚattachmentsÚelÚtxtÚmdrg   rw   Úfull_srcÚ	body_textr`   Ú	data_linkÚfnameÚ	full_links                        r1   Úparse_detailrŽ   c   s+  € ä�Ó€DÙØ�T˜4 Ð#Ð#Ü˜˜}Ó-€Dà—	‘	˜'Ó"€IÙ.7ˆI×Ñ TÐÔ*¸R€EÜ�F‰F�:˜r 5Ó)€Eà�9‰9˜ˆ9Ó€DÙØ�d˜D "Ð$Ð$à€HÜ�Y‰YÐFÈÓM€FÙØ—<‘< “?ˆà€JØ€KØ�m‰mÒ1¸UˆmÓCò ;ˆØ�7‰7�cŠ>Ø—+‘+˜c¨�+Ó.ˆCÚØ×!Ñ! #Õ&Ø�W‰W˜ÒÜ# BÓ'ˆBÚØ×!Ñ! "Õ%Ø�W‰W˜ÓØ—&‘&˜ Ó#ˆCØ—&‘&˜ Ó#ˆCÚÜ!Ÿ<™<×/Ñ/´¸SÓA�Ø×!Ñ! B s e¨2¨h¨Z°qÐ"9Õ:ð;ð  —‘˜JÓ'€IÜ
ˆ9ƒ~˜Óàˆ
Ø—-‘-Ò 5Ó6ó 	?ˆBà�~‰~˜gÔ&¨2¯7©7°gÒ+=ØØ�w‰w˜#Š~à—>‘> 'Ô*ØØ—k‘k #¨T�kÓ2�ÚØ×%Ñ% cÕ*Ø—‘˜GÒ#Ü'¨Ó+�ÚØ×%Ñ% bÕ)Ø—‘˜EÓ!Ø—f‘f˜U BÓ'�Ø—f‘f˜U BÓ'�ÚÜ%Ÿ|™|×3Ñ3´KÀÓE�HØ×%Ñ%¨¨3¨%¨r°(°¸1Ð&=Ö>ð)	?ð* —K‘K 
Ó+ˆ	à�]‰]˜3Óò BˆØ—E‘E˜+Ó&ˆ	ÚØ—J‘J T�JÓ*ˆEÜŸ™×,Ñ,¬[¸)ÓDˆIØ×Ñ¨°iÑ@ÕAðBð �(˜IÒ-¨¨{Ð:Ð:rp   é   c                 ó6  — g }d}t        d| dz   «      D ]T  }t        |«      \  }}|s nB|}|j                  |«       t        d|› dt	        |«      › d�d¬«       t	        |«      |k\  sŒT n t        dt	        |«      › �d¬«       g }d}d}t        |«      D ]Ñ  \  }	}
|
d	   }|
d
   }|
d   }t        |«      \  }}}}|xs |}|xs |}|r"dj                  |D �cg c]  }|d   ‘Œ	 c}«      nd}|r|dz  }|s|dz  }|j                  t        |||||xs d|rt        j                  dd|xs d«      d d nd|dœ«       |	dz   dz  dk(  sŒ³t        d|	dz   › dt	        |«      › �d¬«       ŒÓ t        dt	        |«      › d|› d|› d�d¬«       t        |d«       y c c}w )Nr   r   zPage r   z itemsT)Úflushz
Total list items: r=   r,   r>   ú;rz   rb   z\s+rv   i,  )Ú	site_namer=   r,   Ú
source_urlÚpub_dateÚcontentÚsummaryr…   é
   z
  Parsed: ú/zParsed: z items (z empty body, z with attachments)Úlongkou_hjhp)r    r]   Úextendr)   r   Ú	enumeraterŽ   r~   rS   Ú	SITE_NAMErJ   r}   r   )ÚpagesÚ	all_itemsÚtotal_countrT   rX   rW   ÚresultsÚ
empty_bodyÚwith_attachÚiÚitemr=   r[   Ú	list_dateÚparsed_titler•   Úbodyr…   Úfinal_titleÚ
final_dater`   Ú
attach_strs                         r1   Úcrawlr¬   ­   sé  € Ø€IØ€KÜ˜˜E A™IÓ&ò ˆÜ& wÓ/‰ˆˆuÙÙØˆØ×Ñ˜ÔÜ��g�Y˜b¤ U£ ¨FÐ3¸4Õ@Üˆy‹>˜[Ó(Ùðô 
Ð ¤ Y£Ð 0Ð
1¸Õ>Ø€GØ€JØ€KÜ˜YÓ'ò C‰ˆˆ4Ø�W‘ˆØ˜%‘[ˆ
Ø˜‘Lˆ	Ü4@ÀÓ4LÑ1ˆ�h  kØ"Ò+ eˆØÒ* ˆ
ÙCN�S—X‘X°+Ö>¨Q˜q ›yÒ>Ô?ÐTVˆ
ÙØ˜1ÑˆKÙØ˜!‰OˆJØ�‰Ü"Ø ØØ$Ø"Ø’z˜rÙBFœŸ™˜v s¨DªJ°BÓ7¸¸Ñ=ÈBØ%ñ	
ô 		ð �‰E�R‰<˜1ÓÜ�J˜q ™s˜e 1¤S¨£^Ð$4Ð5¸TÖBð/Cô0 
ˆH”S˜“\�N (¨:¨,°mÀKÀ=ÐPbÐ
cÐkoÕpÜ�W˜nÕ-ùò% ?s   ÃF
Ú__main__z--pagesr   )rž   )é   )rb   )r�   )&Ú__doc__r*   ÚosrJ   rA   Úurllib.parserE   r'   ÚwarningsÚfilterwarningsÚpathÚinsertÚdirnameÚabspathÚ__file__Úcrawler_libr   r!   rh   r   r�   ÚBASE_DOMAINÚCDN_IPrD   rQ   r@   rC   r#   r2   r]   ro   rŽ   r¬   Ú__name__ÚargvrL   Úindexrž   © rp   r1   ú<module>rÀ      s"  ðñ÷ 7× 6× 6Ó 6Ø €× Ñ ˜Ô !à ‡�‡���2—7‘7—?‘? 2§7¡7§?¡?°8Ó#<Ó=Ô >Ý (ã Ý à*€	Ø"€Ø	€Ø�f�XÐTÐU€Ø˜˜Ð!€ð ØØ'ØØØØñ€
ð €	àð Dñ€óòó<òG;óT).ðX ˆzÒØ<EÈÏÉÑ<Q‰C�—‘˜Ÿ™Ÿ™¨	Ó2°QÑ6Ñ7Ô8ÐWX€EÙ	�Öð rp   