Ë
    ¸¬uj²  ã                   óÜ  — d Z ddlZddlZddlZddlZddlZ ej                  d«       ej                  j                  dej                  j                  ej                  j                  e«      «      «       ddlmZ ddlZddlmZ dZdZdd	iZdd
„Zd„ Zd„ Zdd„Zedk(  rKdej4                  v r1 eej4                  ej4                  j9                  d«      dz      «      ndZ ee¬«       yy)uå   
çŽ¯ä¿�å…¬ç¤ºç½‘ (www.huanbao58.com) â€” å…¨éƒ¨æ �ç›®
=========================================
ä¼�ä¸šè‡ªè¡Œå�‘å¸ƒçš„çŽ¯ä¿�å…¬ç¤ºè�šå�ˆå¹³å�°ã€‚
åˆ—è¡¨: /?page=N (20æ�¡/é¡µ)
è¯¦æƒ…: detail.php?itemid=N (å†…å®¹åœ¨ table.detailtb)
é    NÚignore)Úpush_to_searchdb)ÚBeautifulSoupu   çŽ¯ä¿�å…¬ç¤ºç½‘-å…¨éƒ¨zhttps://www.huanbao58.comz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c                 óB  — t        |«      D ]3  }	 t        j                  | t        d¬«      }d|_        |j
                  c S  y # t        $ rR}||dz
  k  rdd l}|j                  d«       n*t        d| d d › d	|› �t        j                  ¬
«       Y d }~ y Y d }~Œ�d }~ww xY w)Né   )ÚheadersÚtimeoutzutf-8é   r   é   z
  [ERROR] éP   ú: )Úfile)ÚrangeÚrequestsÚgetÚHEADERSÚencodingÚtextÚ	ExceptionÚtimeÚsleepÚprintÚsysÚstderr)ÚurlÚretriesÚattemptÚrÚer   s         ú$/root/gov_crawler/crawl_huanbao58.pyÚfetchr!      s‘   € Ü˜“>ò ˆð
	Ü—‘˜S¬'¸2Ô>ˆAØ ˆAŒJØ—6‘6ŠMñ	øô
 ò 	Ø˜ 1™Ò$ÛØ—
‘
˜1•ä˜
 3 s¨ 8 *¨B¨q¨cÐ2¼¿¹ÕDÝô ûð	ús   �.AÁ	BÁABÂBc           	      óv  — | dk(  r
t         dz   }nt         d| › �z   }t        |«      }|sg S t        |d«      }g }|j                  d«      }|s|S |j	                  d«      D ]×  }|j                  d«      }|sŒ|j                  dd	«      }d
|vrŒ.t        j                  d|«      }	|	sŒG|j                  d¬«      }
|j	                  d«      }t        |«      dkD  r|d   j                  d¬«      nd	}t        j                  j                  t         |«      }|j                  |
|||	j                  d«      dœ«       ŒÙ |S )u!   Fetch a page of the 'å…¨éƒ¨' listr
   ú/z/?page=úhtml.parserÚtableÚtrÚaÚhrefÚ zdetail.php?itemid=zitemid=(\d+)T©ÚstripÚtdéÿÿÿÿ)Útitler   ÚdateÚitemid)ÚBASE_URLr!   r   ÚfindÚfind_allr   ÚreÚsearchÚget_textÚlenÚurllibÚparseÚurljoinÚappendÚgroup)Úpage_nor   r   ÚsoupÚitemsr%   r&   r'   r(   r0   r.   Útdsr/   Ú
detail_urls                 r    Úfetch_list_pagerB   (   s9  € à�!‚|Ü˜‰n‰ä˜7 7 )Ð,Ñ,ˆÜ�‹:€DÙØˆ	Ü˜˜}Ó-€DØ€EØ�I‰I�gÓ€EÙØˆØ�n‰n˜TÓ"ò cˆØ�G‰G�C‹LˆÙØØ�u‰u�V˜RÓ ˆØ tÑ+ØÜ—‘˜?¨DÓ1ˆÙØà—
‘
 �
Ó&ˆà�k‰k˜$ÓˆÜ/2°3«x¸!ª|ˆs�2‰w×Ñ dÐÔ+ÀˆÜ—\‘\×)Ñ)¬(°DÓ9ˆ
Ø�‰˜u¨ZÀÐQW×Q]ÑQ]Ð^_ÓQ`ÑaÕbð!cð" €Ló    c           	      ó�  — t        | «      }|sddddg fS t        |d«      }|j                  d«      }|r|j                  d¬«      nd}t	        j
                  dd|«      j                  «       }|j                  dd	¬
«      }|s|dddg fS d}g }d}g }	|j                  d«      D �]  }
|
j                  d«      }|
j                  d«      }|r|sŒ+|j                  d¬«      }|dk(  r*|j                  d¬«      }|	j                  d|› d|› �«       Œl|dk(  r2|j                  d¬«      }|	j                  d|› d|› �«       |sŒ�|rŒ |}Œ£|dk(  r*|j                  d¬«      }|	j                  d|› d|› �«       ŒÒ|dk(  rU|j                  d«      }|r|j                  d¬«      }n|j                  d¬«      }|s�Œ|	j                  d|› d|› �«       �Œ,|dk(  s�Œ3g }|j                  d«      D ]—  }|j                  dd«      }|j                  d¬«      xs d}d|v sŒ0t        j                  j                  t        |«      }|j                  ||dœ«       |r|j                  d|› d|› d�«       Œƒ|j                  d|› d�«       Œ™ |s�Œä|	j                  dd j                  |«      z   «       �Œ	 |	rd!j                  |	«      nd}|||||fS )"z Extract content from detail pageNr$   r.   Tr*   r)   u   -çŽ¯ä¿�å…¬ç¤ºç½‘$r%   Údetailtb)Úclass_r&   Úthr,   u   ç±»åž‹z**z**: u   æ ‡é¢˜u   å…¬ç¤ºæ—¥æœŸu   è¯¦ç»†ä»‹ç»�Úprez**:

u   é™„ä»¶åˆ—è¡¨r'   r(   z/download.php)Únamer   z- [z](ú)u   - [ä¸‹è½½](u   **é™„ä»¶åˆ—è¡¨**:
ú
z

)r!   r   r2   r6   r4   Úsubr+   r3   r;   r   r8   r9   r:   r1   Újoin)rA   r   r>   Ú	title_tagr.   r%   Úpub_dateÚattachmentsÚ	cate_typeÚ
field_rowsr&   rG   r,   ÚkeyÚtxtrH   Úattach_linksr'   r(   ÚfnameÚfull_urlÚbodys                         r    Úparse_detailrY   J   sç  € ä�Ó€DÙØ�T˜4  rÐ)Ð)Ü˜˜}Ó-€Dà—	‘	˜'Ó"€IÙ.7ˆI×Ñ TÐÔ*¸R€EÜ�F‰FÐ'¨¨UÓ3×9Ñ9Ó;€Eà�I‰I�g jˆIÓ1€EÙØ�d˜D $¨Ð*Ð*Ø€HØ€KØ€IØ€Jà�n‰n˜TÓ"ó (TˆØ�W‰W�T‹]ˆØ�W‰W�T‹]ˆÙ™ØØ�k‰k ˆkÓ%ˆà�(Š?ØŸ™¨$˜Ó/ˆIØ×Ñ  3 % t¨I¨;Ð7Õ8Ø�HŠ_Ø—+‘+ D�+Ó)ˆCØ×Ñ  3 % t¨C¨5Ð1Ô2Úš5Ø‘Ø�NÒ"Ø—{‘{¨�{Ó.ˆHØ×Ñ  3 % t¨H¨:Ð6Õ7Ø�NÒ"à—'‘'˜%“.ˆCÙØ—l‘l¨�lÓ.‘à—k‘k¨�kÓ-�ÛØ×!Ñ! B s e¨7°3°%Ð"8Ö9Ø�NÔ"ØˆLØ—[‘[ Ó%ò 	G�Ø—u‘u˜V RÓ(�ØŸ
™
¨˜
Ó.Ò4°"�Ø" dÒ*Ü%Ÿ|™|×3Ñ3´H¸dÓC�HØ×&Ñ&°¸hÑ'GÔHÙØ$×+Ñ+¨c°%°¸¸8¸*ÀAÐ,FÕGà$×+Ñ+¨k¸(¸À1Ð,EÕFð	Gó Ø×!Ñ!Ð$7¸4¿9¹9À\Ó;RÑ"RÖSðQ(TñV '1ˆ6�;‰;�zÔ"°d€DØ�(˜I t¨[Ð8Ð8rC   é   c                 ó  — g }t        dd«      D ]?  }t        |«      }|s n0|j                  |«       t        d|› dt	        |«      › d�d¬«       ŒA t        dt	        |«      › �d¬«       g }d	}d	}t        |«      D ]ß  \  }}|d
   }	|d   }
|d   }t        |
«      \  }}}}}|xs |	}|xs |}|r"dj                  |D �cg c]  }|d   ‘Œ	 c}«      nd}|r|dz  }|s|dz  }|j                  t        ||
|
||xs d|rt        j                  dd|xs d«      d d nd||xs d|rd|› �nddœ
«       |dz   dz  d	k(  sŒÁt        d|dz   › dt	        |«      › �d¬«       Œá t        dt	        |«      › d|› d|› d�d¬«       t        |d«       y c c}w )Nr
   r   zPage r   z itemsT)Úflushz
Total list items: r   r.   r   r/   ú;rI   r)   z\s+ú i,  z
huanbao58,Ú	huanbao58)
Ú	site_namer.   r   Ú
source_urlrO   ÚcontentÚsummaryrP   ÚcategoryÚtagsé
   z
  Parsed: r#   zParsed: z items (z empty body, z with attachments)Úhuanbao58_all)r   rB   Úextendr   r7   Ú	enumeraterY   rM   r;   Ú	SITE_NAMEr4   rL   r   )ÚpagesÚ	all_itemsr=   r?   ÚresultsÚ
empty_bodyÚwith_attachÚiÚitemr.   rA   Ú	list_dateÚparsed_titlerO   rQ   rX   rP   Úfinal_titleÚ
final_dater'   Ú
attach_strs                        r    Úcrawlrw   Œ   sà  € Ø€Iä˜˜A“;ò AˆÜ Ó(ˆÙÙØ×Ñ˜ÔÜ��g�Y˜b¤ U£ ¨FÐ3¸4Ö@ðAô 
Ð ¤ Y£Ð 0Ð
1¸Õ>Ø€GØ€JØ€KÜ˜YÓ'ò C‰ˆˆ4Ø�W‘ˆØ˜%‘[ˆ
Ø˜‘Lˆ	Ü?KÈJÓ?WÑ<ˆ�h 	¨4°Ø"Ò+ eˆØÒ* ˆ
ÙCN�S—X‘X°+Ö>¨Q˜q ›yÒ>Ô?ÐTVˆ
ÙØ˜1ÑˆKÙØ˜!‰OˆJØ�‰Ü"Ø ØØ$Ø"Ø’z˜rÙBFœŸ™˜v s¨DªJ°BÓ7¸¸Ñ=ÈBØ%Ø!š RÙ09�j  Ñ,¸{ñ
ô 	ð �‰E�R‰<˜1ÓÜ�J˜q ™s˜e 1¤S¨£^Ð$4Ð5¸TÖBð3Cô4 
ˆH”S˜“\�N (¨:¨,°mÀKÀ=ÐPbÐ
cÐkoÕpÜ�W˜oÕ.ùò) ?s   Â>F

Ú__main__z--pagesr
   )rk   )é   )rZ   )Ú__doc__r   Úosr4   Úurllib.parser8   ÚwarningsÚfilterwarningsÚpathÚinsertÚdirnameÚabspathÚ__file__Úcrawler_libr   r   Úbs4r   rj   r1   r   r!   rB   rY   rw   Ú__name__ÚargvÚintÚindexrk   © rC   r    ú<module>r‹      sÕ   ðñ÷ +× *Ó *Ø €× Ñ ˜Ô !à ‡�‡���2—7‘7—?‘? 2§7¡7§?¡?°8Ó#<Ó=Ô >Ý (ã Ý à$€	Ø&€àð  Dð€ó
òòD?9óD(/ðV ˆzÒØ<EÈÏÉÑ<Q‰C�—‘˜Ÿ™Ÿ™¨	Ó2°QÑ6Ñ7Ô8ÐWX€EÙ	�Öð rC   