Ë
    £,j‰  ã                   óæ  — d Z ddlZddlZddlZddlZddlZ ej                  d«       ej                  j                  dej                  j                  ej                  j                  e«      «      «       ddlmZ ddlZddlmZ dZdZdd	iZd
Zd„ Zd„ Zd„ Zdd„Zdd„Zedk(  rKdej8                  v r1 eej8                  ej8                  j=                  d«      dz      «      ndZ ee¬«       yy)u÷   
å¹¿ä¸°åŒºç”Ÿæ€�çŽ¯å¢ƒå±€ â€” çŽ¯è¯„ä¿¡æ�¯
================================
CMS: UCAP CMS
åˆ—è¡¨: /qsthjj/HPXXS/zwgk_list.shtml (ç¬¬1é¡µ), zwgk_list_{page}.shtml (ç¬¬2é¡µèµ·)
è¯¦æƒ…: /qsthjj/HPXXS/{year}/{id}.shtml (div#zoomcon > ucapcontent > p)
é    NÚignore)Úpush_to_searchdb©ÚBeautifulSoupu%   å¹¿ä¸°åŒºç”Ÿæ€�çŽ¯å¢ƒå±€-çŽ¯è¯„ä¿¡æ�¯zhttps://www.gfx.gov.cnz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36é   c                 óŽ   — 	 t        j                  | t        d¬«      }d|_        |j                  S # t
        $ r
}Y d }~y d }~ww xY w)Né   )ÚheadersÚtimeoutzutf-8)ÚrequestsÚgetÚHEADERSÚencodingÚtextÚ	Exception)ÚurlÚrÚes      ú$/root/gov_crawler/crawl_guangfeng.pyÚ
fetch_pager      s<   € ðÜ�L‰L˜¤g°rÔ:ˆØˆŒ
Ø�v‰vˆøÜò Üûðús   ‚.1 ±	A¿Ac                 óP  — | dk(  r
t         › d�}nt         › d| › d�}t        |«      }|sg dfS t        |d«      }|j                  d«      }g }|D ]Á  }|j	                  d«      }|j	                  d	«      }|sŒ(|j                  d
d«      }	|j                  dd«      }
|
s|j                  d¬«      }
|r|j                  d¬«      nd}|	r5|	j                  d«      s$t        j                  j                  t         |	«      }	|j                  |
|	|dœ«       ŒÃ t        |«      t        k\  }||fS )zHFetch list page: base URL for page 1, zwgk_list_{page}.shtml for page 2+é   z/qsthjj/HPXXS/zwgk_list.shtmlz/qsthjj/HPXXS/zwgk_list_z.shtmlFúhtml.parserzul#doclist > liÚaÚspanÚhrefÚ ÚtitleT©ÚstripÚhttp)r   r   Údate)ÚBASE_URLr   r   ÚselectÚfindr   Úget_textÚ
startswithÚurllibÚparseÚurljoinÚappendÚlenÚ	PAGE_SIZE)Úpage_nor   ÚhtmlÚsoupÚitemsÚresultsÚlir   r   r   r   Údate_strÚhas_mores                r   Úfetch_list_pager6   $   s)  € à�!‚|Ü�
Ð7Ð8‰ä�
Ð2°7°)¸6ÐBˆä�c‹?€DÙØ�5ˆyÐä˜˜}Ó-€DØ�K‰KÐ)Ó*€Eà€GØò HˆØ�G‰G�C‹LˆØ�w‰w�v‹ˆÙØØ�u‰u�V˜RÓ ˆØ—‘�g˜rÓ"ˆÙØ—J‘J T�JÓ*ˆEÙ04�4—=‘= t�=Ô,¸"ˆÙ˜Ÿ™¨Ô/Ü—<‘<×'Ñ'¬°$Ó7ˆDØ�‰ ¨t¸XÑFÕGðHô �5‹zœYÑ&€HØ�HÐÐó    c           	      óˆ  — t        | «      }|sdddg fS t        |d«      }d}t        j                  d|«      }|r|j	                  d«      }|j                  dd¬«      xs |j                  dd¬	«      }|sd|dg fS |j                  d
«      }|r|n|}g }g }	|j                  d«      D �]C  }
|
j                  d«      D ]¢  }|j                  dd«      }|j                  d¬«      }|sŒ*t        j                  d|t        j                  «      sŒPt        j                  j                  t        |«      }|	j                  ||dœ«       |sŒ‹|j                  d|› d|› d�«       Œ¤ d}|
j                  d«      D ]  }|j                  |«       Œ |
j                  dd¬«      }|j!                  |d«      }t        j"                  dd|«      }|s�Œ#t%        |«      dkD  s�Œ3|j                  |«       �ŒF |j                  d«      D ]'  }t'        |t        ¬«      }|sŒ|j                  |«       Œ) |rdj)                  |«      nd}d|||	fS )z@Extract content from detail page (div#zoomcon > ucapcontent > p)Nr   z(\d{4}-\d{1,2}-\d{1,2})r   ÚdivÚzoomcon)ÚidÚzwcon)Úclass_ÚucapcontentÚpr   r   r   Tr   z+\.(pdf|doc|docx|xls|xlsx|rar|zip|ppt|pptx)$)Únamer   z- [ú](ú)z BR Úbrú ú
z\n\s*\nz

é   Útable)Úbase_url)r   r   ÚreÚsearchÚgroupr%   Úfind_allr   r&   ÚIr(   r)   r*   r#   r+   Úreplace_withÚreplaceÚsubr,   Úhtml_table_to_htmlÚjoin)Ú
detail_urlr/   r0   r4   Údate_mr:   ÚucapÚcontent_rootÚ
body_partsÚattachmentsr?   r   r   ÚfnameÚfull_urlÚDELIMrC   ÚtxtrG   ÚmdÚbodys                        r   Úparse_detailr_   E   s:  € ä�jÓ!€DÙØ�T˜4 Ð#Ð#ä˜˜}Ó-€Dð €HÜ�Y‰YÐ1°4Ó8€FÙØ—<‘< “?ˆð �i‰i˜ )ˆiÓ,ÒP°·	±	¸%È°	Ó0P€GÙØ�X˜t RÐ'Ð'à�<‰<˜Ó&€DÙ‘4 W€Là€JØ€Kð ×"Ñ" 3Ó'ó #ˆà—‘˜C“ò 	BˆAØ—5‘5˜ Ó$ˆDØ—J‘J T�JÓ*ˆEÚœŸ	™	Ð"PÐRVÔXZ×X\ÑX\Õ]Ü!Ÿ<™<×/Ñ/´¸$Ó?�Ø×"Ñ"¨E¸(Ñ#CÔDÚØ×%Ñ%¨¨E¨7°"°X°J¸aÐ&@ÕAð	Bð ˆØ—*‘*˜TÓ"ò 	#ˆBØ�O‰O˜EÕ"ð	#à�j‰j˜ DˆjÓ)ˆØ�k‰k˜% Ó&ˆÜ�f‰f�Z ¨Ó-ˆã”3�s“8˜a”<Ø×Ñ˜cÖ"ð)#ð. ×&Ñ& wÓ/ò "ˆÜ ´Ô9ˆÚØ×Ñ˜bÕ!ð"ñ
 '1ˆ6�;‰;�zÔ"°d€DØ�˜4 Ð,Ð,r7   c                 óØ  — ddl m}  |t        | «      d«      }|j                  d«      D ]P  }|j	                  dd«      }|sŒ|j                  d«      rŒ*|r t        j                  j                  ||«      n||d<   ŒR |j                  d«      D ]P  }|j	                  d	d«      }|sŒ|j                  d
«      rŒ*|r t        j                  j                  ||«      n||d	<   ŒR t        |«      S )uA   ä¿�ç•™ HTML è¡¨æ ¼ç»“æž„ï¼Œä»…å°†ç›¸å¯¹é“¾æŽ¥/å›¾ç‰‡è½¬ç»�å¯¹ URLr   r   r   r   r   r   )r!   Ú
javascriptú#ÚimgÚsrc)r!   z//zdata:)	Úbs4r   ÚstrrL   r   r'   r(   r)   r*   )rG   rH   r   Útblr   r   rc   rd   s           r   rQ   rQ   ‚   sÐ   € å!Ù
œ˜E›
 MÓ
2€CØ�\‰\˜#Óò SˆØ�u‰u�V˜RÓ ˆÚ˜Ÿ™Ð(CÕDÙ@HœŸ™×,Ñ,¨X°tÔ<ÈdˆAˆfŠIðSð �|‰|˜EÓ"ò RˆØ�g‰g�e˜RÓ ˆÚ�s—~‘~Ð&=Õ>Ù@HœŸ™×-Ñ-¨h¸Ô<ÈcˆC�ŠJðRô ˆs‹8€Or7   rF   c                 ó*  — g }t        d| dz   «      D ]F  }t        |«      \  }}|s n4|j                  |«       t        d|› dt	        |«      › d�d¬«       |rŒF n t        dt	        |«      › �d¬«       g }d}d}t        |«      D ]Û  \  }}	|	d	   }
|	d
   }|	d   }t        |«      \  }}}}|xs |
}|xs |}|r,dj                  |D �cg c]  }d|d   › d|d
   › d�‘Œ c}«      nd}|r|dz  }|s|dz  }|j                  t        |||||xs d|rt        j                  dd|xs d«      d d nd|dœ«       |dz   dz  dk(  sŒ½t        d|dz   › dt	        |«      › �d¬«       ŒÝ t        dt	        |«      › d|› d|› d�d¬«       t        |d«       y c c}w )Nr   zPage z: z itemsT)Úflushz
Total list items: r   r   r   r"   ú;ú[r@   rA   rB   r   z\s+rD   i,  )Ú	site_namer   r   Ú
source_urlÚpub_dateÚcontentÚsummaryrX   é
   z
  Parsed: ú/zParsed: z items (z empty body, z with attachments)Úguangfeng_hpxx)Úranger6   ÚextendÚprintr,   Ú	enumerater_   rR   r+   Ú	SITE_NAMErI   rP   r   )ÚpagesÚ	all_itemsr.   r1   r5   r2   Ú
empty_bodyÚwith_attachÚiÚitemr   rS   Ú	list_dateÚparsed_titlern   r^   rX   Úfinal_titleÚ
final_dater   Ú
attach_strs                        r   Úcrawlr„   ‘   së  € Ø€IÜ˜˜E A™IÓ&ò ˆÜ)¨'Ó2‰ˆˆxÙÙØ×Ñ˜ÔÜ��g�Y˜b¤ U£ ¨FÐ3¸4Õ@ÚÙðô 
Ð ¤ Y£Ð 0Ð
1¸Õ>à€GØ€JØ€Kä˜YÓ'ò C‰ˆˆ4Ø�W‘ˆØ˜%‘[ˆ
Ø˜‘Lˆ	Ü4@ÀÓ4LÑ1ˆ�h  kØ"Ò+ eˆØÒ* ˆ
ÙVa�S—X‘XÀ[ÖQÀ  1 V¡9 +¨R°°%±¨z¸Ò;ÒQÔRÐgiˆ
ÙØ˜1ÑˆKÙØ˜!‰OˆJØ�‰Ü"Ø ØØ$Ø"Ø’z˜rÙBFœŸ™˜v s¨DªJ°BÓ7¸¸Ñ=ÈBØ%ñ	
ô 		ð �‰E�R‰<˜1ÓÜ�J˜q ™s˜e 1¤S¨£^Ð$4Ð5¸TÖBð/Cô2 
ˆH”S˜“\�N (¨:¨,°mÀKÀ=ÐPbÐ
cÐkoÕpÜ�WÐ.Õ/ùò' Rs   ÃF
Ú__main__z--pagesr   )ry   )r   )rF   ) Ú__doc__ÚsysÚosrI   Úurllib.parser(   ÚwarningsÚfilterwarningsÚpathÚinsertÚdirnameÚabspathÚ__file__Úcrawler_libr   r   re   r   rx   r#   r   r-   r   r6   r_   rQ   r„   Ú__name__ÚargvÚintÚindexry   © r7   r   ú<module>r—      sá   ðñ÷ +× *Ó *Ø €× Ñ ˜Ô !à ‡�‡���2—7‘7—?‘? 2§7¡7§?¡?°8Ó#<Ó=Ô >Ý (ã Ý à3€	Ø#€àð  Dð€ð €	òòòB:-ózó+0ð\ ˆzÒØ<EÈÏÉÑ<Q‰C�—‘˜Ÿ™Ÿ™¨	Ó2°QÑ6Ñ7Ô8ÐWX€EÙ	�Öð r7   