Ë
    €Ûµjê  ã                   ó  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 dZ
dZdZdZd	Zd
dddœZ ej                   «       Zej                  j%                  e«       dd„Zd„ Zd„ Zefd„Zd„ Zd„ Zedk(  r e«        yy)u�   
panzhou.gov.cn - ç›˜å·žå¸‚äººæ°‘æ”¿åºœ / é€šçŸ¥å…¬å‘Š
ç«™ç‚¹: www.panzhou.gov.cn/jjpz/tzgg/
CMS: TRSï¼ˆæ¸…å�Žå�Œæ–¹æ€�åˆ›ï¼‰
åˆ†ç»„: è´µå·ž
é    N)ÚBeautifulSoup)Úurljoinz%https://www.panzhou.gov.cn/jjpz/tzgg/u   ç›˜å·žå¸‚äººæ°‘æ”¿åºœu   è´µå·žé   z/root/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagec                 óÔ   — t        |«      D ]$  }	 t        j                  | d¬«      }d|_        |c S  y # t        $ r*}||dz
  k  rdd l}|j                  d«       n|‚Y d }~ŒVd }~ww xY w)Né   ©Útimeoutzutf-8é   r   é   )ÚrangeÚsessionÚgetÚencodingÚ	ExceptionÚtimeÚsleep)ÚurlÚretriesÚattemptÚrÚer   s         ú"/root/gov_crawler/crawl_pz_tzgg.pyÚ
fetch_pager   #   ss   € Ü˜“>ò 
ˆð		Ü—‘˜C¨�Ó,ˆAØ ˆAŒJØŠHð	
ð øô ò 	Ø˜ 1™Ò$ÛØ—
‘
˜1•à�ô ûð	ús   �4´	A'½ A"Á"A'c                 óÜ  — t        | d«      }g }|j                  dd¬«      }|s|S |j                  dd¬«      }|s|S |j                  dd¬	«      D ]™  }|j                  d
«      }|j                  d«      }|r|j                  d«      sŒ9|d   }	|j                  dd«      xs |j	                  d¬«      }
|r|j	                  d¬«      nd}t        ||	«      }|j                  |
||f«       Œ› |S )Núhtml.parserÚdivÚPageMainBox©Úclass_ÚulÚNewsListÚliF)Ú	recursiveÚaÚspanÚhrefÚtitleÚ T©Ústrip)r   ÚfindÚfind_allr   Úget_textr   Úappend)ÚhtmlÚlist_urlÚsoupÚitemsÚpmbr!   r#   r%   r&   r'   r(   Údate_strÚ
detail_urls                r   Úparse_list_itemsr7   2   sð   € Ü˜˜}Ó-€DØ€EØ
�)‰)�E -ˆ)Ó
0€CÙØˆà	�‰�$˜zˆÓ	*€BÙØˆà�k‰k˜$¨%ˆkÓ0ò 4ˆØ�G‰G�C‹LˆØ�w‰w�v‹ˆÙ˜Ÿ™˜fœØà�‰yˆØ—‘�g˜rÓ"Ò< a§j¡j°t jÓ&<ˆÙ04�4—=‘= t�=Ô,¸"ˆä˜X tÓ,ˆ
Ø�‰�e˜Z¨Ð2Õ3ð4ð €Ló    c                 óð  ‡— t        | «      }|syt        |j                  d«      }d}|j                  dddi¬«      }|r|j	                  dd«      }d}|j                  ddd	i¬«      }|rK|j	                  dd«      }t        j                  d
|«      }|r!|j                  d«      j                  dd«      }g }	g }
|j                  dd¬«      }|s|j                  dd„ ¬«      }|rÁ|j                  D ]8  }|j                  dk(  sŒ|j                  d¬«      }|sŒ(|	j                  |«       Œ: |j                  dd¬«      D ]d  }|d   Št        ˆfd„dD «       «      sŒ|j                  d¬«      xs ‰j                  d«      d   }t!        | ‰«      }|
j                  ||dœ«       Œf |s%|j                  d«      }|r|j                  d¬«      }dj#                  |	«      }|t%        j&                  |
d¬«      ||fS ) N)Nú[]r)   r)   r   r)   ÚmetaÚnameÚArticleTitle)ÚattrsÚcontentÚPubDatez(\d{4}[-/]\d{1,2}[-/]\d{1,2})r   ú/ú-r   Útrs_editor_viewr   c                 ó6   — | xr d| xs dj                  «       v S )NÚTRSr)   )Úupper)Úcs    r   ú<lambda>zparse_detail.<locals>.<lambda>i   s   € ¨qÒ/O°U¸qºwÀB¿o¹oÓ>OÐ5O€ r8   ÚpTr*   r%   )r'   r'   c              3   óB   •K  — | ]  }|‰j                  «       v –— Œ y ­w)N)Úlower)Ú.0Úextr'   s     €r   ú	<genexpr>zparse_detail.<locals>.<genexpr>u   s   øè ø€ Òm¨3�3˜$Ÿ*™*›,Ô&Ñmùs   ƒ)z.pdfz.docz.docxz.xlsz.xlsxz.zipz.raréÿÿÿÿ)r<   r   Úh1ú

F)Úensure_ascii)r   r   Útextr,   r   ÚreÚsearchÚgroupÚreplaceÚchildrenr<   r.   r/   r-   ÚanyÚsplitr   ÚjoinÚjsonÚdumps)r6   r   r2   r(   Úmeta_tr5   Úmeta_dÚrawÚmÚ
body_partsÚattachmentsÚteÚchildÚtxtÚatagÚfnameÚfull_urlrP   Úbodyr'   s                      @r   Úparse_detailrk   L   só  ø€ Ü�:Ó€AÙØ!ä˜Ÿ™ Ó/€Dð €EØ�Y‰Y�v f¨nÐ%=ˆYÓ>€FÙØ—
‘
˜9 bÓ)ˆð €HØ�Y‰Y�v f¨iÐ%8ˆYÓ9€FÙØ�j‰j˜ BÓ'ˆÜ�I‰IÐ6¸Ó<ˆÙØ—w‘w˜q“z×)Ñ)¨#¨sÓ3ˆHð €JØ€Kð 
�‰�5Ð!2ˆÓ	3€BÙØ�Y‰Y�uÑ%OˆYÓPˆá	Ø—[‘[ò 	+ˆEØ�z‰z˜SÓ Ø—n‘n¨4�nÓ0�ÚØ×%Ñ% cÕ*ð		+ð —K‘K ¨$�KÓ/ò 	EˆDØ˜‘<ˆDÜÓmÐ2lÔmÕmØŸ™¨D˜Ó1ÒH°T·Z±ZÀ³_ÀRÑ5H�Ü" :¨tÓ4�Ø×"Ñ"¨E¸(Ñ#CÕDð	Eñ Ø�Y‰Y�t‹_ˆÙØ—K‘K d�KÓ+ˆEà�;‰;�zÓ"€DØ”—‘˜K°eÔ<¸hÈÐMÐMr8   c                 óŽ  — g }t        | «      D �]3  }|dk(  r
t        dz   }nt        › d|› d�}t        d|dz   › d| › d|› �d	¬
«       t        |«      }|st        dd	¬
«       ŒSt	        |j
                  |«      }|st        dd	¬
«       Œyt        dt        |«      › d�d	¬
«       |D ]œ  \  }}}t        |«      \  }	}
}}|xs |}|xs |}|	xs d}t        |«      dkD  }|rt        |j                  d«      «      nd}|j                  ||||	|
t        t        ||dœ	«       t        d|d d d›d|d›d|› d|
dk7  rdnd› �d	¬
«       Œž �Œ6 |S )Nr   z
index.htmlÚindex_z.htmlz  [r   rA   z] T©Úflushz    FAILED to fetchz    No items foundz
    Found z itemsr)   rQ   )	r(   Úpage_urlÚdaterj   rc   Ú	site_namerV   Ú_has_contentÚ
_seg_countz    é(   Ú40sz | Ú12sz | seg=z
 | attach=r:   ÚYESÚno)r   ÚBASE_URLÚprintr   r7   rS   Úlenrk   rZ   r/   Ú	SITE_NAMEÚGROUP)Ú	max_pagesÚ	all_itemsÚpage_idxr   r   r3   r(   r6   Ú	list_daterj   Úattachments_jsonÚdetail_dateÚdetail_titleÚ	use_titleÚuse_dateÚbody_for_checkÚhas_contentÚ	seg_counts                     r   Úcrawlr‹   „   s­  € Ø€Iä˜)Ó$ó 'UˆØ�qŠ=Ü˜\Ñ)‰Cä�J˜f X J¨eÐ4ˆCä��H˜q‘L�>  9 +¨R°¨uÐ5¸TÕBÜ�s‹OˆÙÜÐ'°Õ5Øä  §¡¨Ó-ˆÙÜÐ&¨tÕ4ØÜ�
œ3˜u›:˜, fÐ-°TÕ:à,1ò 	UÑ(ˆE�:˜yÜ@LÈZÓ@XÑ=ˆDÐ" K°Ø$Ò-¨ˆIØ"Ò/ iˆHà!šZ RˆNÜ˜nÓ-°Ñ1ˆKÙ=Kœ˜N×0Ñ0°Ó8Ô9ÐQRˆIà×ÑØ"Ø&Ø ØØ/Ü&ÜØ +Ø'ñ
ô 
ô �D˜ 3 B˜¨Ð,¨C°¸¨~¸WÀYÀKÈzÐcsÐw{Òc{ÑZ_ð  BFð  ZGð  Hð  PTö  Uò+	Uð%'UðR Ðr8   c                 ó6  — dd l }|j                  t        d¬«      }|j                  d«       |j	                  «       }d}| D ]5  }|j                  d|d   |d   |d   |d	   |d
   |d   |d   f«       |dz  }Œ7 |j                  «        |j                  «        |S )Nr   é<   r	   zPRAGMA busy_timeout=60000zÍ
            INSERT OR REPLACE INTO gov_raw (title, page_url, site_name, publish_date, content, attachments, date_rank, group_name, script_name) VALUES (?, ?, ?, ?, ?, ?, 0, ?, 'crawl_pz_tzgg.py')
        r(   rp   rr   rq   rj   rc   rV   r   )Úsqlite3ÚconnectÚDB_PATHÚexecuteÚcursorÚcommitÚclose)r3   rŽ   ÚconnrG   ÚcountÚitems         r   Úimport_to_dbr˜   ³   sµ   € ÛØ�?‰?œ7¨Bˆ?Ó/€DØ‡L�LÐ,Ô-Ø�‰‹€Að
 €EØò ˆØ	�	‰	ð ð �‰MØ�ÑØ�ÑØ�‰LØ�‰LØ�ÑØ�‰Mð
ô
	ð 	�‰
‰ðð 	‡K�K„MØ‡J�J„LØ€Lr8   c                  ó  — dd l } | j                  «       }t        }t        t        j                  «      dkD  r2t        j                  d   dk(  rt        t        j                  d   «      }t        dd¬«       t        d|› �d¬«       t        |¬	«      }|st        d
d¬«       y t        |«      }t        d„ |D «       «      }|rt        d„ |D «       «      |z  nd}t        d„ |D «       «      }t        dd¬«       t        d|› �d¬«       t        d|› d||z  dz  d›d�d¬«       t        d|d›�d¬«       t        d|› �d¬«       t        |«      }t        d|› �d¬«       | j                  «       |z
  }	t        d|	d›d�d¬«       y )Nr   r   z--pagesr   u-   === panzhou.gov.cn ç›˜å·žå¸‚ é€šçŸ¥å…¬å‘Š ===Trn   zPages: )r   zNo items collected!c              3   ó,   K  — | ]  }|d    sŒ	d–— Œ y­w)rs   r   N© ©rL   Úis     r   rN   zmain.<locals>.<genexpr>â   s   è ø€ Ò:˜!¨¨.Ó(9”AÑ:ùs   ‚
�c              3   ó&   K  — | ]	  }|d    –— Œ y­w)rt   Nr›   rœ   s     r   rN   zmain.<locals>.<genexpr>ã   s   è ø€ Ò1 a�!�L•/Ñ1ùs   ‚c              3   ó2   K  — | ]  }|d    dk7  sŒd–— Œ y­w)rc   r:   r   Nr›   rœ   s     r   rN   zmain.<locals>.<genexpr>ä   s   è ø€ ÒC˜A¨!¨MÑ*:¸dÓ*B”aÑCùs   ‚�z
=== Results ===zTotal: zWith body: z (éd   z.1fz%)zAvg segments: zWith attachments: z
Imported: z	Elapsed: Ús)
r   Ú	MAX_PAGESr|   ÚsysÚargvÚintr{   r‹   Úsumr˜   )
r   ÚstartÚmax_pr3   ÚtotalÚ	with_bodyÚavg_segÚwith_attachÚimportedÚelapseds
             r   Úmainr¯   Ð   sf  € ÛØ�I‰I‹K€Eä€EÜ
Œ3�8‰8ƒ}�qÒœSŸX™X a™[¨IÒ5Ü”C—H‘H˜Q‘KÓ ˆä	Ð9À$ÕGÜ	ˆG�E�7Ð
 4Õ(ä˜EÔ"€EáÜÐ#¨4Õ0Øä�‹J€EÜÑ:˜uÔ:Ó:€IÙ=BŒcÑ1¨5Ô1Ó1°EÒ9È€GÜÑC ÔCÓC€Kä	Ð dÕ+Ü	ˆG�E�7Ð
 4Õ(Ü	ˆK˜	�{ " Y°Ñ%6¸Ñ%<¸SÐ$AÀÐ
DÈDÕQÜ	ˆN˜7 3˜-Ð
(°Õ5Ü	Ð˜{˜mÐ
,°DÕ9ä˜EÓ"€HÜ	ˆJ�x�jÐ
!¨Õ.à�i‰i‹k˜EÑ!€GÜ	ˆI�g˜c�] !Ð
$¨DÖ1r8   Ú__main__)é   )Ú__doc__ÚrequestsrT   r\   r£   ÚosÚbs4r   Úurllib.parser   rz   r}   r~   r¢   r�   ÚheadersÚSessionr   Úupdater   r7   rk   r‹   r˜   r¯   Ú__name__r›   r8   r   ú<module>r»      s¬   ðñó Û 	Û Û 
Û 	Ý Ý  à2€Ø#€	Ø€Ø€	à
€ðGàOØ0ñ	€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ óòò45Nðp ó ,ò^ò: 2ðF ˆzÒÙ…Fð r8   