Ë
    €Ûµj  ã                   ó  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 dZ
dZdZdZd	Zd
dddœZ ej                   «       Zej                  j%                  e«       dd„Zd„ Zd„ Zefd„Zd„ Zd„ Zedk(  r e«        yy)u}   
panshan.gov.cn - ç›˜å±±åŽ¿äººæ°‘æ”¿åºœ / é€šçŸ¥å…¬å‘Š
ç«™ç‚¹: www.panshan.gov.cn/13480/
CMS: è‡ªå®šä¹‰ (Q25)
åˆ†ç»„: è¾½å®�
é    N)ÚBeautifulSoup)Úurljoinz http://www.panshan.gov.cn/13480/u   ç›˜å±±åŽ¿äººæ°‘æ”¿åºœu   è¾½å®�é   z/root/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagec                 óÔ   — t        |«      D ]$  }	 t        j                  | d¬«      }d|_        |c S  y # t        $ r*}||dz
  k  rdd l}|j                  d«       n|‚Y d }~ŒVd }~ww xY w)Né   ©Útimeoutzutf-8é   r   é   )ÚrangeÚsessionÚgetÚencodingÚ	ExceptionÚtimeÚsleep)ÚurlÚretriesÚattemptÚrÚer   s         ú$/root/gov_crawler/crawl_ps_notice.pyÚ
fetch_pager   #   ss   € Ü˜“>ò 
ˆð		Ü—‘˜C¨�Ó,ˆAØ ˆAŒJØŠHð	
ð øô ò 	Ø˜ 1™Ò$ÛØ—
‘
˜1•à�ô ûð	ús   �4´	A'½ A"Á"A'c                 óÖ  — t        | d«      }g }|j                  dd¬«      }|s|S |j                  dd¬«      D ]­  }|j                  d«      }|r|j                  d	«      sŒ(|d	   }|j                  d
d«      }|j                  d«      }	d}
t	        |	«      dk\  r|	d   j                  d¬«      }
|s|	r|	d   j                  d¬«      }t        ||«      }|j                  |||
f«       Œ¯ |S )Núhtml.parserÚulÚQ25_ListContlist©Úclass_ÚliF)Ú	recursiveÚaÚhrefÚtitleÚ Úspanr   r   T©Ústripr   )r   ÚfindÚfind_allr   ÚlenÚget_textr   Úappend)ÚhtmlÚlist_urlÚsoupÚitemsr   r!   r#   r$   r%   ÚspansÚdate_strÚ
detail_urls               r   Úparse_list_itemsr6   2   s÷   € Ü˜˜}Ó-€DØ€EØ	�‰�4Ð 2ˆÓ	3€BÙØˆà�k‰k˜$¨%ˆkÓ0ò 4ˆØ�G‰G�C‹LˆÙ˜Ÿ™˜fœØà�‰yˆØ—‘�g˜rÓ"ˆà—
‘
˜6Ó"ˆØˆÜˆu‹:˜Š?Ø˜Q‘x×(Ñ(¨tÐ(Ó4ˆHÙ™Ø˜!‘H×%Ñ%¨DÐ%Ó1ˆEä˜X tÓ,ˆ
Ø�‰�e˜Z¨Ð2Õ3ð!4ð" €Ló    c                 óŠ  ‡— t        | «      }|syt        |j                  d«      }d}|j                  dddi¬«      }|r|j	                  dd«      }d}|j                  ddd	i¬«      }|rK|j	                  dd«      }t        j                  d
|«      }|r!|j                  d«      j                  dd«      }g }	g }
|j                  dd¬«      }|rË|j                  dd¬«      }|r¶|j                  d«      D ](  }|j                  d¬«      }|sŒ|	j                  |«       Œ* |j                  dd¬«      D ]d  }|d   Št        ˆfd„dD «       «      sŒ|j                  d¬«      xs ‰j                  d«      d   }t        | ‰«      }|
j                  ||dœ«       Œf dj                  |	«      }|t!        j"                  |
d¬«      ||fS )N)Nú[]r&   r&   r   r&   ÚmetaÚnameÚArticleTitle)ÚattrsÚcontentÚPubDatez(\d{4}[-/]\d{1,2}[-/]\d{1,2})r   ú/ú-ÚdivÚQ25_articlecontr   ÚQ25_artcontÚpTr(   r#   )r$   r$   c              3   óB   •K  — | ]  }|‰j                  «       v –— Œ y ­w)N)Úlower)Ú.0Úextr$   s     €r   ú	<genexpr>zparse_detail.<locals>.<genexpr>s   s   øè ø€ Òq¨s�s˜dŸj™j›lÔ*Ñqùs   ƒ)z.pdfz.docz.docxz.xlsz.xlsxz.zipz.raréÿÿÿÿ)r;   r   ú

F)Úensure_ascii)r   r   Útextr*   r   ÚreÚsearchÚgroupÚreplacer+   r-   r.   ÚanyÚsplitr   ÚjoinÚjsonÚdumps)r5   r   r1   r%   Úmeta_tr4   Úmeta_dÚrawÚmÚ
body_partsÚattachmentsÚacÚartcontrE   ÚtxtÚatagÚfnameÚfull_urlÚbodyr$   s                      @r   Úparse_detailre   M   sË  ø€ Ü�:Ó€AÙØ!ä˜Ÿ™ Ó/€Dð €EØ�Y‰Y�v f¨nÐ%=ˆYÓ>€FÙØ—
‘
˜9 bÓ)ˆð €HØ�Y‰Y�v f¨iÐ%8ˆYÓ9€FÙØ�j‰j˜ BÓ'ˆÜ�I‰IÐ6¸Ó<ˆÙØ—w‘w˜q“z×)Ñ)¨#¨sÓ3ˆHð €JØ€Kà	�‰�5Ð!2ˆÓ	3€BÙ	Ø—'‘'˜%¨�'Ó6ˆÙØ×%Ñ% cÓ*ò +�Ø—j‘j t�jÓ,�ÚØ×%Ñ% cÕ*ð+ð  ×(Ñ(¨°4Ð(Ó8ò I�Ø˜F‘|�ÜÓqÐ6pÔqÕqØ ŸM™M°˜MÓ5ÒL¸¿¹ÀC»ÈÑ9L�EÜ& z°4Ó8�HØ×&Ñ&°¸hÑ'GÕHðIð �;‰;�zÓ"€DØ”—‘˜K°eÔ<¸hÈÐMÐMr7   c                 óŽ  — g }t        | «      D �]3  }|dk(  rt        }nt        › d|dz   › d�}t        d|dz   › d| › d|› �d¬	«       t        |«      }|st        d
d¬	«       ŒSt	        |j
                  |«      }|st        dd¬	«       Œyt        dt        |«      › d�d¬	«       |D ]œ  \  }}}t        |«      \  }	}
}}|xs |}|xs |}|	xs d}t        |«      dkD  }|rt        |j                  d«      «      nd}|j                  ||||	|
t        t        ||dœ	«       t        d|d d d›d|d›d|› d|
dk7  rdnd› �d¬	«       Œž �Œ6 |S )Nr   zlist-r   z.htmlz  [r@   z] T©Úflushz    FAILED to fetchz    No items foundz
    Found z itemsr&   rL   )	r%   Úpage_urlÚdaterd   r]   Ú	site_namerQ   Ú_has_contentÚ
_seg_countz    é(   Ú40sz | Ú12sz | seg=z
 | attach=r9   ÚYESÚno)r   ÚBASE_URLÚprintr   r6   rN   r,   re   rT   r.   Ú	SITE_NAMEÚGROUP)Ú	max_pagesÚ	all_itemsÚpage_idxr   r   r2   r%   r5   Ú	list_daterd   Úattachments_jsonÚdetail_dateÚdetail_titleÚ	use_titleÚuse_dateÚbody_for_checkÚhas_contentÚ	seg_counts                     r   Úcrawlrƒ   |   s¬  € Ø€Iä˜)Ó$ó 'UˆØ�qŠ=Ü‰Cä�J˜e H¨q¡L >°Ð7ˆCä��H˜q‘L�>  9 +¨R°¨uÐ5¸TÕBÜ�s‹OˆÙÜÐ'°Õ5Øä  §¡¨Ó-ˆÙÜÐ&¨tÕ4ØÜ�
œ3˜u›:˜, fÐ-°TÕ:à,1ò 	UÑ(ˆE�:˜yÜ@LÈZÓ@XÑ=ˆDÐ" K°Ø$Ò-¨ˆIØ"Ò/ iˆHà!šZ RˆNÜ˜nÓ-°Ñ1ˆKÙ=Kœ˜N×0Ñ0°Ó8Ô9ÐQRˆIà×ÑØ"Ø&Ø ØØ/Ü&ÜØ +Ø'ñ
ô 
ô �D˜ 3 B˜¨Ð,¨C°¸¨~¸WÀYÀKÈzÐcsÐw{Òc{ÑZ_ð  BFð  ZGð  Hð  PTö  Uò+	Uð%'UðR Ðr7   c                 ó6  — dd l }|j                  t        d¬«      }|j                  d«       |j	                  «       }d}| D ]5  }|j                  d|d   |d   |d   |d	   |d
   |d   |d   f«       |dz  }Œ7 |j                  «        |j                  «        |S )Nr   é<   r	   zPRAGMA busy_timeout=60000zÏ
            INSERT OR REPLACE INTO gov_raw (title, page_url, site_name, publish_date, content, attachments, date_rank, group_name, script_name) VALUES (?, ?, ?, ?, ?, ?, 0, ?, 'crawl_ps_notice.py')
        r%   ri   rk   rj   rd   r]   rQ   r   )Úsqlite3ÚconnectÚDB_PATHÚexecuteÚcursorÚcommitÚclose)r2   r†   ÚconnÚcÚcountÚitems         r   Úimport_to_dbr‘   «   sµ   € ÛØ�?‰?œ7¨Bˆ?Ó/€DØ‡L�LÐ,Ô-Ø�‰‹€Að
 €EØò ˆØ	�	‰	ð ð �‰MØ�ÑØ�ÑØ�‰LØ�‰LØ�ÑØ�‰Mð
ô
	ð 	�‰
‰ðð 	‡K�K„MØ‡J�J„LØ€Lr7   c                  ó  — dd l } | j                  «       }t        }t        t        j                  «      dkD  r2t        j                  d   dk(  rt        t        j                  d   «      }t        dd¬«       t        d|› �d¬«       t        |¬	«      }|st        d
d¬«       y t        |«      }t        d„ |D «       «      }|rt        d„ |D «       «      |z  nd}t        d„ |D «       «      }t        dd¬«       t        d|› �d¬«       t        d|› d||z  dz  d›d�d¬«       t        d|d›�d¬«       t        d|› �d¬«       t        |«      }t        d|› �d¬«       | j                  «       |z
  }	t        d|	d›d�d¬«       y )Nr   r   z--pagesr   u-   === panshan.gov.cn ç›˜å±±åŽ¿ é€šçŸ¥å…¬å‘Š ===Trg   zPages: )rw   zNo items collected!c              3   ó,   K  — | ]  }|d    sŒ	d–— Œ y­w)rl   r   N© ©rH   Úis     r   rJ   zmain.<locals>.<genexpr>Ú   s   è ø€ Ò:˜!¨¨.Ó(9”AÑ:ùs   ‚
�c              3   ó&   K  — | ]	  }|d    –— Œ y­w)rm   Nr”   r•   s     r   rJ   zmain.<locals>.<genexpr>Û   s   è ø€ Ò1 a�!�L•/Ñ1ùs   ‚c              3   ó2   K  — | ]  }|d    dk7  sŒd–— Œ y­w)r]   r9   r   Nr”   r•   s     r   rJ   zmain.<locals>.<genexpr>Ü   s   è ø€ ÒC˜A¨!¨MÑ*:¸dÓ*B”aÑCùs   ‚�z
=== Results ===zTotal: zWith body: z (éd   z.1fz%)zAvg segments: zWith attachments: z
Imported: z	Elapsed: Ús)
r   Ú	MAX_PAGESr,   ÚsysÚargvÚintrt   rƒ   Úsumr‘   )
r   ÚstartÚmax_pr2   ÚtotalÚ	with_bodyÚavg_segÚwith_attachÚimportedÚelapseds
             r   Úmainr¨   È   sf  € ÛØ�I‰I‹K€Eä€EÜ
Œ3�8‰8ƒ}�qÒœSŸX™X a™[¨IÒ5Ü”C—H‘H˜Q‘KÓ ˆä	Ð9À$ÕGÜ	ˆG�E�7Ð
 4Õ(ä˜EÔ"€EáÜÐ#¨4Õ0Øä�‹J€EÜÑ:˜uÔ:Ó:€IÙ=BŒcÑ1¨5Ô1Ó1°EÒ9È€GÜÑC ÔCÓC€Kä	Ð dÕ+Ü	ˆG�E�7Ð
 4Õ(Ü	ˆK˜	�{ " Y°Ñ%6¸Ñ%<¸SÐ$AÀÐ
DÈDÕQÜ	ˆN˜7 3˜-Ð
(°Õ5Ü	Ð˜{˜mÐ
,°DÕ9ä˜EÓ"€HÜ	ˆJ�x�jÐ
!¨Õ.à�i‰i‹k˜EÑ!€GÜ	ˆI�g˜c�] !Ð
$¨DÖ1r7   Ú__main__)é   )Ú__doc__ÚrequestsrO   rV   rœ   ÚosÚbs4r   Úurllib.parser   rs   ru   rv   r›   rˆ   ÚheadersÚSessionr   Úupdater   r6   re   rƒ   r‘   r¨   Ú__name__r”   r7   r   ú<module>r´      s¬   ðñó Û 	Û Û 
Û 	Ý Ý  à-€Ø#€	Ø€Ø€	à
€ðGàOØ0ñ	€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ óòò6,Nð^ ó ,ò^ò: 2ðF ˆzÒÙ…Fð r7   