Ë
    €Ûµjô  ã                   ó  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddlZ
dZdZdZdZd	Zd
ZddddœZ ej$                  «       Zej"                  j)                  e«       dd„Zdd„Zd„ Zd„ Zefd„Zd„ Zd„ Zedk(  r e«        yy)u„   
panji.gov.cn - æ½˜é›†åŒºäººæ°‘æ”¿åºœ / é€šçŸ¥å…¬å‘Š
ç«™ç‚¹: www.panji.gov.cn/pjzx/tzgg/
CMS: é¾™è®¯ (Ls.pagination)
åˆ†ç»„: å®‰å¾½
é    N©ÚBeautifulSoup)Úurljoinz#https://www.panji.gov.cn/pjzx/tzgg/u   æ½˜é›†åŒºäººæ°‘æ”¿åºœu   å®‰å¾½é   z/https://www.panji.gov.cn/content/column/6787369z/root/search.dbzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagec                 óØ  — ddl m}  |t        | «      d«      }|j                  d«      D ]P  }|j	                  dd«      }|sŒ|j                  d«      rŒ*|r t        j                  j                  ||«      n||d<   ŒR |j                  d«      D ]P  }|j	                  d	d«      }|sŒ|j                  d
«      rŒ*|r t        j                  j                  ||«      n||d	<   ŒR t        |«      S )uA   ä¿�ç•™ HTML è¡¨æ ¼ç»“æž„ï¼Œä»…å°†ç›¸å¯¹é“¾æŽ¥/å›¾ç‰‡è½¬ç»�å¯¹ URLr   r   úhtml.parserÚaÚhrefÚ )ÚhttpÚ
javascriptú#ÚimgÚsrc)r   z//zdata:)	Úbs4r   ÚstrÚfind_allÚgetÚ
startswithÚurllibÚparser   )ÚtableÚbase_urlr   Útblr
   r   r   r   s           ú$/root/gov_crawler/crawl_pj_notice.pyÚhtml_table_to_htmlr   &   sÐ   € å!Ù
œ˜E›
 MÓ
2€CØ�\‰\˜#Óò SˆØ�u‰u�V˜RÓ ˆÚ˜Ÿ™Ð(CÕDÙ@HœŸ™×,Ñ,¨X°tÔ<ÈdˆAˆfŠIðSð �|‰|˜EÓ"ò RˆØ�g‰g�e˜RÓ ˆÚ�s—~‘~Ð&=Õ>Ù@HœŸ™×-Ñ-¨h¸Ô<ÈcˆC�ŠJðRô ˆs‹8€Oó    c                 óÔ   — t        |«      D ]$  }	 t        j                  | d¬«      }d|_        |c S  y # t        $ r*}||dz
  k  rdd l}|j                  d«       n|‚Y d }~ŒVd }~ww xY w)Né   ©Útimeoutzutf-8é   r   é   )ÚrangeÚsessionr   ÚencodingÚ	ExceptionÚtimeÚsleep)ÚurlÚretriesÚattemptÚrÚer)   s         r   Ú
fetch_pager0   5   ss   € Ü˜“>ò 
ˆð		Ü—‘˜C¨�Ó,ˆAØ ˆAŒJØŠHð	
ð øô ò 	Ø˜ 1™Ò$ÛØ—
‘
˜1•à�ô ûð	ús   �4´	A'½ A"Á"A'c                 ó¼  — t        | d«      }g }|j                  dd„ ¬«      }|s|S |j                  dd¬«      D ]Ÿ  }|j                  dd	¬
«      }|sŒ|d   }|j                  dd«      }|s%|j                  d«      }	|	r|	j	                  d	¬«      }|j                  dd¬«      }
|
r|
j	                  d	¬«      nd}t        ||«      }|j                  |||f«       Œ¡ |S )Nr	   Úulc                 ó   — | xr d| xs dv S )NÚdoc_listr   © )Úxs    r   ú<lambda>z"parse_list_items.<locals>.<lambda>G   s   € ¨!Ò*G°
¸qºwÀBÐ0G€ r   ©Úclass_ÚliF)Ú	recursiver
   T©r   r   Útitler   Úspan©ÚstripÚdate)r   Úfindr   r   Úget_textr   Úappend)ÚhtmlÚlist_urlÚsoupÚitemsr2   r:   r
   r   r=   r>   Ú	date_spanÚdate_strÚ
detail_urls                r   Úparse_list_itemsrL   D   sï   € Ü˜˜}Ó-€DØ€EØ	�‰�4Ñ GˆÓ	H€BÙØˆà�k‰k˜$¨%ˆkÓ0ò 4ˆØ�G‰G�C˜dˆGÓ#ˆÙØà�‰yˆØ—‘�g˜rÓ"ˆÙØ—6‘6˜&“>ˆDÙØŸ™¨D˜Ó1�ð —G‘G˜F¨6�GÓ2ˆ	Ù5>�9×%Ñ%¨DÐ%Ô1ÀBˆä˜X tÓ,ˆ
Ø�‰�e˜Z¨Ð2Õ3ð#4ð$ €Lr   c                 óê  ‡— t        | «      }|syt        |j                  d«      }d}|j                  dddi¬«      }|r|j	                  dd«      }d}|j                  ddd	i¬«      }|rK|j	                  dd«      }t        j                  d
|«      }|r!|j                  d«      j                  dd«      }g }	g }
|j                  dd¬«      }|rû|j                  d«      D ]<  }|j                  ddg«      rŒ|j                  d¬«      }|sŒ,|	j                  |«       Œ> |j                  d«      D ]  }|	j                  t        || «      «       Œ |j                  dd¬«      D ]d  }|d   Št        ˆfd„dD «       «      sŒ|j                  d¬«      xs ‰j                  d«      d   }t!        | ‰«      }|
j                  ||dœ«       Œf dj#                  |	«      }|t%        j&                  |
d¬ «      ||fS )!N)Nú[]r   r   r	   r   ÚmetaÚnameÚArticleTitle)ÚattrsÚcontentÚPubDatez(\d{4}[-/]\d{1,2}[-/]\d{1,2})r#   ú/ú-ÚdivÚwzconr8   ÚpÚtdÚthTr?   r   r
   r<   r   c              3   óB   •K  — | ]  }|‰j                  «       v –— Œ y ­w)N)Úlower)Ú.0Úextr   s     €r   ú	<genexpr>zparse_detail.<locals>.<genexpr>‹   s   øè ø€ Òm¨3�3˜$Ÿ*™*›,Ô&Ñmùs   ƒ)z.pdfz.docz.docxz.xlsz.xlsxz.zipz.raréÿÿÿÿ)rP   r+   ú

F)Úensure_ascii)r0   r   ÚtextrB   r   ÚreÚsearchÚgroupÚreplacer   Úfind_parentsrC   rD   r   ÚanyÚsplitr   ÚjoinÚjsonÚdumps)rK   r.   rG   r=   Úmeta_trJ   Úmeta_dÚrawÚmÚ
body_partsÚattachmentsrX   rY   Útxtr   ÚatagÚfnameÚfull_urlÚbodyr   s                      @r   Úparse_detailrz   `   sõ  ø€ Ü�:Ó€AÙØ!ä˜Ÿ™ Ó/€Dð €EØ�Y‰Y�v f¨nÐ%=ˆYÓ>€FÙØ—
‘
˜9 bÓ)ˆð €HØ�Y‰Y�v f¨iÐ%8ˆYÓ9€FÙØ�j‰j˜ BÓ'ˆÜ�I‰IÐ6¸Ó<ˆÙØ—w‘w˜q“z×)Ñ)¨#¨sÓ3ˆHð €JØ€Kà�I‰I�e GˆIÓ,€EÙØ—‘ Ó$ò 	'ˆAà�~‰~˜t T˜lÔ+ØØ—*‘* 4�*Ó(ˆCÚØ×!Ñ! #Õ&ð	'ð —^‘^ GÓ,ò 	EˆEØ×ÑÔ0°¸
ÓCÕDð	Eð —N‘N 3¨T�NÓ2ò 	EˆDØ˜‘<ˆDÜÓmÐ2lÔmÕmØŸ™¨D˜Ó1ÒH°T·Z±ZÀ³_ÀRÑ5H�Ü" :¨tÓ4�Ø×"Ñ"¨E¸(Ñ#CÕDð	Eð �;‰;�zÓ"€DØ”—‘˜K°eÔ<¸hÈÐMÐMr   c                 ó’  — g }t        | «      D �]5  }|dk(  r
t        dz   }nt        › d|dz   › �}t        d|dz   › d| › d|› �d¬	«       t	        |«      }|st        d
d¬	«       ŒUt        |j                  |«      }|st        dd¬	«       Œ{t        dt        |«      › d�d¬	«       |D ]œ  \  }}}t        |«      \  }	}
}}|xs |}|xs |}|	xs d}t        |«      dkD  }|rt        |j                  d«      «      nd}|j                  ||||	|
t        t        ||dœ	«       t        d|d d d›d|d›d|› d|
dk7  rdnd› �d¬	«       Œž �Œ8 |S )Nr   z
index.htmlz?pageIndex=r#   z  [rU   z] T©Úflushz    FAILED to fetchz    No items foundz
    Found z itemsr   rb   )	r=   Úpage_urlrA   ry   rt   Ú	site_namerg   Ú_has_contentÚ
_seg_countz    é(   Ú40sz | Ú12sz | seg=z
 | attach=rN   ÚYESÚno)r%   ÚBASE_URLÚ
COLUMN_URLÚprintr0   rL   rd   Úlenrz   rk   rD   Ú	SITE_NAMEÚGROUP)Ú	max_pagesÚ	all_itemsÚpage_idxr+   r.   rH   r=   rK   Ú	list_datery   Úattachments_jsonÚdetail_dateÚdetail_titleÚ	use_titleÚuse_dateÚbody_for_checkÚhas_contentÚ	seg_counts                     r   Úcrawlr™   ”   s¯  € Ø€Iä˜)Ó$ó 'UˆØ�qŠ=Ü˜\Ñ)‰Cä�L ¨H°q©L¨>Ð:ˆCä��H˜q‘L�>  9 +¨R°¨uÐ5¸TÕBÜ�s‹OˆÙÜÐ'°Õ5Øä  §¡¨Ó-ˆÙÜÐ&¨tÕ4ØÜ�
œ3˜u›:˜, fÐ-°TÕ:à,1ò 	UÑ(ˆE�:˜yÜ@LÈZÓ@XÑ=ˆDÐ" K°Ø$Ò-¨ˆIØ"Ò/ iˆHà!šZ RˆNÜ˜nÓ-°Ñ1ˆKÙ=Kœ˜N×0Ñ0°Ó8Ô9ÐQRˆIà×ÑØ"Ø&Ø ØØ/Ü&ÜØ +Ø'ñ
ô 
ô �D˜ 3 B˜¨Ð,¨C°¸¨~¸WÀYÀKÈzÐcsÐw{Òc{ÑZ_ð  BFð  ZGð  Hð  PTö  Uò+	Uð%'UðR Ðr   c                 ó6  — dd l }|j                  t        d¬«      }|j                  d«       |j	                  «       }d}| D ]5  }|j                  d|d   |d   |d   |d	   |d
   |d   |d   f«       |dz  }Œ7 |j                  «        |j                  «        |S )Nr   é<   r!   zPRAGMA busy_timeout=60000zÏ
            INSERT OR REPLACE INTO gov_raw (title, page_url, site_name, publish_date, content, attachments, date_rank, group_name, script_name) VALUES (?, ?, ?, ?, ?, ?, 0, ?, 'crawl_pj_notice.py')
        r=   r~   r   rA   ry   rt   rg   r#   )Úsqlite3ÚconnectÚDB_PATHÚexecuteÚcursorÚcommitÚclose)rH   rœ   ÚconnÚcÚcountÚitems         r   Úimport_to_dbr§   Ã   sµ   € ÛØ�?‰?œ7¨Bˆ?Ó/€DØ‡L�LÐ,Ô-Ø�‰‹€Að
 €EØò ˆØ	�	‰	ð ð �‰MØ�ÑØ�ÑØ�‰LØ�‰LØ�ÑØ�‰Mð
ô
	ð 	�‰
‰ðð 	‡K�K„MØ‡J�J„LØ€Lr   c                  ó  — dd l } | j                  «       }t        }t        t        j                  «      dkD  r2t        j                  d   dk(  rt        t        j                  d   «      }t        dd¬«       t        d|› �d¬«       t        |¬	«      }|st        d
d¬«       y t        |«      }t        d„ |D «       «      }|rt        d„ |D «       «      |z  nd}t        d„ |D «       «      }t        dd¬«       t        d|› �d¬«       t        d|› d||z  dz  d›d�d¬«       t        d|d›�d¬«       t        d|› �d¬«       t        |«      }t        d|› �d¬«       | j                  «       |z
  }	t        d|	d›d�d¬«       y )Nr   r#   z--pagesr$   u+   === panji.gov.cn æ½˜é›†åŒº é€šçŸ¥å…¬å‘Š ===Tr|   zPages: )r�   zNo items collected!c              3   ó,   K  — | ]  }|d    sŒ	d–— Œ y­w)r€   r#   Nr5   ©r^   Úis     r   r`   zmain.<locals>.<genexpr>ò   s   è ø€ Ò:˜!¨¨.Ó(9”AÑ:ùs   ‚
�c              3   ó&   K  — | ]	  }|d    –— Œ y­w)r�   Nr5   rª   s     r   r`   zmain.<locals>.<genexpr>ó   s   è ø€ Ò1 a�!�L•/Ñ1ùs   ‚c              3   ó2   K  — | ]  }|d    dk7  sŒd–— Œ y­w)rt   rN   r#   Nr5   rª   s     r   r`   zmain.<locals>.<genexpr>ô   s   è ø€ ÒC˜A¨!¨MÑ*:¸dÓ*B”aÑCùs   ‚�z
=== Results ===zTotal: zWith body: z (éd   z.1fz%)zAvg segments: zWith attachments: z
Imported: z	Elapsed: Ús)
r)   Ú	MAX_PAGESrŠ   ÚsysÚargvÚintr‰   r™   Úsumr§   )
r)   ÚstartÚmax_prH   ÚtotalÚ	with_bodyÚavg_segÚwith_attachÚimportedÚelapseds
             r   Úmainr½   à   sf  € ÛØ�I‰I‹K€Eä€EÜ
Œ3�8‰8ƒ}�qÒœSŸX™X a™[¨IÒ5Ü”C—H‘H˜Q‘KÓ ˆä	Ð7ÀÕEÜ	ˆG�E�7Ð
 4Õ(ä˜EÔ"€EáÜÐ#¨4Õ0Øä�‹J€EÜÑ:˜uÔ:Ó:€IÙ=BŒcÑ1¨5Ô1Ó1°EÒ9È€GÜÑC ÔCÓC€Kä	Ð dÕ+Ü	ˆG�E�7Ð
 4Õ(Ü	ˆK˜	�{ " Y°Ñ%6¸Ñ%<¸SÐ$AÀÐ
DÈDÕQÜ	ˆN˜7 3˜-Ð
(°Õ5Ü	Ð˜{˜mÐ
,°DÕ9ä˜EÓ"€HÜ	ˆJ�x�jÐ
!¨Õ.à�i‰i‹k˜EÑ!€GÜ	ˆI�g˜c�] !Ð
$¨DÖ1r   Ú__main__)r   )é   )Ú__doc__Úrequestsre   rm   r±   Úosr   r   Úurllib.parser   r   r‡   r‹   rŒ   r°   rˆ   rž   ÚheadersÚSessionr&   Úupdater   r0   rL   rz   r™   r§   r½   Ú__name__r5   r   r   ú<module>rÈ      s¹   ðñó Û 	Û Û 
Û 	Ý Ý  Û à0€Ø#€	Ø€Ø€	à>€
à
€ðGàOØ0ñ	€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ óóòò81Nðh ó ,ò^ò: 2ðF ˆzÒÙ…Fð r   