Ë
    ÉÚµj2$  ã                   óà   — d Z ddlZddlZddlZddlZddlZddlZddlZddlmZ ddl	m
Z
 ddlmZ dZdZdZd	Zd
ZdZdZddddœZdd„Zd„ Zd„ Zd„ Zd„ Zdd„Zd„ Zd„ Zd„ Zedk(  r e«        yy)uá  
Crawler for å¹³ç½—åŽ¿äººæ°‘æ”¿åºœ - å…¬ç¤ºå…¬å‘Š (pingluo.gov.cn)
CMS: TRS CMS
List: div.gl-list-con.newslist.gl-w75 > li > a[href][title] + span(date MM-DD)
Pagination: createPageHTML(28, 0, "index", "html") â†’ index.html, index_N.html
Detail: div.view.TRS_UEDITOR trs_paper_default > p content
Title: meta ArticleTitle (clean ã€�å…¬ç¤ºå…¬å‘Šã€‘prefix)
Date: meta PubDate
Attachments: div.enclosure#fj > a[href*=./P0]

Usage:
    python3 /root/gov_crawler/crawl_pingluo_gsgg.py
é    N)Údatetime)ÚBeautifulSoup)Úurljoinz/root/search.dbu   å¹³ç½—åŽ¿-å…¬ç¤ºå…¬å‘Šu   é€šçŸ¥å…¬å‘Šu   å®�å¤�zhttps://www.pingluo.gov.cnz
/xwzx/gsggé   zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagec                 ó<  — t        |«      D ]3  }	 t        j                  | t        d¬«      }d|_        |j
                  c S  y # t        $ rO}||dz
  k  rt        j                  d«       n't        d| › d|› �t        j                  ¬«       Y d }~ y Y d }~ŒŠd }~ww xY w)	Né   )ÚheadersÚtimeoutzutf-8é   é   z  [WARN] Failed to fetch z: ©Úfile)ÚrangeÚrequestsÚgetÚHEADERSÚencodingÚtextÚ	ExceptionÚtimeÚsleepÚprintÚsysÚstderr)ÚurlÚretriesÚiÚrÚes        ú'/root/gov_crawler/crawl_pingluo_gsgg.pyÚfetchr"   %   s�   € Ü�7‹^ò 
ˆð		Ü—‘˜S¬'¸2Ô>ˆAØ ˆAŒJØ—6‘6ŠMð	
ð øô ò 	Ø�7˜Q‘;ŠÜ—
‘
˜1•äÐ1°#°°b¸¸Ð<Ä3Ç:Á:ÕNÝô ûð	ús   �.AÁ	BÁ?BÂBc                 óN   — | dk(  rt         › t        › d�S t         › t        › d| › d�S )Nr   z/index.htmlz/index_z.html)ÚBASE_URLÚLIST_DIR)Úpage_nums    r!   Úget_list_urlr'   6   s1   € Ø�1‚}Ü�œH˜: [Ð1Ð1ÜˆZœ�z ¨¨
°%Ð8Ð8ó    c                 óæ  — t        | «      }t        |«      }|sg S t        |d«      }g }|j                  dd¬«      }|sg S |j	                  dd¬«      D ]›  }|j                  dd	¬
«      }|sŒ|d   }t        ||«      }	|j                  dd«      j                  «       xs |j                  d	¬«      }
|
sŒa|j                  d«      }|r|j                  d	¬«      nd}|j                  |
|	|f«       Œ� |S )Núhtml.parserÚdivzgl-list-con newslist gl-w75©Úclass_ÚliF)Ú	recursiveÚaT©Úhrefr2   ÚtitleÚ ©ÚstripÚspan)
r'   r"   r   ÚfindÚfind_allr   r   r6   Úget_textÚappend)r&   r   ÚhtmlÚsoupÚitemsÚlist_divr.   r0   r2   Úpage_urlr3   r7   Údate_strs                r!   Úextract_list_pagerB   <   sÿ   € Ü
�xÓ
 €CÜ�‹:€DÙØˆ	ä˜˜}Ó-€DØ€Eà�y‰y˜Ð'DˆyÓE€HÙØˆ	à×Ñ °ÐÓ6ò 2ˆØ�G‰G�C˜dˆGÓ#ˆÙØØ�‰yˆÜ˜3 Ó%ˆØ—‘�g˜rÓ"×(Ñ(Ó*ÒD¨a¯j©j¸t¨jÓ.DˆÙØØ�w‰w�v‹ˆÙ04�4—=‘= t�=Ô,¸"ˆØ�‰�e˜X xÐ0Õ1ð2ð €Lr(   c                 ól   — | syt        j                  d|«      }|r|j                  d«      }|› d| › �S | S )z.List date is MM-DD, extract year from URL pathr4   z/(20\d{2})\d{2}/r   ú-)ÚreÚsearchÚgroup)Ú	list_dater@   ÚmÚyears       r!   Únormalize_daterK   Y   s@   € áØä
�	‰	Ð% xÓ0€AÙØ�w‰w�q‹zˆØ��q˜˜Ð$Ð$ØÐr(   c                 óÞ  — t        | «      }|sd d d g fS t        |d«      }d}|j                  dddi¬«      }|r;|j                  d«      r*|d   j	                  «       }t        j                  dd|«      }|sE|j                  d	«      }|r2|j                  d
¬«      j                  d«      d   j	                  «       }d}|j                  dddi¬«      }|r'|j                  d«      r|d   j	                  «       d d }d}|j                  dt        j                  d«      ¬«      }	|	r¾|	j                  ddg«      D ]  }
|
j                  «        Œ g }|	j                  d«      D ]-  }|j                  d
¬«      }|sŒ|dvsŒ|j                  |«       Œ/ |	j                  d«      D ].  }|j                  d«      rŒ|j                  t        |«      «       Œ0 dj                  |«      }t        j                  dd|«      j	                  «       }g }t!        «       }|j                  dd¬«      }|r�|j                  dd
¬«      D ]w  }|d   }t#        | |«      }|j                  d
¬«      xs& |j                  d«      d   j                  d «      d   }||vsŒS|j%                  |«       |j                  ||d!œ«       Œy |j                  dd
¬«      D ]�  }|d   }t        j&                  d"|t
        j(                  «      sŒ.t#        | |«      }|j                  d
¬«      xs& |j                  d«      d   j                  d «      d   }||vsŒy|j%                  |«       |j                  ||d!œ«       ŒŸ ||||fS )#Nr*   r4   ÚmetaÚnameÚArticleTitle)ÚattrsÚcontentu   ^ã€�[^ã€‘]*ã€‘\s*r3   Tr5   Ú_r   ÚPubDateé
   r+   ÚTRS_UEDITORr,   ÚscriptÚstyleÚp)ú õ   Â z&nbsp;Útablez

z\n{3,}Ú	enclosurer0   r1   r2   ú/éÿÿÿÿú?)rN   r   z"\.(xls|xlsx|pdf|doc|docx|rar|zip)$)r"   r   r8   r   r6   rE   Úsubr:   ÚsplitÚcompiler9   Ú	decomposer;   Úfind_parentÚstrÚjoinÚsetr   ÚaddrF   ÚI)r   r<   r=   r3   Ú
meta_titleÚtrA   Ú	meta_daterQ   ÚeditorÚsÚpartsrX   ÚtblÚattachmentsÚseenÚenc_divÚa_tagr2   Úfull_urlrN   s                        r!   Úfetch_detailrv   g   sW  € Ü�‹:€DÙØ�T˜4 Ð#Ð#ä˜˜}Ó-€Dð €EØ—‘˜6¨&°.Ð)A�ÓB€JÙ�j—n‘n YÔ/Ø˜9Ñ%×+Ñ+Ó-ˆä—‘Ð+¨R°Ó7ˆÙØ�I‰I�gÓˆÙØ—J‘J T�JÓ*×0Ñ0°Ó5°aÑ8×>Ñ>Ó@ˆEð €HØ—	‘	˜&¨°Ð(;�	Ó<€IÙ�Y—]‘] 9Ô-Ø˜YÑ'×-Ñ-Ó/°°Ð4ˆð €GØ�Y‰Y�u¤R§Z¡Z°Ó%?ˆYÓ@€FÙØ—‘ (¨GÐ!4Ó5ò 	ˆAØ�K‰K�Mð	àˆØ—‘ Ó%ò 	 ˆAØ—
‘
 �
Ó&ˆAÚ�QÐ5Ò5Ø—‘˜Q•ð	 ð —?‘? 7Ó+ò 	'ˆCØ—?‘? 7Õ+Ø—‘œS ›XÕ&ð	'ð —+‘+˜eÓ$ˆÜ�f‰f�Y ¨Ó0×6Ñ6Ó8€Gð €KÜ‹5€Dð �i‰i˜ kˆiÓ2€GÙØ×%Ñ% c°Ð%Ó5ò 	DˆEØ˜‘=ˆDÜ˜s DÓ)ˆHØ—>‘>¨�>Ó-ÒR°·±¸C³ÀÑ1D×1JÑ1JÈ3Ó1OÐPQÑ1RˆDØ˜tÒ#Ø—‘˜Ô"Ø×"Ñ"¨D¸Ñ#BÕCð	Dð —‘˜s¨�Ó.ò DˆØ�V‰}ˆÜ�9‰9Ð:¸DÄ"Ç$Á$ÕGÜ˜s DÓ)ˆHØ—>‘>¨�>Ó-ÒR°·±¸C³ÀÑ1D×1JÑ1JÈ3Ó1OÐPQÑ1RˆDØ˜tÒ#Ø—‘˜Ô"Ø×"Ñ"¨D¸Ñ#BÕCðDð �(˜G [Ð0Ð0r(   c                 ól  — t        d|dz   «      D ]  }	  | «       c S  y
# t        j                  $ r�}dt        |«      v rC||k  r>t	        d|› d|› d|› d|› d�	t
        j                  ¬«       t        j                  |«       n't	        d|› d	|› �t
        j                  ¬«       Y d }~ y
Y d }~Œžd }~ww xY w)Nr   Úlockedz  z: locked (attempt r]   z), retry in zs...r   z	 failed: F)	r   Úsqlite3ÚOperationalErrorre   r   r   r   r   r   )ÚfnÚdescÚmax_attemptsÚdelayÚattemptr    s         r!   Ú
with_retryr€   ¯   s¸   € Ü˜˜L¨1Ñ,Ó-ò 	ˆð	Ù“4ŠKð	ð øô ×'Ñ'ò 	Øœ3˜q›6Ñ! g°Ò&<Ü˜˜4˜&Ð 2°7°)¸1¸\¸NÈ,ÐW\ÐV]Ð]aÐbÔil×isÑisÕtÜ—
‘
˜5Õ!ä˜˜4˜& 	¨!¨Ð-´C·J±JÕ?Ýô "ûð	ús   ”ŸB3²A1B.Â.B3c                  ó€   — t        j                  t        d¬«      } | j                  d«       | j                  d«       | S )Nr	   )r   zPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000)ry   ÚconnectÚDB_PATHÚexecute©Úconns    r!   Úinit_dbr‡   ½   s1   € Ü�?‰?œ7¨BÔ/€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø€Kr(   c                 óˆ  — |r|d d n|}t        j                  dd|«      j                  «       }|rt        j                  |d¬«      nd}	 | j                  dt        |||j                  «       |||t        |t        f
«      }|j                  dkD  S # t        $ r(}	t        d	|	› �t        j                  ¬
«       Y d }	~	yd }	~	ww xY w)NéÈ   z\s+rY   F)Úensure_asciiz[]zÍINSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r   z  DB error: r   )rE   r`   r6   ÚjsonÚdumpsr„   Ú	SITE_NAMEÚCATEGORYÚGROUPÚrowcountr   r   r   r   )
r†   r3   r@   Úpublish_daterQ   rq   ÚsummaryÚatt_jsonÚcurr    s
             r!   Úsave_articler•   Ä   s´   € Ù&ˆg�d�s‰m¨E€GÜ�f‰f�V˜S 'Ó*×0Ñ0Ó2€GÙ>IŒt�z‰z˜+°EÕ:Èt€HðØ�l‰lð8ô ˜ (¨E¯K©K«M¸<Ø�gœx¨´5ð:ó	
ˆð �|‰|˜aÑÐøÜò Ü�˜Q˜CÐ ¤s§z¡zÕ2Üûðús   ÁAB Â	CÂB<Â<Cc                  óà  ‡— t         } t        t        j                  «      dkD  r	 t	        t        j                  d   «      } t        dt        › d| › �t        j                  ¬«       t        «       Šd}d}t        j                  «       }ˆfd„}t        dt        j                  ¬«       t        |dt        › �¬	«       t        d
t        › �t        j                  ¬«       t        d| dz   «      D �]‡  }t        |«      }|s"t        d|› d�t        j                  ¬«        �nX|t        |«      z  }d}|D ]’  \  }}	}
‰j                  d|	f«      }|j                  «       rŒ+t        |	«      \  }}}}|xs |}|xs t!        |
|	«      }|xs	 d|› d|	› d�}|xs g }t#        ‰||	|||«      r
|dz  }|dz  }t        j$                  d«       Œ” t	        t        j                  «       |z
  «      }| |z
  }|dkD  r-|dkD  r(t	        |t'        d|t'        d|«      z  «      z  dz  «      nd}t        d|› d| › d|› d|› d|› d|dz  › d|dz  d›d|› d�t        j                  ¬«       |dz  dk(  s�Œx‰j)                  «        �ŒŠ ‰j)                  «        t	        t        j                  «       |z
  «      }‰j+                  «        t        d|› d|› d|dz  › d|dz  d›d �	t        j                  ¬«       y #  Y �Œ¬xY w)!Nr   z	Scraping z, max pages: r   r   c                  ó&   •— ‰ j                  «        y )N)Úcommitr…   s   €r!   Údo_cleanzmain.<locals>.do_cleanæ   s   ø€ à�‰�r(   z  Cleaning old data...zClean )r|   z  Cleaned old data for z  Page z: 0 items (end)z)SELECT id FROM gov_raw WHERE page_url = ?ú[z](ú)g333333Ó?é<   r]   z: +z new, z total, ú:Ú02dz elapsed, ETA ÚminrT   z
Done! Found: z, New: z, Time: rI   rn   )ÚTOTAL_PAGESÚlenr   ÚargvÚintr   r�   r   r‡   r   r€   r   rB   r„   Úfetchonerv   rK   r•   r   Úmaxr˜   Úclose)Ú	max_pagesÚ	total_newÚtotal_foundÚ
start_timer™   ÚpnÚarticlesÚ	batch_newr3   r@   Údater”   Ú	det_titleÚdet_dateÚdet_contentÚdet_attsÚfinal_titleÚ
final_dateÚfinal_contentÚ
final_attsÚelapsedÚ
pages_leftÚetar†   s                          @r!   Úmainrº   Ø   sç  ø€ Ü€IÜ
Œ3�8‰8ƒ}�qÒð	ÜœCŸH™H Q™KÓ(ˆIô 
ˆI”i�[ ¨i¨[Ð
9ÄÇ
Á
ÕKÜ‹9€DØ€IØ€KÜ—‘“€Jôô 
Ð
"¬¯©Õ4Üˆx ¤y kÐ2Õ3Ü	Ð#¤I ;Ð
/´c·j±jÕAä�A�y 1‘}Ó%ó $ˆÜ$ RÓ(ˆÙÜ�G˜B˜4˜Ð/´c·j±jÕAÚà”s˜8“}Ñ$ˆØˆ	à%-ò 	Ñ!ˆE�8˜TØ—,‘,ÐJÈXÈKÓXˆCØ�|‰|Œ~Øä9EÀhÓ9OÑ6ˆI�x ¨hØ#Ò, uˆKØ!ÒC¤^°D¸(Ó%CˆJØ'ÒI¨Q¨{¨m¸2¸h¸ZÀqÐ+IˆMØ!š RˆJä˜D +¨x¸À]ÐT^Ô_Ø˜Q‘�	Ø˜Q‘�	ä�J‰J�s�Oð	ô" ”d—i‘i“k JÑ.Ó/ˆØ ‘^ˆ
ØELÈqÂ[ÐUWÐZ[ÒU[Œc�*œs 1 b¬3¨q°'«?Ñ&:Ó;Ñ;¸bÑ@ÔAÐabˆä�˜�t˜1˜Y˜K s¨9¨+°VØ�˜1˜[˜M¨Ø˜"‘�˜Q˜w r™z¨#Ð.ð /Ø�E˜ðô %(§J¡Jõ	0ð
 �‰7�aŒ<Ø�K‰KŽMðI$ðL 	‡K�K„MÜ”$—)‘)“+ 
Ñ*Ó+€GØ‡J�J„LÜ	ˆO˜K˜=¨°	¨{¸(À7ÈBÁ;À-ÈqÐQXÐY[ÑQ[Ð\_ÐP`Ð`aÐ
bÔil×isÑisÖtøðs	Úús   ¥K( Ë(K-Ú__main__)é   )zDB oprT   é   )Ú__doc__r   rE   r‹   ry   r   Úosr   r   Úbs4r   Úurllib.parser   rƒ   r�   rŽ   r�   r$   r%   r    r   r"   r'   rB   rK   rv   r€   r‡   r•   rº   Ú__name__© r(   r!   ú<module>rÄ      sœ   ðñ÷ 2× 1× 1Ó 1Ý Ý Ý  à
€Ø$€	Ø€Ø€Ø'€Ø€Ø€ð DØOØ0ñ€óò"9òò:	òC1óPòòò(>uðB ˆzÒÙ…Fð r(   