Ë
    ¸¬uj&  ã                   óš   — d Z ddlZddlZddlZddlZddlZddlmZ dZdZ	dZ
dZdZd	d
iZdad„ Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uD  
crawl_ningguo_qtxx.py - å®�å›½å¸‚ç”Ÿæ€�çŽ¯å¢ƒåˆ†å±€-å…¶ä»–ä¿¡æ�¯
CMS: è‡ªå®šä¹‰ASP.NET MVC
åˆ—è¡¨: /XxgkContent/showList/383/22474/page_N.html (15æ�¡/é¡µ, ~10é¡µ)
      ul > li > a[href] + dateæ–‡æœ¬
è¯¦æƒ…: /OpennessContent/show/{id}.html
æ ‡é¢˜: <title>æ ‡ç­¾ (å�–" - "ä¹‹å‰�)
æ—¥æœŸ: <meta PubDate>
æ­£æ–‡: div#zoom > p
é    N)ÚBeautifulSoupz/root/search.dbzhttps://www.ningguo.gov.cnu   å®�å›½å¸‚-å…¶ä»–ä¿¡æ�¯u   å®‰å¾½õ   çŽ¯å¢ƒå…¬ç¤ºz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c                 ó¸  — t         €~t        j                  «       a t         j                  j	                  t
        «       	 t         j                  | dd¬«      }d|j                  v r t         j                  j                  dd«       	 t         j                  | dd¬«      }d|_
        |j                  S #  Y Œ0xY w# t        $ r}t        d|› �«       Y d }~y d }~ww xY w)	Né<   F)ÚtimeoutÚverifyztoken_verified=trueÚtoken_verifiedÚtruezutf-8zERR:)ÚSESSIONÚrequestsÚSessionÚheadersÚupdateÚHEADERSÚgetÚtextÚcookiesÚsetÚencodingÚ	ExceptionÚprint)ÚurlÚrÚes      ú'/root/gov_crawler/crawl_ningguo_qtxx.pyr   r   "   s¶   € ä€Ü×"Ñ"Ó$ˆÜ�‰×ÑœwÔ'ð	Ü—‘˜C¨°E�Ó:ˆAØ$¨¯©Ñ.Ü—‘×#Ñ#Ð$4°fÔ=ðÜ�K‰K˜ R°ˆKÓ6ˆØˆŒ
Ø�v‰vˆøð	Ùûô
 ò Ü��Q�CˆjÔÜûðús$   ¿AB1 Â*B8 Â1B5Â8	CÃCÃCc                 ó0   — t         › d| › d�}t        |«      S )Nz%/XxgkContent/showList/383/22474/page_z.html)ÚBASE_URLr   )Úpage_nor   s     r   Úfetch_list_pager   7   s   € ÜˆJÐ;¸G¸9ÀEÐ
J€CÜˆs‹8€Oó    c                 ó   — g }t        | d«      }|j                  d«      D �]*  }|j                  dd¬«      }dt        |«      cxk  rdk  sn Œ.t        d„ |D «       «      }|d	k\  sŒF|D ]ß  }|j	                  d
d¬«      }|sŒ|j                  dd«      j                  «       }|j                  d¬«      }	t        j                  dd|	«      j                  «       }	|	r|sŒu|j                  d¬«      }
t        j                  d|
«      }|r|j                  d«      nd}|j                  d«      r	t        |z   }|j                  ||	|f«       Œá  |S  |S )z)Extract (url, title, date) from list pageúhtml.parserÚulÚliF)Ú	recursiveé
   é   c              3   ó�   K  — | ]>  }|j                  d d¬«      r)d|j                  d d¬«      j                  dd«      v rd–— Œ@ y­w)ÚaT©Úhrefz/OpennessContent/show/r+   Ú é   N)Úfindr   )Ú.0r$   s     r   ú	<genexpr>zparse_list.<locals>.<genexpr>G   sM   è ø€ ò 
ØØ�w‰w�s ˆwÔ&Ð+CÀrÇwÁwÈsÐY]ÀwÓG^×GbÑGbÐciÐkmÓGnÑ+nô ñ
ùs   ‚AAé   r)   Tr*   r+   r,   ©Ústripz\d{4}-\d{2}-\d{2}$ú(\d{4}-\d{2}-\d{2})r-   ú/)r   Úfind_allÚlenÚsumr.   r   r3   Úget_textÚreÚsubÚsearchÚgroupÚ
startswithr   Úappend)ÚhtmlÚitemsÚsoupr#   ÚlisÚopenness_countr$   r)   r+   ÚtitleÚtxtÚ
date_matchÚdate_strs                r   Ú
parse_listrI   <   sZ  € à€EÜ˜˜}Ó-€Dð �m‰m˜DÓ!ó ˆØ�k‰k˜$¨%ˆkÓ0ˆØ”c˜#“hÔ$ "Ô$Øäñ 
Øô
ó 
ˆð ˜QÓØò 6�Ø—G‘G˜C d�GÓ+�ÙØØ—u‘u˜V RÓ(×.Ñ.Ó0�àŸ
™
¨˜
Ó.�äŸ™Ð4°b¸%Ó@×FÑFÓH�Ù¡DØð —k‘k¨�kÓ-�ÜŸY™YÐ'=¸sÓC�
Ù2<˜:×+Ñ+¨AÔ.À"�à—?‘? 3Ô'Ü# d™?�Dà—‘˜d E¨8Ð4Õ5ð)6ð* Ø€LðAð@ €Lr    c                 ó®  — t        | d«      }d}|j                  d«      }|r:|j                  d¬«      }|j                  d«      }|r|d   j	                  «       n|}d}|j                  dd	d
i«      xs |j                  dd	di«      }|r;|j                  dd«      }	t        j                  d|	«      }
|
r|
j                  d«      }|sJ|j                  d«      }|r7t        j                  d|j                  «       «      }
|
r|
j                  d«      }d}	|j                  dd¬«      }|s|j                  dd¬«      }|rrg }|j                  d«      D ](  }|j                  d¬«      }|sŒ|j                  |«       Œ* |rdj                  |«      }	n |j                  d¬«      j	                  «       }	|	rt        j                  dd|	«      }	|	r|	d d nd}|||	|fS )Nr"   r,   rE   Tr2   z - r   ÚmetaÚnameÚPubDateÚpubdateÚcontentr4   r-   Úbodyu"   å�‘å¸ƒæ—¶é—´ï¼š(\d{4}-\d{2}-\d{2})ÚdivÚzoom)Úidzg-detailbox)Úclass_Úpz

ú
)Ú	separatorz&nbsp;éÈ   )r   r.   r9   Úsplitr3   r   r:   r<   r=   r6   r?   Újoinr;   )r@   r   rB   rE   Ú	title_tagÚrawÚpartsrH   Úpubdate_metarO   ÚmrP   rR   Ú
paragraphsrU   rF   Úsummarys                    r   Úextract_detailrb   e   sÍ  € Ü˜˜}Ó-€Dð €EØ—	‘	˜'Ó"€IÙØ× Ñ  tÐ Ó,ˆØ—	‘	˜%Ó ˆÙ$)��a‘—‘Ô ¨sˆð €HØ—9‘9˜V f¨iÐ%8Ó9Òc¸T¿Y¹YÀvÐPVÐXaÐObÓ=c€LÙØ×"Ñ" 9¨bÓ1ˆÜ�I‰IÐ,¨gÓ6ˆÙØ—w‘w˜q“zˆHáØ�y‰y˜Ó ˆÙÜ—	‘	Ð?ÀÇÁÃÓQˆAÙØŸ7™7 1›:�ð €GØ�9‰9�U˜vˆ9Ó&€DÙà�y‰y˜ }ˆyÓ5ˆáØˆ
Ø—‘˜sÓ#ò 	'ˆAØ—*‘* 4�*Ó(ˆCÚØ×!Ñ! #Õ&ð	'ñ Ø—k‘k *Ó-‰Gà—m‘m¨d�mÓ3×9Ñ9Ó;ˆGáÜ—&‘&˜ B¨Ó0ˆá&ˆg�d�s‰m¨B€GØ�(˜G WÐ,Ð,r    c                 óø  — dd l }|j                  t        d¬«      }|j                  «       }d}g }| D ]†  \  }}}}	}
	 |j	                  d|
f«       |j                  «       rŒ.|j	                  d|
|t        |||	|
dt        t        df«       |j                  }|r|j                  ||t        |	xs df«       |d	z  }Œˆ |j                  «        |rK|D ]  \  }}}}		 |j	                  d||||	f«       Œ  |j                  «        t        dt        |«      › d�«       |j                  «        |S # t        $ r}t        d
|d d › d|› �«       Y d }~�Œd }~ww xY w# t        $ r}t        d|› d|› �«       Y d }~Œ¥d }~ww xY w)Nr   r&   )r   z)SELECT id FROM gov_raw WHERE page_url = ?zíINSERT INTO gov_raw 
                   (page_url, title, site_name, publish_date, content, summary,
                    source_url, category, industry, group_name, script_name)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r   zcrawl_ningguo_qtxx.pyr,   r-   z  [DB ERROR] é   z: zVINSERT OR IGNORE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)z  [FTS ERROR] id=u     FTSå�Œæ­¥: õ   æ�¡)Úsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚfetchoneÚ	SITE_NAMEÚINDUSTRYÚGROUPÚ	lastrowidr?   r   r   Úcommitr7   Úclose)Ú
items_datarf   ÚconnÚcÚinsertedÚ	fts_batchrE   rH   rO   ra   Úpage_urlÚnew_idr   ÚridÚsites                  r   Ú
save_to_dbr{   ™   s›  € ÛØ�?‰?œ7¨Bˆ?Ó/€DØ�‰‹€AØ€HØ€IØ7Aò 5Ñ3ˆˆx˜ '¨8ð	5Ø�I‰IÐAÀHÀ;ÔOØ�z‰zŒ|ØØ�I‰Ið?ð ˜5¤)¨X°wÀØ˜>¬8´UÐ<SðUôð —[‘[ˆFÙØ× Ñ  &¨%´¸GºMÀrÐ!JÔKØ˜‰M‰Hð!5ð& 	‡K�K„MÙØ)2ò 	6Ñ%ˆC�˜˜gð6Ø—	‘	ØlØ˜%  wÐ/õð	6ð 	�‰ŒÜ�œc )›nÐ-¨SÐ1Ô2Ø‡J�J„LØ€Oøô ò 	5Ü�M %¨¨ * ¨R°¨sÐ3×4Ò4ûð	5ûô ò 6ÜÐ)¨#¨¨b°°Ð4×5Ñ5ûð6ús6   ¼#D*Á AD*ÃEÄ*	EÄ3EÅEÅ	E9ÅE4Å4E9c            
      óÜ  — dd l } | j                  t        › d�¬«      }|j                  dt        dd¬«       |j                  dd	d
¬«       |j                  «       }|j                  rdn|j                  }d}t        d|dz   «      D �]7  }t        dt        j                  d«      › d|› d�dd¬«       t        |«      }|st        d«        nõt        |«      }|st        d«        nÛt        t        |«      › d�d¬«       g }|D ]X  \  }	}
}t        |	«      }|sŒt!        ||	«      \  }}}}|j#                  |xs |
|xs ||||	f«       t        j$                  d«       ŒZ t'        |«      }||z  }t        d|› dt        |«      › d|› d�d¬«       |j                  r�Œ|dk(  s�Œt        |«      dkD  s�Œ-t        d«        n t        d«       t        d|› d �«       t        d!t        › �«       y )"Nr   u   çˆ¬è™«)Údescriptionz--pagesr-   u   çˆ¬å�–é¡µæ•°(å…±çº¦10é¡µ))ÚtypeÚdefaultÚhelpz--fullÚ
store_trueu   å…¨é‡�çˆ¬å�–(10é¡µ))Úactionr€   r&   ú[z%H:%M:%Su   ] ç¬¬u   é¡µ...ú T)ÚendÚflushu   è¯·æ±‚å¤±è´¥ï¼Œå�œæ­¢u   æ— æ•°æ�®ï¼Œå�œæ­¢re   )r†   g      à?u     å…¥åº“r5   u   æ�¡ (ç´¯è®¡ú)u      å…¨éƒ¨å·²å­˜åœ¨ï¼Œå¢žé‡�å�œæ­¢u   
===== å®Œæˆ� =====u   æ–°å¢žå…¥åº“: u    æ�¡u   ç«™ç‚¹: )ÚargparseÚArgumentParserrl   Úadd_argumentÚintÚ
parse_argsÚfullÚpagesÚranger   ÚtimeÚstrftimer   rI   r7   r   rb   r?   Úsleepr{   )rˆ   ÚparserÚargsÚ	max_pagesÚ	total_newr   r@   rA   Úbatchr+   rE   rH   Údetail_htmlÚdtÚddrO   ra   Úsaveds                     r   Úmainrœ   Â   sè  € ÛØ×$Ñ$´I°;¸fÐ1EÐ$ÓF€FØ
×Ñ˜	¬°QÐ=XÐÔYØ
×Ñ˜¨Ð<QÐÔRØ×ÑÓ€Dà—i’i‘ T§Z¡Z€IØ€Iä˜˜I¨™MÓ*ó ˆÜ�”$—-‘- 
Ó+Ð,¨E°'°¸&ÐAÀsÐRVÕWÜ˜wÓ'ˆÙÜÐ)Ô*Ùä˜4Ó ˆÙÜÐ&Ô'ÙÜ”�U“�˜CÐ ¨Õ-àˆØ%*ò 	Ñ!ˆD�%˜Ü˜d›)ˆKÙØÜ'5°kÀ4Ó'HÑ$ˆB��G˜WØ�L‰L˜"š+  r¢~¨X°wÀÈÐNÔOÜ�J‰J�s�Oð	ô ˜5Ó!ˆØ�UÑˆ	Ü�˜˜˜q¤ U£ ¨K¸	°{À!ÐDÈDÕQØ�yŒy˜U aœZ¬C°«J¸¬NÜÐ4Ô5Ùð7ô: 
Ð Ô"Ü	ˆN˜9˜+ TÐ
*Ô+Ü	ˆH”Y�KÐ
 Õ!r    Ú__main__)Ú__doc__r   r:   ÚsysÚosr�   Úbs4r   rh   r   rl   rn   rm   r   r   r   r   rI   rb   r{   rœ   Ú__name__© r    r   ú<module>r¤      s†   ðñ	ó Û 	Û 
Û 	Û Ý à
€Ø'€Ø$€	Ø€Ø€ð ð  Dð€ð €òò*ò
&òR1-òh&òR)"ðX ˆzÒÙ…Fð r    