Ë
    ¸¬ujJ  ã                   óÊ   — d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl	Z	 ej                  dd«      ZdZdZdZd	Zd
ZdddœZdd„Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u¸   å¤ªä»“çŽ¯ä¿�å…¬ä¼—ç½‘-çŽ¯è¯„å…¬ç¤ºçˆ¬è™«
ç«™ç‚¹: http://www.tchbgz.com/
æ �ç›®: çŽ¯è¯„å…¬ç¤º (?hpgs/)
åˆ†é¡µ: /?hpgs_N/ (N=2..11), 15æ�¡/é¡µ, å…±11é¡µçº¦165æ�¡
CMS: è‡ªå®šä¹‰ä¼�ä¸šCMS
é    N)ÚBeautifulSoupÚ	SEARCH_DBz/mnt/data/search.dbzhttp://www.tchbgz.comé   u"   å¤ªä»“çŽ¯ä¿�å…¬ä¼—ç½‘-çŽ¯è¯„å…¬ç¤ºu   çŽ¯è¯„å…¬ç¤ºu   ä¼�ä¸šz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36zzh-CN,zh;q=0.9)z
User-AgentzAccept-Languagec                 ó<  — t        |«      D ]3  }	 t        j                  | t        d¬«      }d|_        |j
                  c S  y # t        $ rO}||dz
  k  rt        j                  d«       n't        d| › d|› �t        j                  ¬«       Y d }~ y Y d }~ŒŠd }~ww xY w)	Né   )ÚheadersÚtimeoutzutf-8é   é   z
  [ERROR] ú: ©Úfile)ÚrangeÚrequestsÚgetÚHEADERSÚencodingÚtextÚ	ExceptionÚtimeÚsleepÚprintÚsysÚstderr)ÚurlÚretriesÚiÚrÚes        ú&/root/gov_crawler/crawl_tchbgz_hpgs.pyÚfetchr!      s‡   € Ü�7‹^ò 
ˆð		Ü—‘˜S¬'¸2Ô>ˆAØ ˆAŒJØ—6‘6ŠMñ	
øô
 ò 	Ø�7˜Q‘;ŠÜ—
‘
˜1•ä˜
 3 % r¨!¨Ð-´C·J±JÕ?Ýô ûð	ús   �.AÁ	BÁ?BÂBc                 ó  — g }t        | d«      }|j                  d«      D ]ç  }|j                  d«      }|j                  d«      }|sŒ(|j                  dd«      }|rd|vrŒA|j	                  d«      s	t
        |z   }|j                  d	d«      xs |j                  d
¬«      }|sŒ„|r|j                  d
¬«      nd}t        j                  d|«      }	|	r|	j                  d«      nd}
|j                  ||j                  «       |
dœ«       Œé |S )u9   è§£æž�åˆ—è¡¨é¡µï¼Œè¿”å›ž [ {page_url, title, date}, ... ]úhtml.parserzli.pagearticlelistzdiv.pagelistleft azdiv.pagelistrightÚhrefÚ ÚhpgsÚhttpÚtitleT©Ústripz(\d{4}-\d{1,2}-\d{1,2})r
   )Úpage_urlr(   Údate)r   ÚselectÚ
select_oner   Ú
startswithÚBASE_URLÚget_textÚreÚsearchÚgroupÚappendr*   )ÚhtmlÚitemsÚsoupÚliÚleftÚrightr$   r(   Údate_strÚdmÚpublish_dates              r    Ú
parse_listr?   (   s  € à€EÜ˜˜}Ó-€Dà�k‰kÐ.Ó/ò ˆØ�}‰}Ð1Ó2ˆØ—‘Ð1Ó2ˆÙØØ�x‰x˜ Ó#ˆÙ�v TÑ)ØØ�‰˜vÔ&Ü˜d‘?ˆDà—‘˜ "Ó%ÒB¨¯©¸T¨Ó)BˆÙØá16�5—>‘>¨�>Ô-¸Bˆä�Y‰YÐ1°8Ó<ˆÙ&(�r—x‘x ”{¨bˆà�‰ØØ—[‘[“]Ø ñ
õ 	ð)ð2 €Ló    c                 óJ  — t        | d«      }d}d}d}d}|j                  d«      }|rJ|j                  d¬«      }|j                  d«      d   j                  d«      d   j	                  «       }|r|}|j                  «       }	t        j                  d	|	«      }
|
r|
j                  d
«      }|j                  d«      }|s|j                  d«      }|rÎ|j                  d«      }g }|D ],  }|j                  «        |j                  t        |«      «       Œ. g }|j                  d«      D ](  }|j                  d¬«      }|sŒ|j                  |«       Œ* |s|s|j                  d¬«      }n*dj                  |«      }|r|ddj                  |«      z   z  }t        |«      }||||fS )uJ   è§£æž�è¯¦æƒ…é¡µï¼Œè¿”å›ž (title, publish_date, content_text, content_html)r#   r%   r(   Tr)   u   -çŽ¯è¯„å…¬ç¤ºr   ú-u-   å�‘å¸ƒæ—¶é—´[ï¼š:]?\s*(\d{4}-\d{1,2}-\d{1,2})r
   zdiv.articleshowzdiv.a-content-pagearticleÚtableÚpú

ú
)r   Úfindr1   Úsplitr*   r2   r3   r4   r.   Úfind_allÚextractr5   ÚstrÚjoin)r6   r   r8   r(   r>   ÚcontentÚcontent_htmlÚ	title_tagÚtÚ	body_textr=   Úcontent_divÚtablesÚtable_htmlsÚ
paragraphsrD   r   s                    r    Úparse_detailrV   I   s­  € ä˜˜}Ó-€DØ€EØ€LØ€GØ€Lð —	‘	˜'Ó"€IÙØ
×
Ñ
 4Ð
Ó
(€aØ
�'‰'�/Ó
" 1Ñ
%×
+Ñ
+¨CÓ
0°Ñ
3×
9Ñ
9Ó
;€aÙ	
Øˆð —‘“€IÜ	�‰ÐCÀYÓ	O€BÙ	Ø—X‘X˜a“[€lð —/‘/Ð"3Ó4€KÙØ—O‘OÐ$?Ó@€káà×#Ñ# GÓ,€fØ€kØò #ˆ!Ø	�	‰	ŒØ×Ñœ3˜q›6Õ"ð#ð
 €jØ×#Ñ# CÓ(ò "ˆ!à�z‰z ˆzÓ%ˆÚØ
×
Ñ
˜DÕ
!ð	"ñ ¡à×&Ñ&¨TÐ&Ó2‰à—+‘+˜jÓ)ˆÙØ
�V˜dŸi™i¨Ó4Ñ4Ñ
4ˆ'ä˜Ó%€là�, ¨Ð5Ð5r@   c            	      ó  — t        t        j                  «      dkD  rt        j                  d   nt        t        «      } 	 t        | «      }t        dt        › d|› �d¬«       g }t        › d�}t        |«      }|st        dt        j                  ¬«       y t        |«      }t        d	t        |«      › d
�d¬«       |j                  |«       t        d|dz   «      D ]j  }t        › d|› d�}t        |«      }|s nOt        |«      }|s n@t        d|› dt        |«      › d
�d¬«       |j                  |«       t        |«      dk  sŒj n t        dt        |«      › �d¬«       d}d}d}	|D ]ž  }
|
d   }t        d|
d   d d › d�d¬«       t        |«      }|s|	dz  }	Œ2t        ||«      \  }}}}|s|
j!                  dd«      }|s|
j!                  dd«      }|s|	dz  }	Œs	 t#        |||||¬«      }|dk(  r|dz  }n|dk(  r|dz  }n|	dz  }	Œ  t        dt        › d�d¬«       t        d |› d!|› d"|	› �d¬«       y # t        $ r
 t        }Y �Œýw xY w# t$        $ r.}t        d|› �t        j                  ¬«       |	dz  }	Y d }~�Œd }~ww xY w)#Nr
   ú[z] Starting crawl, max_pages=T)Úflushz/?hpgs/z[ERROR] Cannot fetch page 1r   z
  Page 1: z itemsr   z/?hpgs_ú/z  Page r   é   zTotal list items: r   r+   z  Fetching: r(   é(   z...r,   r%   )r+   r(   r>   rM   rN   ÚnewÚdupz  [ERROR] DB insert: z
=== z	 Done ===zNew: z, Dup: z	, Error: )Úlenr   ÚargvrK   Ú	MAX_PAGESÚintÚ
ValueErrorr   Ú	SITE_NAMEr0   r!   r   r?   Úextendr   rV   r   Ú
save_to_dbr   )Úmax_pages_strÚ	max_pagesÚ	all_itemsr   r6   r7   ÚpageÚ	new_countÚ	dup_countÚerror_countÚitemr+   Údetail_htmlr(   r>   rM   rN   Úresultr   s                      r    Úcrawlrq   ‚   sÃ  € Ü#&¤s§x¡x£=°1Ò#4”C—H‘H˜Q’K¼#¼i».€MðÜ˜Ó&ˆ	ô 
ˆAŒiˆ[Ð4°Y°KÐ
@ÈÕMØ€Iô ˆJ�gÐ
€CÜ�‹:€DÙÜÐ+´#·*±*Õ=ØÜ�tÓ€EÜ	ˆJ”s˜5“z�l &Ð
)°Õ6Ø×Ñ�UÔô �a˜ Q™Ó'ò ˆÜ�
˜' $  qÐ)ˆÜ�S‹zˆÙÙÜ˜4Ó ˆÙÙÜ�˜�v˜R¤ E£
˜|¨6Ð2¸$Õ?Ø×Ñ˜ÔÜˆu‹:˜‹?Ùðô 
Ðœs 9›~Ð.Ð
/°tÕ<ð €IØ€IØ€Kàò %ˆØ˜
Ñ#ˆÜ�˜T '™]¨3¨BÐ/Ð0°Ð4¸DÕAä˜H“oˆÙØ˜1ÑˆKØä5AÀ+ÈxÓ5XÑ2ˆˆ|˜W lñ ØŸ8™8 F¨BÓ/ˆLÙØ—H‘H˜W bÓ)ˆEÙØ˜1ÑˆKØð	ÜØ!ØØ)ØØ)ôˆFð ˜ŠØ˜Q‘‘	Ø˜5’Ø˜Q‘‘	à˜qÑ �øðE%ôN 
ˆF”9�+˜YÐ
'¨tÕ4Ü	ˆE�)�˜G I ;¨i¸°}Ð
EÈTÖRøô[ ò Ü‹	ðûôP ò 	ÜÐ)¨!¨Ð-´C·J±JÕ?Ø˜1ÑŽKûð	ús*   Á H6 Ç+IÈ6I	ÉI	É	JÉ#I>É>Jc                 óü  — d| j                  t        d«      t        d«      t        d«      z   «      › d�}t        j                  dt        |gddd¬«      }|j
                  j                  «       ryd	„ }|r%t        j                  d
d|j                  «       «      }|sd}|r|dd nd}d || «      › d ||«      › d ||«      › d ||«      › d |t        «      › dt        › d ||«      › d�}	t        j                  dt        |	gddd¬«      }|j                  dk7  r7d|j                  vr)t        d|j                  › �t        j                  ¬«       yd || «      › d�}
t        j                  dt        |
gddd¬«       y)u   æ�’å…¥åˆ°search.dbz,SELECT rowid FROM gov_raw WHERE page_url = 'é'   ú'Úsqlite3Té
   )Úcapture_outputr   r	   r^   c                 ó.   — | r| j                  dd«      S dS )Nrt   z''r%   )Úreplace)Úss    r    Úesczsave_to_db.<locals>.escâ   s   € Ù'(ˆq�y‰y˜˜dÓ#Ð0¨bÐ0r@   z\s{3,}rE   r%   NéÈ   zgINSERT INTO gov_raw (page_url, title, publish_date, content, site_name, industry, summary)
VALUES (
  'z',
  'z'
)r   ÚUNIQUEz  [DB ERROR] r   Úerrorz�INSERT INTO gov_search (rowid, title, site_name, summary)
SELECT rowid, title, site_name, summary
FROM gov_raw WHERE page_url = 'z1' AND rowid NOT IN (SELECT rowid FROM gov_search)r]   )ry   ÚchrÚ
subprocessÚrunÚDB_PATHÚstdoutr*   r2   Úsubrd   ÚINDUSTRYÚ
returncoder   r   r   )r+   r(   r>   rM   rN   Ú	check_sqlrp   r{   ÚsummaryÚsqlÚsync_sqls              r    rf   rf   Ö   sª  € ð ?¸x×?OÑ?OÔPSÐTVÓPWÔY\Ð]_ÓY`ÔadÐegÓahÑYhÓ?iÐ>jÐjkÐl€IÜ�^‰^Ø	”G˜YÐ'Ø $°ô€Fð ‡}�}×ÑÔØò1ñ Ü—&‘&˜ F¨G¯M©M«OÓ<ˆÙØˆá&ˆg�d�s‰m¨B€GðáˆƒM€?ð ÙˆƒJ€<ð ÙˆÓÐð ÙˆƒL€>ð ÙŒ	ƒNÐð Ü€:ð ÙˆƒL€>ð ð	€Cô �^‰^Ø	”G˜SÐ!Ø $°ô€Fð ×Ñ˜AÒ (°&·-±-Ñ"?Ü�˜fŸm™m˜_Ð-´C·J±JÕ?Øð á # H£˜Ð._ðc€Hô ‡N�NØ	”G˜XÐ&Ø $°õð
 r@   Ú__main__)é   )Ú__doc__Úosr2   r   r   Újsonr€   Úbs4r   r   Úgetenvr‚   r0   ra   rd   r…   ÚGROUPr   r!   r?   rV   rq   rf   Ú__name__© r@   r    ú<module>r•      sŠ   ðñ÷ +× *× *Ý Û à
ˆ"�)‰)�KÐ!6Ó
7€Ø"€Ø€	Ø0€	Ø€Ø€ð QØ'ñ€óòòB66òrQSòh4ðn ˆzÒÙ	…Gð r@   