Ë
    ¸¬uj  ã                   óœ  — d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
  ej                  dd«      ZdZedz   Zed	z   Z ej                   «        ed
¬«      z
  j#                  d«      ZdZdZddiZ ede› �«       d„ Zd„ Zd„ Z ee«      Z ee«      Z ed ee«      › d�«       dZ ej<                  de«      Zer% e ejC                  d«      «      Z"dZ ede› �«       n ed«       eZ# e$dedz   «      D ]_  Z%edz   e%z
  Z&e› de&› d	�Z' ee'«      Z(e(r1 ee(«      Z)e#jU                  e)«        ede%› de&› d  ee)«      › d�«        ejV                  d!«       Œa  ed" ee#«      › �«       e#D � ��cg c]  \  } }}|ek\  sŒ| ||f‘Œ c}}} Z, ee#«       ee,«      z
  Z- ed# ee,«      › d$e-› �«       ddl.Z. e.j^                  ed%¬&«      Z0e0jc                  «       Z2dxZ3xZ4Z5 e6e,d«      D �]]  \  Z7\  Z8Z9Z:e9jw                  d'«      re9Z<n9e9jw                  d(«      r	ee9dd z   Z<ne9jw                  d«      ree9z   Z<nedz   e9z   Z<	 e2j{                  d)e<ef«       e2j}                  «       re4dz  Z4Œƒ ee<«      \  Z?Z@ZAe?se8Z?e@se:Z@e@rd*e@v r e e@j…                  d*d+«      «      ndZCd+ZDeAr  e
eAd,«      ZEeEj�                  d-¬.«      ZGeGdd/ ZDe2j{                  d0ee?e<e@eAeCeeDf«       e2j�                  dkD  re3dz  Z3e0j—                  «         ejV                  d5«       e7d6z  dk(  se7 ee,«      k(  s�Œ@ ed7e7› d ee,«      › d8e3› d9e4› d:e5› d;�«       �Œ` e0j™                  «         ed<e3› d9e4› d:e5› d;�«        e ejš                  ee3e4e5 ee,«      d=œd>¬?«      «       yc c}}} w # eI$ r#ZJe5dz  Z5 ed1e7› d2e8dd3 › d4eJ› �«       Y dZJ[JŒËdZJ[Jww xY w)@u  
crawl_hhpb.py - å±�è¾¹è‹—æ—�è‡ªæ²»åŽ¿äººæ°‘æ”¿åºœ-å…¬ç¤ºå…¬å‘Š
https://www.hhpb.gov.cn/zfxx/gsgg.htm
CMS: VSB 9 (Visual SiteBuilder 9)
Detail: div#vsb_content, meta ArticleTitle/PubDate, list ul.dot.ulist
Pagination: gsgg/{N}.htm (reverse numbering: page N -> gsgg/{6-N}.htm)
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupÚ	SEARCH_DBz/root/search.dbzhttps://www.hhpb.gov.cnz
/zfxx/gsggz.htmiG  )Údaysz%Y-%m-%du   å±�è¾¹åŽ¿æ”¿åºœ-å…¬ç¤ºå…¬å‘Šu   çŽ¯ä¿�å…¬ç¤ºz
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36z[hhpb] 3-year cutoff: c                 ó²   — 	 t        j                  | t        dd¬«      }d|_        |j                  S # t
        $ r}t        d| › d|› �«       Y d }~yd }~ww xY w)Né   F)ÚheadersÚtimeoutÚverifyzutf-8z[hhpb] fetch fail ú: Ú )ÚrequestsÚgetÚHEADERSÚencodingÚtextÚ	ExceptionÚprint)ÚurlÚrespÚes      ú/root/gov_crawler/crawl_hhpb.pyÚfetchr      sU   € ðÜ�|‰|˜C¬¸"ÀUÔKˆØˆŒØ�y‰yÐøÜò ÜÐ" 3 % r¨!¨Ð-Ô.Üûðús   ‚/2 ²	A»AÁAc                 óJ  — g }t        j                  d| t         j                  «      }|s|S |j                  d«      }t        j                  d|t         j                  «      }|D ]¼  }t        j                  d|«      }t        j                  d|«      }t        j                  d|«      }|sŒH|sŒK|sŒN|j                  d«      }	t        |j                  d«      d«      j                  d¬	«      }
|j                  d«      j                  «       }|j                  |
|	|f«       Œ¾ |S )
z!Parse ul.dot.ulist from list pagez <ul class="dot ulist">(.*?)</ul>é   z<li>(.*?)</li>zhref="([^"]*)"z!<div class="h4 eclip">(.*?)</div>z<div class="time">(.*?)</div>úhtml.parserT©Ústrip)	ÚreÚsearchÚDOTALLÚgroupÚfindallr   Úget_textr   Úappend)ÚhtmlÚitemsÚmÚul_htmlÚlisÚliÚhref_mÚtitle_mÚdate_mÚhrefÚtitleÚdate_strs               r   Ú
parse_listr3   "   sô   € à€EÜ
�	‰	Ð5°t¼R¿Y¹YÓG€AÙØˆØ�g‰g�a‹j€GÜ
�*‰*Ð&¨´·±Ó
;€CØò 2ˆÜ—‘Ð,¨bÓ1ˆÜ—)‘)Ð@À"ÓEˆÜ—‘Ð;¸RÓ@ˆÚ’g¢&Ø—<‘< “?ˆDÜ! '§-¡-°Ó"2°MÓB×KÑKÐRVÐKÓWˆEØ—|‘| A“×,Ñ,Ó.ˆHØ�L‰L˜%  xÐ0Õ1ð2ð €Ló    c                 óh  — t        | «      }|syt        |d«      }d}|j                  dddi¬«      }|r$|j                  d«      r|d   j	                  «       }|s%|j                  d	«      }|r|j                  d
¬«      }d}|j                  dddi¬«      }|r=|j                  d«      r,t        j                  d|d   «      }|r|j                  d«      }g }	|j                  d«      }
|
r|	j                  t        |
«      «       |j                  dd¬«      }|r	 |	rdj                  |	«      nd}|||fS )z0Extract title, date, body from VSB 9 detail page)NNNr   r   ÚmetaÚnameÚArticleTitle)ÚattrsÚcontentÚh1Tr   ÚPubDatez(\d{4}-\d{2}-\d{2})r   zdiv[id^="vsb_content"]Údivzxxgk-con)Úclass_ú
)r   r   Úfindr   r   r%   r    r!   r#   Ú
select_oner&   ÚstrÚfind_allÚjoin)r   r'   Úsoupr1   Úmtr;   r2   Úmdr)   Ú
body_partsÚvsbÚ	xxgk_consÚbodys                r   Ú
get_detailrL   5   s5  € ä�‹:€DÙØä˜˜}Ó-€Dð €EØ	�‰�6 &¨.Ð!9ˆÓ	:€BÙ	ˆb�f‰f�YÔØ�9‘×#Ñ#Ó%ˆÙØ�Y‰Y�t‹_ˆÙØ—K‘K d�KÓ+ˆEð €HØ	�‰�6 &¨)Ð!4ˆÓ	5€BÙ	ˆb�f‰f�YÔÜ�I‰IÐ,¨b°©mÓ<ˆÙØ—w‘w˜q“zˆHð €Jà
�/‰/Ð2Ó
3€CÙ
Ø×Ñœ#˜c›(Ô#ð —‘˜e¨J�Ó7€Iáð 	á$.ˆ4�9‰9�ZÔ °B€DØ�(˜DÐ Ð r4   z[hhpb] Page 1: z itemsé   u>   class="p_last p_fun"[^>]*><a href="gsgg/(\d+)\.htm">å°¾é¡µ</a>r   z[hhpb] Total pages: z[hhpb] Defaulting to 1 pageé   ú/z[hhpb] Page z (z.htm): g      à?z#[hhpb] Total items from all pages: z[hhpb] Within 3yr: z	, older: é   )r   Úhttpz../z7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?ú-r   r   Tr   éÈ   z�INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)z	  ERROR [z] é(   r   g333333Ó?é
   z  Progress z: +z new, z dup, z errz
[hhpb] Summary: +)ÚsiteÚnewÚdupÚerrÚtotalF)Úensure_ascii)NÚ__doc__Úosr    ÚsysÚjsonÚtimer   r   r   Úbs4r   ÚgetenvÚDB_PATHÚBASEÚ	LIST_BASEÚ
LIST_PAGE1ÚnowÚstrftimeÚCUTOFF_DATEÚ	SITE_NAMEÚCATEGORYr   r   r   r3   rL   Úhtml1r(   ÚlenÚtotal_pagesr!   Úpag_mÚintr#   Úlast_page_numÚ	all_itemsÚrangeÚpage_idxÚurl_numÚpage_urlÚhÚ
page_itemsÚextendÚsleepÚactiveÚskippedÚsqlite3ÚconnectÚconnÚcursorÚcurÚ	new_countÚ	dup_countÚerror_countÚ	enumerateÚidxr1   r0   r2   Ú
startswithÚ
detail_urlÚexecuteÚfetchoneÚd_titleÚd_daterK   ÚreplaceÚ	date_rankÚsummaryÚ	text_soupr%   ÚplainÚrowcountr   r   ÚcommitÚcloseÚdumps)ÚtÚuÚds   000r   ú<module>r™      s¡  ðñ÷ )× (× (ß (Ý à
ˆ"�)‰)�KÐ!2Ó
3€Ø €Ø�<Ñ€	Ø˜Ñ€
Øˆx�|‰|‹~¡	¨uÔ 5Ñ5×?Ñ?À
ÓK€Ø*€	Ø€àÐWÐ
X€á Ð˜{˜mÐ,Ô -òòò&+!ñ\ 	ˆjÓ€Ù�5Ó€Ù ˆ™˜E›
�| 6Ð*Ô +ð €Øˆ�	‰	ÐSÐUZÓ[€ÙÙ˜Ÿ™ A›Ó'€MØ€KÙ	Ð   Ð
.Õ/á	Ð'Ô)ð €	Ù�a˜ q™Ó)ò 	€Hà˜A‰o Ñ(€GØ�˜A˜g˜Y dÐ+€HÙˆh‹€AÙÙ “]ˆ
Ø×Ñ˜Ô$Ù�˜X˜J b¨¨	°¹¸Z»Ð8IÈÐPÔQØ€D‡J�Jˆs…Oð	ñ Ð+©C°	«NÐ+;Ð<Ô =ð $-×	AÐ	A™˜˜1˜a°°[Ó0@ˆ1ˆa�Š)Ô	A€Ù
ˆi‹.™3˜v›;Ñ
&€Ù Ð™C ›K˜=¨	°'°Ð;Ô <ó Ø€w‡��w¨Ô+€Ø
‡k�kƒm€à&'Ð '€	Ð 'ˆI˜á$-¨f°aÓ$8ó /gÑ €CÑ	 ˆ%��xð ‡��vÔØ‰
Ø	�‰˜Ô	Ø˜D  ˜H‘_‰
Ø	�‰˜Ô	Ø˜D‘[‰
à˜C‘Z $Ñ&ˆ
ð4Ø�‰ÐMÐPZÐ\eÐOfÔgØ�<‰<Œ>Ø˜‰NˆIØá *¨:Ó 6Ñˆ�˜ÙØˆGÙØˆFá4:¸sÀf¹}‘C˜Ÿ™ s¨BÓ/Ô0ÐRSˆ	ØˆÙÙ% d¨MÓ:ˆIØ×&Ñ&¨TÐ&Ó2ˆEØ˜D˜S�kˆGà�‰ð.ð ˜ ¨V°T¸9ÀhÐPWÐXô		
ð �<‰<˜!ÒØ˜‰NˆIð
 	‡K�K„MØ€D‡J�Jˆs„Oà
ˆR�x�1‚}˜™s 6›{Ô*Ù�˜C˜5 ¡# f£+ ¨c°)°¸FÀ9À+ÈVÐT_ÐS`Ð`dÐeÖfð_/gðb ‡
�
„Ù Ð˜I˜; f¨Y¨K°v¸k¸]È$ÐOÔ PÙ €j€d‡j�j˜)¨I¸iÐP[ÑfiÐjpÓfqÑrð  BGô  Hõ  Iùô} 
Bøðd ò 4Ø�qÑˆÙ�	˜#˜˜b  s¨  ¨B¨q¨cÐ2×3Ñ3ûð4ús+   Å;NÆ
NÉ)N#É5BN#Î#OÎ(OÏO