Ë
    ¸¬ujå  ã                   óŠ  — d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
  ej                  dd«      ZdZdZed	z   Zed
z   Z ej"                  «        ed¬«      z
  j%                  d«      Zdedz   dœZ ej*                  «       Zej.                  j1                  e«       d„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u,  
crawl_hexgroup.py â€” æ²³åŒ—æƒ å°”ä¿¡æ–°æ��æ–™è‚¡ä»½æœ‰é™�å…¬å�¸-èµ„æ–™å…¬ç¤º
Site: https://www.hexgroup.cn/news/4/
CMS: ä¸­ä¼�åŠ¨åŠ› SaaS (portal-saas)
List: div.cbox-2.p_loopitem > p.e_text-5 > a
Detail: div.e_richText-23.s_title.clearfix
Pagination: /news_list/1777254464689545216-{from}-10.html
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupÚ	SEARCH_DBz/root/search.dbu   hexgroup.cn-èµ„æ–™å…¬ç¤ºzhttps://www.hexgroup.cnz/news/4/z)/news_list/1777254464689545216-{}-10.htmliG  )Údaysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36ú/)z
User-AgentÚRefererc                 ó¦   — 	 t         j                  | d¬«      }d|_        |j                  S # t        $ r}t        d| › d|› �«       Y d }~y d }~ww xY w)Né   )Útimeoutzutf-8z  [ERR] fetch failed: z - )ÚsessionÚgetÚencodingÚtextÚ	ExceptionÚprint)ÚurlÚrÚes      ú#/root/gov_crawler/crawl_hexgroup.pyÚfetchr      sR   € ðÜ�K‰K˜ RˆKÓ(ˆØˆŒ
Ø�v‰vˆøÜò ÜÐ& s e¨3¨q¨cÐ2Ô3Üûðús   ‚), ¬	AµAÁAc                 ó–  — g }t        | d«      }|j                  d«      D ]¦  }|j                  d«      }|sŒ|j                  dd«      }|j	                  d¬«      }|r|sŒ@|j                  d«      s	t        |z   }|j                  d	«      }d}|r$|j	                  d¬«      }|j                  d
d«      }|j                  |||f«       Œ¨ |S )z6Parse list page, return list of (url, title, date_str)úhtml.parserzdiv.cbox-2.p_loopitemzp.e_text-5.s_title a[href]ÚhrefÚ T©ÚstripÚhttpzp.e_timeFormat-9.s_titler   ú-)	r   ÚselectÚ
select_oner   Úget_textÚ
startswithÚBASE_URLÚreplaceÚappend)	ÚhtmlÚitemsÚsoupÚitem_divÚar   ÚtitleÚdate_elÚdate_strs	            r   Ú
parse_listr/   (   sÚ   € à€EÜ˜˜}Ó-€DØ—K‘KÐ 7Ó8ò .ˆà×ÑÐ <Ó=ˆÙØØ�u‰u�V˜RÓ ˆØ—
‘
 �
Ó&ˆÙ™5ØØ�‰˜vÔ&Ü˜d‘?ˆDð ×%Ñ%Ð&@ÓAˆØˆÙØ×'Ñ'¨dÐ'Ó3ˆHà×'Ñ'¨¨SÓ1ˆHà�‰�d˜E 8Ð,Õ-ð).ð* €Ló    c                 óÈ  — t        | d«      }|j                  d«      }d}|r7|j                  d¬«      }t        j                  dd|«      j                  «       }|j                  d«      }d}|rt        |«      }|j                  «       }d}t        j                  d| «      }	|	r|	j                  d	«      }|s)t        j                  d
| «      }	|	r|	j                  d	«      }|||fS )z8Parse detail page, return (title, publish_date, content)r   r,   r   Tr   u=   \s*[-â€“â€”]\s*æ²³åŒ—æƒ å°”ä¿¡æ–°æ��æ–™è‚¡ä»½æœ‰é™�å…¬å�¸\s*$z"div.e_richText-23.s_title.clearfixz(\d{4}-\d{2}-\d{2})\s*</P>é   z(\d{4}-\d{2}-\d{2})\s*</p>)
r   Úfindr"   ÚreÚsubr   r!   ÚstrÚsearchÚgroup)
r'   r   r)   Ú	title_tagr,   ÚrawÚcontent_divÚcontentr.   Úms
             r   Úparse_detailr>   D   sÞ   € ä˜˜}Ó-€Dð —	‘	˜'Ó"€IØ€EÙØ× Ñ  tÐ Ó,ˆä—‘ÐWÐY[Ð]`Óa×gÑgÓiˆð —/‘/Ð"FÓG€KØ€GÙÜ�kÓ"ˆØ—-‘-“/ˆð €Hä
�	‰	Ð/°Ó6€AÙØ—7‘7˜1“:ˆáÜ�I‰IÐ3°TÓ:ˆÙØ—w‘w˜q“zˆHà�(˜GÐ#Ð#r0   c                  ó<  — t        t        «      } | syt        | d«      }t        «       }|j	                  dd¬«      D ]I  }|d   }t        j                  d|«      }|sŒ!|j                  t        |j                  d«      «      «       ŒK |syt        |«      }|dz  dz   S )	z(Get total pages by checking page 1 linksr2   r   r+   T)r   r   z/news_list/\d+-(\d+)-\d+\.htmlé
   )r   ÚLIST_URLr   ÚsetÚfind_allr4   r7   ÚaddÚintr8   Úmax)r'   r)   Úfrom_valuesr+   r   r=   Úmax_froms          r   Úget_total_pagesrI   f   s–   € ä”‹?€DÙØä˜˜}Ó-€DÜ“%€KØ�]‰]˜3 Tˆ]Ó*ò -ˆØ�‰yˆÜ�I‰IÐ7¸Ó>ˆÚØ�O‰OœC §¡¨£
›OÕ,ð	-ñ Øä�;Ó€HØ˜‰N˜aÑÐr0   c                  ó¤  — t        dt        › d�«       t        dt        › �«       t        «       } t        d| › �«       t	        j
                  t        «      }|j                  d«       |j                  d«       |j                  «       }d}d}t        d| dz   «      D �]ê  }|dz
  d	z  }|dkD  rt        j                  |«      nt        }t        d
|› d| › d|› �«       t        |«      }|st        d|› d�«       Œ]t        |«      }	|	st        d|› d�«       Œzt        dt        |	«      › d�«       d}
|	D �]  \  }}}|j                  d|f«       |j!                  «       r|dz  }Œ1|r|t        k  r|dz  }ŒBt        |«      }|s|dz  }ŒUt#        ||«      \  }}}|s|}|xs |}|s|dz  }Œw|t        k  r|dz  }Œ†|j%                  «       }|s|dz  }Œžd}|r#t'        |d«      }|j)                  d¬«      }|d d }	 |j                  dt        ||||t+        |j-                  dd«      «      df«       |j.                  dkD  r
|
dz  }
|dz  }�Œ |j3                  «        t        d|› d|
› d|› d|› d �	«       t5        j6                  d!«       �Œí |j9                  «        t        d"|› d#|› d$�«       y # t0        $ r}t        d|› �«       Y d }~�Œ˜d }~ww xY w)%Nz=== z ===zCutoff: zTotal pages: zPRAGMA journal_mode=WALzPRAGMA busy_timeout=10000r   r2   r@   z
Page r   z: z  [SKIP] page z fetch failedz	 no itemsz  Found z itemsz)SELECT id FROM gov_raw WHERE page_url = ?r   r   Tr   éÈ   zƒINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category) VALUES (?, ?, ?, ?, ?, ?, ?)r   Útzggz  [ERR] insert: z  Page z: +z new (total z, skip ú)g      à?z
=== Done: z new, z skipped ===)r   Ú	SITE_NAMEÚCUTOFFrI   Úsqlite3Úconnectr   ÚexecuteÚcursorÚrangeÚPAGE_URL_TPLÚformatrA   r   r/   ÚlenÚfetchoner>   r   r   r"   rE   r%   Úrowcountr   ÚcommitÚtimeÚsleepÚclose)Útotal_pagesÚconnÚcurÚ	total_newÚ
total_skipÚpageÚfrom_valÚlist_urlr'   r(   Úpage_newr   r,   r.   Údetail_htmlÚ
full_titleÚdetail_dater<   Ú
final_dateÚsummaryÚ	text_soupÚplainr   s                          r   Úmainrn   {   s  € Ü	ˆD”�˜4Ð
 Ô!Ü	ˆH”V�HÐ
Ôä!Ó#€KÜ	ˆM˜+˜Ð
'Ô(ä�?‰?œ9Ó%€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø
�+‰+‹-€Cà€IØ€Jä�a˜ q™Ó)ó PˆØ˜1‘H ‘?ˆØ48¸1²H”<×&Ñ& xÔ0Ä(ˆÜ�˜�v˜Q˜{˜m¨2¨h¨ZÐ8Ô9ä�X‹ˆÙÜ�N 4 &¨Ð6Ô7Øä˜4Ó ˆÙÜ�N 4 &¨	Ð2Ô3Øä�œ˜U›˜ FÐ+Ô,àˆØ$)ó 9	.Ñ ˆC�˜à�K‰KÐCÀcÀVÔLØ�|‰|Œ~Ø˜a‘�
Øñ ˜H¤vÒ-Ø˜a‘�
Øô   ›*ˆKÙØ˜a‘�
Øä/;¸KÈÓ/MÑ,ˆJ˜ WáØ"�
ð %Ò0¨ˆJÙØ˜a‘�
ØØœFÒ"Ø˜a‘�
Øð —m‘m“oˆGÙØ˜a‘�
Øð ˆGÙÜ)¨'°=ÓA�	Ø!×*Ñ*°Ð*Ó6�Ø  ˜+�ð.Ø—‘ð3ô  
¨C°¸WÜ˜×+Ñ+¨C°Ó4Ó5°vð?ô	ð —<‘< !Ò#Ø ‘M�HØ ‘N�Iùðo9	.ðv 	�‰ŒÜ�˜�v˜S  
¨,°y°kÀÈÈÐTUÐVÔWä�
‰
�3ŽðaPðd 	‡J�J„LÜ	ˆL˜˜ 6¨*¨°\Ð
BÕCøô ò .ÜÐ(¨¨Ð,×-Ò-ûð.ús   Ç8AJ-Ê-	KÊ6K
Ë
KÚ__main__)Ú__doc__r4   ÚosÚsysr[   rP   Úrequestsr   r   Úbs4r   Úgetenvr   rN   r$   rA   rU   ÚnowÚstrftimerO   ÚHEADERSÚSessionr   ÚheadersÚupdater   r/   r>   rI   rn   Ú__name__© r0   r   ú<module>r~      sØ   ðñ÷ ,× +× +ß (Ý àˆB�I‰I�kÐ#4Ó5€	Ø&€	Ø$€Ø�jÑ €ØÐEÑE€Ø
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€ð DØ˜#‰~ñ€ð
 ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ òòò8$òD ò*bDðJ ˆzÒÙ…Fð r0   