Ë
    ¸¬ujR  ã                   óŠ  — d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
  ej                  dd«      ZdZdZed	z   Zed
z   Z ej"                  «        ed¬«      z
  j%                  d«      Zdedz   dœZ ej*                  «       Zej.                  j1                  e«       d„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u  
crawl_henanlt.py â€” æ²³å�—è“�å¤©çŽ¯å¢ƒå·¥ç¨‹æœ‰é™�å…¬å�¸-å…¬ç¤ºå…¬å‘Š
Site: https://henanlt.com/news/4
CMS: Custom PHP (ThinkPHP)
List: ul.news > li > a[title][href]
Detail: div.content (rich HTML + file attachments)
Pagination: /news/news_type/type/4/p/{page}
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupÚ	SEARCH_DBz/root/search.dbu   henanlt.com-å…¬ç¤ºå…¬å‘Šzhttps://henanlt.comz/news/4z/news/news_type/type/4/p/{}iG  )Údaysz%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36ú/)z
User-AgentÚRefererc                 ó¦   — 	 t         j                  | d¬«      }d|_        |j                  S # t        $ r}t        d| › d|› �«       Y d }~y d }~ww xY w)Né   )Útimeoutzutf-8z  [ERR] fetch failed: z - )ÚsessionÚgetÚencodingÚtextÚ	ExceptionÚprint)ÚurlÚrÚes      ú"/root/gov_crawler/crawl_henanlt.pyÚfetchr      sR   € ðÜ�K‰K˜ RˆKÓ(ˆØˆŒ
Ø�v‰vˆøÜò ÜÐ& s e¨3¨q¨cÐ2Ô3Üûðús   ‚), ¬	AµAÁAc                 ó’  — g }t        | d«      }|j                  dd¬«      }|s|S |j                  d«      D ]�  }|j                  d«      }|sŒ|j                  d«      sŒ)|d   }|j                  d«      xs |j	                  d	¬
«      }|j                  d«      s	t        |z   }|j                  ||j                  «       f«       Œ� |S )z,Parse list page, return list of (url, title)úhtml.parserÚulÚnews©Úclass_ÚliÚaÚhrefÚtitleT©ÚstripÚhttp)	r   ÚfindÚfind_allr   Úget_textÚ
startswithÚBASE_URLÚappendr#   )ÚhtmlÚitemsÚsoupr   r   r   r    r!   s           r   Ú
parse_listr.   (   s¸   € à€EÜ˜˜}Ó-€DØ	�‰�4 ˆÓ	'€BÙØˆØ�k‰k˜$Óò 0ˆØ�G‰G�C‹LˆÚ�—‘�v•Ø�V‘9ˆDØ—E‘E˜'“NÒ< a§j¡j°t jÓ&<ˆEØ—?‘? 6Ô*Ü $‘�Ø�L‰L˜$ §¡£Ð.Õ/ð0ð €Ló    c                 óž  — t        | d«      }|j                  dd¬«      }|sy|j                  d«      }|r|j                  d¬«      nd	}|j                  dd
¬«      }d	}|r7t        j                  d|j                  «       «      }|r|j                  d«      }|j                  dd¬«      }	d}
|	rt        |	«      }
|
j                  «       }
|||
fS )z8Parse detail page, return (title, publish_date, content)r   ÚdivÚgsjjnr)Úid)NNNÚh2Tr"   NÚnew_fbr   u(   å�‘å¸ƒæ—¶é—´[ï¼š:]\s*(\d{4}-\d{2}-\d{2})é   ÚcontentÚ )r   r%   r'   ÚreÚsearchÚgroupÚstrr#   )r+   r   r-   r2   r4   r!   r5   Údate_strÚmÚcontent_divr7   s              r   Úparse_detailr@   :   sÎ   € ä˜˜}Ó-€Dð �Y‰Y�u ˆYÓ*€FÙØð 
�‰�TÓ	€BÙ')ˆB�K‰K˜dˆKÔ#¨t€Eð �[‰[˜ xˆ[Ó0€FØ€HÙÜ�I‰IÐAÀ6Ç?Á?ÓCTÓUˆÙØ—w‘w˜q“zˆHð —+‘+˜e¨I�+Ó6€KØ€GÙÜ�kÓ"ˆà—-‘-“/ˆà�(˜GÐ#Ð#r/   c                  ód  — t        t        «      } | syt        j                  d| «      }|rHt	        |j                  d«      «      }t	        |j                  d«      «      }t        d|› d|› d�«       |S t        j                  d| «      }|r&t        j                  d	| «      }|rt	        |d
   «      S y)zGet total pages from page 1r   u#   (\d+)\s*æ�¡è®°å½•\s*\d+/(\d+)\s*é¡µr6   é   z	  Found: z items, z pagesz<option[^>]*value="(\d+)"[^>]*>z<option[^>]*value="(\d+)"éÿÿÿÿ)r   ÚLIST_URLr9   r:   Úintr;   r   Úfindall)r+   r>   Útotal_itemsÚtotal_pagesÚoptionss        r   Úget_total_pagesrJ   Z   s¢   € ä”‹?€DÙØô 	�	‰	Ð8¸$Ó?€AÙÜ˜!Ÿ'™' !›*“oˆÜ˜!Ÿ'™' !›*“oˆÜ�	˜+˜ h¨{¨m¸6ÐBÔCØÐô 	�	‰	Ð4°dÓ;€AÙä—*‘*Ð9¸4Ó@ˆÙÜ�w˜r‘{Ó#Ð#àr/   c                  óP  — t        dt        › d�«       t        dt        › �«       t        «       } | dk(  rt        d«       y t	        j
                  t        «      }|j                  d«       |j                  d«       |j                  «       }d}d}t        d| dz   «      D �]½  }|dk(  rt        }nt        j                  |«      }t        d	|› d
| › d|› �«       t        |«      }|st        d|› d�«       ŒVt        |«      }|st        d|› d�«       Œsd}	|D �]  \  }
}|j                  d|
f«       |j                  «       r|dz  }Œ0t        |
«      }|s|dz  }ŒCt!        ||
«      \  }}}|s|s|dz  }Œ]|s|}|s|dz  }Œi|t        k  r|dz  }Œx|j#                  «       }|s|dz  }Œ�d}|r#t%        |d«      }|j'                  d¬«      }|d d }	 |j                  dt        ||
||t)        |j+                  dd«      «      df«       |j,                  dkD  r
|	dz  }	|dz  }�Œ	 |j1                  «        t        d|› d|	› d|› d|› d�	«       t3        j4                  d«       �ŒÀ |j7                  «        t        d|› d |› d!�«       y # t.        $ r}t        d|› �«       Y d }~�ŒŠd }~ww xY w)"Nz=== z ===zCutoff: r   z&ERROR: Could not determine total pageszPRAGMA journal_mode=WALzPRAGMA busy_timeout=10000r6   z
Page r   z: z  [SKIP] page z fetch failedz	 no itemsz)SELECT id FROM gov_raw WHERE page_url = ?r8   r   Tr"   éÈ   zƒINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category) VALUES (?, ?, ?, ?, ?, ?, ?)ú-Útzggz  [ERR] insert: z  Page z: +z new (total z, skip ú)g      à?z
=== Done: z new, z skipped ===)r   Ú	SITE_NAMEÚCUTOFFrJ   Úsqlite3Úconnectr   ÚexecuteÚcursorÚrangerD   ÚPAGE_URL_TPLÚformatr   r.   Úfetchoner@   r#   r   r'   rE   ÚreplaceÚrowcountr   ÚcommitÚtimeÚsleepÚclose)rH   ÚconnÚcurÚ	total_newÚ
total_skipÚpageÚlist_urlr+   r,   Úpage_newr   r!   Údetail_htmlÚ
full_titler=   r7   ÚsummaryÚ	text_soupÚplainr   s                       r   Úmainrl   s   sï  € Ü	ˆD”�˜4Ð
 Ô!Ü	ˆH”V�HÐ
Ôä!Ó#€KØ�aÒÜÐ6Ô7Øä�?‰?œ9Ó%€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø
�+‰+‹-€Cà€IØ€Jä�a˜ q™Ó)ó OˆØ�1Š9Ü‰Hä#×*Ñ*¨4Ó0ˆHä�˜�v˜Q˜{˜m¨2¨h¨ZÐ8Ô9Ü�X‹ˆÙÜ�N 4 &¨Ð6Ô7Øä˜4Ó ˆÙÜ�N 4 &¨	Ð2Ô3ØàˆØó 7	.‰JˆC�à�K‰KÐCÀcÀVÔLØ�|‰|Œ~Ø˜a‘�
Øô   ›*ˆKÙØ˜a‘�
Øä,8¸ÀcÓ,JÑ)ˆJ˜ 'Ù¡eØ˜a‘�
ØáØ"�
áØ˜a‘�
Øð œ&Ò Ø˜a‘�
Øð —m‘m“oˆGÙØ˜a‘�
Øð ˆGÙÜ)¨'°=ÓA�	Ø!×*Ñ*°Ð*Ó6�Ø  ˜+�ð.Ø—‘ð3ô  
¨C°¸7Ü˜×)Ñ)¨#¨rÓ2Ó3°Vð=ô	ð —<‘< !Ò#Ø ‘M�HØ ‘N�Iùðk7	.ðr 	�‰ŒÜ�˜�v˜S  
¨,°y°kÀÈÈÐTUÐVÔWô 	�
‰
�3Žð_Oðb 	‡J�J„LÜ	ˆL˜˜ 6¨*¨°\Ð
BÕCøô ò .ÜÐ(¨¨Ð,×-Ò-ûð.ús   ÇAJÊ	J%ÊJ Ê J%Ú__main__)Ú__doc__r9   ÚosÚsysr]   rR   Úrequestsr   r   Úbs4r   Úgetenvr   rP   r)   rD   rW   ÚnowÚstrftimerQ   ÚHEADERSÚSessionr   ÚheadersÚupdater   r.   r@   rJ   rl   Ú__name__© r/   r   ú<module>r|      sØ   ðñ÷ ,× +× +ß (Ý àˆB�I‰I�kÐ#4Ó5€	Ø&€	Ø €Ø�iÑ€ØÐ7Ñ7€Ø
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€ð DØ˜#‰~ñ€ð
 ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ òòò$$ò@ò2cDðL ˆzÒÙ…Fð r/   