Ë
    ¸¬ujÙ  ã                   óT  — d Z ddlZddlZddlZddlZddlZddlZddlmZ dZ	 ej                  dd«      ZdZ ej                  «       j                   ej                  «       j                  dz
  ¬	«      j!                  d
«      ZdddddœZddiZdd„Zd„ Zd„ Zd„ Zedk(  r e«        yy)u×   crawl_ouryq.py - ä¼�ä¸šçŽ¯ä¿�ä¿¡æ�¯å…¬ç¤ºç½‘ (www.ouryq.com)
4ä¸ªæ �ç›®ï¼šçŽ¯è¯„ä¿¡æ�¯å…¬ç¤º/eiaã€�ç«£å·¥éªŒæ”¶å…¬ç¤º/epaã€�å›ºåºŸä¿¡æ�¯å…¬ç¤º/eswã€�å…¶ä»–å…¬ç¤º/other
å�•é¡µå…¨é‡�åŠ è½½ï¼Œè¯¦æƒ…é¡µHTMLæ­£æ–‡æ��å�–
é    N)Údatetimezhttps://www.ouryq.comÚ	SEARCH_DBz/root/search.dbu   ä¼�ä¸šçŽ¯ä¿�ä¿¡æ�¯å…¬ç¤ºç½‘é   )Úyearú%Y-%m-%dz/eiaz/epaz/eswz/other)u   çŽ¯è¯„ä¿¡æ�¯å…¬ç¤ºu   ç«£å·¥éªŒæ”¶å…¬ç¤ºu   å›ºåºŸä¿¡æ�¯å…¬ç¤ºu   å…¶ä»–å…¬ç¤ºz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c                 óº  — t        |«      D ]u  }	 t        j                  j                  | t        ¬«      }t        j                  j                  |d¬«      5 }|j                  «       j                  dd¬«      cd d d «       c S  y # 1 sw Y   nxY wŒ†# t        $ r?}||dz
  k  rt        j                  d«       nt        d	| › d
|› �«       Y d }~ y Y d }~ŒÉd }~ww xY w)N)Úheadersé   ©Útimeoutzutf-8Úreplace)Úerrorsé   é   z  [WARN] Failed to fetch z: )ÚrangeÚurllibÚrequestÚRequestÚHEADERSÚurlopenÚreadÚdecodeÚ	ExceptionÚtimeÚsleepÚprint)ÚurlÚretriesÚiÚreqÚrespÚes         ú /root/gov_crawler/crawl_ouryq.pyÚfetchr$      sÎ   € Ü�7‹^ò 
ˆð		Ü—.‘.×(Ñ(¨´gÐ(Ó>ˆCÜ—‘×'Ñ'¨°RÐ'Ó8ð E¸DØ—y‘y“{×)Ñ)¨'¸)Ð)ÓD÷Eó Eñ
÷Eð Eúð Eøäò 	Ø�7˜Q‘;ŠÜ—
‘
˜1•äÐ1°#°°b¸¸Ð<Ô=Ýô ûð	ús0   �ABÁ!BÁ7	BÂB	Â
BÂ	CÂ/CÃCc                 ó˜  — g }d}t        j                  || t         j                  «      D �]  }|j                  d«      }d|v s|j	                  «       sŒ*t        j
                  dt        j                  |«      z   dz   |«      }|sŒ\|j                  d«      }d}	t        j
                  d|«      }
|
r|
j                  d«      j	                  «       }	d}t        j
                  d|«      }|r9|j                  d«      }	 t        j                  |d	«      }|j                  d
«      }|	sŒü|sŒÿ|j                  |	|||f«       �Œ |S #  |}Y Œ&xY w)uG   è§£æž�åˆ†ç±»åˆ—è¡¨é¡µè¡¨æ ¼ï¼Œè¿”å›ž [(title, page_url, pub_date), ...]z<tr[^>]*>(.*?)</tr>r   Útbheadzhref="(https://www\.ouryq\.comz/\d+\.html)"Ú z5<td[^>]*title="([^"]*)"[^>]*style="text-align:\s*leftu.   <td[^>]*title="(\d{4}å¹´\d{1,2}æœˆ\d{1,2}æ—¥)"u   %Yå¹´%mæœˆ%dæ—¥r   )ÚreÚfinditerÚDOTALLÚgroupÚstripÚsearchÚescaper   ÚstrptimeÚstrftimeÚappend)ÚhtmlÚcat_nameÚcat_pathÚitemsÚrow_patternÚtr_matchÚtr_htmlÚ	url_matchÚpage_urlÚtitleÚtitle_matchÚpub_dateÚ
date_matchÚdate_strÚdts                  r#   Úparse_list_pagerA   *   sE  € à€Eð )€KÜ—K‘K ¨T´2·9±9Ó=ó  @ˆØ—.‘. Ó#ˆà�wÑ g§m¡m¤oØô —I‘IÐ?Ä"Ç)Á)ÈHÓBUÑUÐXgÑgÐipÓqˆ	ÙØØ—?‘? 1Ó%ˆð ˆÜ—i‘iÐ XÐZaÓbˆÙØ×%Ñ% aÓ(×.Ñ.Ó0ˆEð ˆä—Y‘YÐPÐRYÓZˆ
ÙØ!×'Ñ'¨Ó*ˆHð$Ü×&Ñ& xÐ1BÓC�ØŸ;™; zÓ2�ò ’XØ�L‰L˜% ¨8°XÐ>Ö?ðA @ðD €Løð$Ø#’ús   Ã<'EÅE	c                 óä   — d}t        j                  d| t         j                  «      }|s%t        j                  d| t         j                  «      }|r|j                  d«      j	                  «       }|S )u   ä»Žè¯¦æƒ…é¡µæ��å�–æ­£æ–‡r'   zW<div\s+style="text-indent:2em;line-height:30px;">(.*?)</div>\s*</div>\s*</div>\s*</div>z<<div\s+style="text-indent:2em;line-height:30px;">(.*?)</div>r   )r(   r-   r*   r+   r,   )r2   r3   ÚcontentÚms       r#   Úextract_detailrE   V   s`   € ð €GÜ
�	‰	ÐlÐnrÔtv×t}Ñt}Ó~€AÙÜ�I‰IÐUÐW[Ô]_×]fÑ]fÓgˆÙØ—'‘'˜!“*×"Ñ"Ó$ˆà€Nó    c                  ó6  — t        dt        j                  «       j                  «       › dt        › �«       t        dt
        › �«       g } t        j                  «       D ]º  \  }}t        |z   }t        d|› d|› d�«       t        |«      }|st        d|› �«       Œ=t        |||«      }t        dt        |«      › d	�«       t        |«      }|D �cg c]  }|d
   t
        k\  sŒ|‘Œ }}t        dt
        › dt        |«      › d|› �«       | j                  |«       Œ¼ t        dt        | «      › d�«       | st        d«       y dd l}|j                  t        d¬«      }	|	j!                  d«       |	j!                  d«       |	j#                  «       }
d}d}t        | «      }t%        | «      D �]:  \  }\  }}}}|
j!                  d|f«       |
j'                  «       r|dz  }Œ5t        d|dz   › d|› d|› d|d d › d�	d¬«       t(        j*                  j-                  «        t        |«      }|st        d«       |dz  }Œ‘t/        ||«      }|st        d«       |dz  }Œ°d |› �}	 |
j!                  d!t        › d"|› �||||||d d# j1                  d$d«      j3                  «       f«       |dz  }|	j5                  «        t        d%t        |«      › d&�«       t;        j<                  d(«       �Œ= |	j?                  «        t        d$d)› �«       t        d*t        j                  «       j                  «       › �«       t        d+|› �«       t        d,|› �«       t        d-|› �«       y c c}w # t6        $ r-}|	j9                  «        t        d'|› �«       |dz  }Y d }~ŒÅd }~ww xY w).Nú[z] Starting crawl: zCutoff date: z
--- z (z) ---z  [ERROR] Failed to fetch z  Found z itemsr   z  After date filter (z): ú/z-
=== Total list items across all categories: z ===zNo items to process. Exiting.r   r
   r   zPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000z'SELECT id FROM gov_raw WHERE page_url=?r   z  [z] z | z...ú )Úendz[SKIP fetch failed]z[SKIP no content]zouryq-zºINSERT OR REPLACE INTO gov_raw 
                (site_name, source_url, page_url, title, publish_date, content, status, summary)
                VALUES (?, ?, ?, ?, ?, ?, 'published', ?)ú-éÈ   ú
z[OK len=ú]z[ERR] g333333Ó?z2==================================================zCompleted: zTotal items: z
Inserted: zSkipped (exists/no content): ) r   r   ÚnowÚ	isoformatÚ	SITE_NAMEÚCUTOFF_DATEÚ
CATEGORIESr5   ÚBASEr$   rA   ÚlenÚextendÚsqlite3ÚconnectÚDB_PATHÚexecuteÚcursorÚ	enumerateÚfetchoneÚsysÚstdoutÚflushrE   r   r,   Úcommitr   Úrollbackr   r   Úclose)Ú	all_itemsr3   r4   r   r2   r5   ÚbeforeÚitrX   ÚconnÚcÚinsertedÚskippedÚtotalr   r;   r:   r=   Údetail_htmlrC   Úcategory_tagr"   s                         r#   Úmainro   c   s‘  € Ü	ˆAŒh�l‰l‹n×&Ñ&Ó(Ð)Ð);¼I¸;Ð
GÔHÜ	ˆMœ+˜Ð
'Ô(à€Iô )×.Ñ.Ó0ò  Ñˆ�(Ü�X‰oˆÜ��x�j  3 % uÐ-Ô.Ü�S‹zˆÙÜÐ.¨s¨eÐ4Ô5Øä  h°Ó9ˆÜ�œ˜U›˜ FÐ+Ô,ô �U“ˆØ#Ö<˜ r¨!¡u´Ó';’Ð<ˆÐ<ÜÐ%¤k ]°#´c¸%³j°\ÀÀ6À(ÐKÔLà×Ñ˜Õð! ô$ 
Ð:¼3¸y»>Ð:JÈ$Ð
OÔPáÜÐ-Ô.Øó à�?‰?œ7¨Bˆ?Ó/€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø�‰‹€Aà€HØ€GÜ�	‹N€Eä4=¸iÓ4Hó (Ñ0ˆÑ0ˆE�8˜X xà	�	‰	Ð;¸h¸[ÔIØ�:‰:Œ<Ø�q‰LˆGØä��A�a‘C�5˜˜%˜  8 *¨C°°c°r°
¨|¸3Ð?ÀSÕIÜ�
‰
×ÑÔä˜H“oˆÙÜÐ'Ô(Ø�q‰LˆGØä  ¨hÓ7ˆÙÜÐ%Ô&Ø�q‰LˆGØð   ˜zÐ*ˆð	Ø�I‰Ið =ô �;˜a ˜zÐ*¨H°hÀÀxÐQXØ˜˜#�×&Ñ& t¨SÓ1×7Ñ7Ó9ð;ô<ð
 ˜‰MˆHà�K‰KŒMÜ�HœS ›\˜N¨!Ð,Ô-ô 	�
‰
�3ŽðQ(ðT 	‡J�J„Lä	ˆBˆvˆhˆ-ÔÜ	ˆKœŸ™›×0Ñ0Ó2Ð3Ð
4Ô5Ü	ˆM˜%˜Ð
!Ô"Ü	ˆJ�x�jÐ
!Ô"Ü	Ð)¨'¨Ð
3Õ4ùòO =øôt ò 	Ø�M‰MŒOÜ�F˜1˜#�,ÔØ�q‰L�Gûð	ús%   ÃMÃ!MÉA0M"Í"	NÍ+#NÎNÚ__main__)r   )Ú__doc__r(   r   r_   ÚosÚurllib.requestr   Úurllib.errorr   rU   ÚgetenvrZ   rR   rP   r   r   r0   rS   rT   r   r$   rA   rE   ro   Ú__name__© rF   r#   ú<module>rx      sÀ   ðñ÷ × ß #Ý à€Ø
ˆ"�)‰)�KÐ!2Ó
3€Ø)€	Øˆx�|‰|‹~×%Ñ%¨<¨8¯<©<«>×+>Ñ+>ÀÑ+BÐ%ÓC×MÑMÈjÓY€ð !Ø Ø Øñ	€
ð ð  Dð€ó
ò)òX
ò[5ð| ˆzÒÙ…Fð rF   