Ë
    ¸¬ujå  ã                   óv  — d dl Z 	 d dlZd dlZd dlZd dl Z d dlZd dlZd dlZd dlmZmZ  e j                  dd«      Z
dZdZddd	ed
z   dœZ ej                  «        ed¬«      z
  j                  d«      Z ej"                  «       Zej&                  j)                  e«       d„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)é    N)ÚdatetimeÚ	timedeltaÚ	SEARCH_DBz/root/search.dbzhttps://hbj.np.gov.cnz=/cms/sitemanage/index.shtml?siteId=100433125044680000&page={}zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8z&/cms/html/npshjbhj/hpspslgg/index.html)z
User-AgentÚAcceptzAccept-LanguageÚRefereriG  )Údaysz%Y-%m-%dc                 ó”  — 	 t         j                  t        t        j	                  | «      z   d¬«      }d|_        g }t        j                  dt        j                  «      }|j                  |j                  «      D ]¡  }|j                  d«      |j                  d	«      j                  «       |j                  d
«      }}}d|v s|sŒK|j                  dd«      j                  dd«      j                  «       }t!        |«      dkD  r|dd }|j#                  |||f«       Œ£ |S # t        $ r}t        d| › d|› �«       g cY d}~S d}~ww xY w)u6   è§£æž�åˆ—è¡¨é¡µï¼Œè¿”å›ž [(url, title, date_str), ...]é   ©Útimeoutúutf-8z  [ERROR] Page z fetch failed: Nz„<A\s+href="(/cms/html/npshjbhj/\d{4}-\d{2}-\d{2}/\d+\.html)"[^>]*>([^<]*)</A>.*?<SPAN\s+class="list-time">(\d{4}-\d{2}-\d{2})</SPAN>é   é   é   u   è�”ç³»õ   Â ú z&nbsp;i,  )ÚsessionÚgetÚBASEÚLIST_URLÚformatÚencodingÚ	ExceptionÚprintÚreÚcompileÚDOTALLÚfinditerÚtextÚgroupÚstripÚreplaceÚlenÚappend)	ÚpageÚrÚeÚitemsÚpatternÚmÚurlÚtitleÚdate_strs	            ú"/root/gov_crawler/crawl_nanping.pyÚ
parse_listr/      s.  € ðÜ�K‰KœœxŸ™¨tÓ4Ñ4¸bˆKÓAˆØˆŒ
ð
 €Eä�j‰jð	Cä
�	‰	ó€Gð
 ×Ñ˜aŸf™fÓ%ò 
-ˆØ Ÿw™w q›z¨1¯7©7°1«:×+;Ñ+;Ó+=¸q¿w¹wÀq»z�HˆUˆà�uÑ¡EØà—‘˜h¨Ó,×4Ñ4°X¸sÓC×IÑIÓKˆäˆu‹:˜ÒØ˜$˜3�KˆEØ�‰�c˜5 (Ð+Õ,ð
-ð €Løô- ò Ü� ˜v _°Q°CÐ8Ô9Ø�	ûðús   ‚8D! Ä!	EÄ*EÄ<EÅEc                 ó.  — 	 t         j                  t        | z   d¬«      }d|_        |j                  }t        j                  d|t        j                  «      }|s%t        j                  d	|t        j                  «      }|r|j                  d
«      j                  «       nd}t        j                  d|t        j                  «      }|s%t        j                  d|t        j                  «      }d}|rj|j                  d
«      j                  «       }|j                  dd«      j                  dd«      }t        j                  d|«      }|r|j                  d
«      }t        j                  d|t        j                  «      }d}	|r |j                  d
«      j                  «       }	nFt        j                  d|t        j                  «      }|r|j                  d
«      j                  «       }	|||	fS # t        $ r}t        d| › d|› �«       Y d}~yd}~ww xY w)u8   è§£æž�è¯¦æƒ…é¡µï¼Œè¿”å›ž (title, pub_date, content_html)r
   r   r   z  [ERROR] Detail z	 failed: N)NNNz'content="([^"]*)"\s+name="ArticleTitle"z'name="ArticleTitle"\s+content="([^"]*)"r   Ú z"content="([^"]*)"\s+name="PubDate"z"name="PubDate"\s+content="([^"]*)"u   â€”ú-u   â€•z(\d{4}-\d{2}-\d{2})z1<DIV\s+class="content"\s+id="Content">(.*?)</DIV>u   <!--\s*æ­£æ–‡\s*-->(.*?)<!--)r   r   r   r   r   r   r   r   ÚsearchÚIr    r!   r"   r   )
r+   r&   r'   Úhtmlr*   r,   Úpub_dateÚrawÚdmÚcontents
             r.   Úparse_detailr:   7   s­  € ð Ü�K‰Kœ˜s™
¨BˆKÓ/ˆØˆŒ
ð
 �6‰6€Dô 	�	‰	Ð<¸dÄBÇDÁDÓI€AÙÜ�I‰IÐ@À$ÌÏÉÓMˆÙ"#ˆA�G‰G�A‹J×ÑÔ¨€Eô 	�	‰	Ð7¸¼r¿t¹tÓD€AÙÜ�I‰IÐ;¸TÄ2Ç4Á4ÓHˆØ€HÙØ�g‰g�a‹j×ÑÓ ˆà�k‰k˜( CÓ(×0Ñ0°¸3Ó?ˆä�Y‰YÐ-¨sÓ3ˆÙØ—x‘x “{ˆHô 	�	‰	ÐFÈÌbÏiÉiÓX€AØ€GÙØ—'‘'˜!“*×"Ñ"Ó$‰ô �I‰IÐ5°t¼R¿Y¹YÓGˆÙØ—g‘g˜a“j×&Ñ&Ó(ˆGà�(˜GÐ#Ð#øôK ò  ÜÐ! #  i°¨sÐ3Ô4Üûð ús   ‚%G0 Ç0	HÇ9HÈHc                 óš  — t        j                  t        d¬«      }|j                  «       }d}d}| D ]M  \  }}}}	 |j	                  dt
        |z   |||xs dt
        |z   ddf«       |j                  dk(  r|dz  }n|dz  }ŒO |j                  «        |j                  «        ||fS # t        $ r}	t        d	|› d
|	› �«       Y d}	~	Œ“d}	~	ww xY w)u   æ‰¹é‡�å†™å…¥search.dbé   r   r   zÄ
                INSERT OR REPLACE INTO gov_raw 
                (page_url, title, content, publish_date, source_url, category, site_name)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            r1   u   çŽ¯è¯„å®¡æ‰¹å�—ç�†å…¬å‘Šu   å�—å¹³å¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€r   z  [ERROR] DB insert z: N)Úsqlite3ÚconnectÚDB_PATHÚcursorÚexecuter   Úrowcountr   r   ÚcommitÚclose)
Ú
items_dataÚconnÚcÚinsertedÚupdatedr+   r,   r6   r9   r'   s
             r.   Ú
save_to_dbrJ   d   sç   € ä�?‰?œ7¨BÔ/€DØ�‰‹€AØ€HØ€GØ)3ò 5Ñ%ˆˆU�H˜gð	5Ø�I‰Ið ô
 �s‘
ØØØ’˜BÜ�s‘
Ø*Ø*ðô	ð �z‰z˜QŠØ˜A‘‘à˜1‘�øð%5ð* 	‡K�K„MØ‡J�J„LØ�WÐÐøô	 ò 	5ÜÐ(¨¨¨R°¨sÐ3×4Ñ4ûð	5ús   »AB&Â&	C
Â/CÃC
c                  ó
  — t        d«       t        dt        › �«       g } t        «       }t        dd«      D ]Æ  }t        d|› d�«       t	        |«      }|st        d«        n�d}d	}|D ]@  \  }}}||vr*|j                  |«       | j                  |||f«       |dz  }|t        k  sŒ?d
}ŒB t        dt        |«      › d|› dt        | «      › �«       |rt        dt        › d�«        nt        j                  d«       ŒÈ t        dt        | «      › d�«       g }	t        | «      D ]¦  \  }
\  }}}t        d|
dz   › dt        | «      › d|d d › d�d¬«       t        |«      \  }}}|r|}|r|}|	j                  ||||f«       t        d|rt        |«      nd› �d¬«       |st        dd¬«       t        «        t        j                  d«       Œ¨ t        |	«      \  }}t        d|› d|› d�«       t        d t        |	«      › �«       y )!Nu'   === å�—å¹³å¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€ çˆ¬è™« ===u   æˆªæ­¢æ—¥æœŸ(è¿‘3å¹´): r   é   z
[Page z] Fetching list...z(  -> No items found, stopping paginationr   FTz  -> Found z items (z new), total queued: z  -> Reached cutoff date z
, stoppingg      à?z
=== Total list items: z ===z  [ú/z] é<   z...r   )Úendzlen(content)=r1   z [WARN empty]z
=== DB write complete: z inserted, z updated ===zTotal in this run: )r   ÚCUTOFF_DATEÚsetÚranger/   Úaddr$   r#   ÚtimeÚsleepÚ	enumerater:   rJ   )Ú	all_itemsÚ	seen_urlsr%   r(   Ú	new_countÚstopr+   r,   r-   Údetail_dataÚiÚ	det_titleÚdet_dater9   ÚinsÚupds                   r.   Úmainra   „   s0  € Ü	Ð3Ô5Ü	Ð#¤K =Ð
1Ô2à€IÜ“€Iô �a˜“ò ˆÜ�˜˜Ð0Ð1Ô2Ü˜4Ó ˆÙÜÐ<Ô>Ùàˆ	ØˆØ$)ò 	Ñ ˆC�˜Ø˜)Ñ#Ø—‘˜cÔ"Ø× Ñ  # u¨hÐ!7Ô8Ø˜Q‘�	Øœ+Ó%Ø‘ð	ô 	�œC ›J˜< x°	¨{Ð:OÔPSÐT]ÓP^ÐO_Ð`ÔaÙÜÐ-¬k¨]¸*ÐEÔFÙÜ�
‰
�3�ð+ô. 
Ð$¤S¨£^Ð$4°DÐ
9Ô:ð €KÜ%.¨yÓ%9ò Ñ!ˆÑ!ˆC�˜Ü��A�a‘C�5˜œ#˜i›.Ð)¨¨C°°¨H¨:°SÐ9¸sÕCÜ'3°CÓ'8Ñ$ˆ	�8˜WÙØˆEÙØˆHØ×Ñ˜C ¨°'Ð:Ô;Ü�©gœc 'œl¸1Ð=Ð>ÀBÕGÙÜ�/ rÕ*ÜŒÜ�
‰
�3�ðô ˜+Ó&�H€CˆÜ	Ð% c U¨+°c°U¸,Ð
GÔHÜ	Ð¤ KÓ 0Ð1Ð
2Õ3ó    Ú__main__)ÚosÚrequestsr=   r   ÚsysrT   Újsonr   r   Úgetenvr?   r   r   ÚHEADERSÚnowÚstrftimerP   ÚSessionr   ÚheadersÚupdater/   r:   rJ   ra   Ú__name__© rb   r.   ú<module>rq      sÈ   ðã 	ð÷ 2× 1× 1Ó 1ß (à
ˆ"�)‰)�KÐ!2Ó
3€Ø€ØJ€ð DØOØ0ØÐ>Ñ>ñ	€ð ˆx�|‰|‹~¡	¨uÔ 5Ñ5×?Ñ?À
ÓK€à
ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ òò<*$òZò@44ðn ˆzÒÙ…Fð rb   