Ë
    ^¨£jì  ã                   óÊ  — d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
 ddlZ ej                  «       Zej                  dedd¬«       ej#                  «       \  ZZej(                  Z ej,                  dd	«      Zd
ZdZdZdZ ej8                  «        ed¬«      z
  j;                  d«      Z ede› �«        e ej@                  ed«      jC                  «       dz  «      Z"dddœZ# ejH                  «       Z%e%jL                  jO                  e#«        ejH                  «       Z(e(jL                  jO                  ddi«       d„ Z)d„ Z*d„ Z+d„ Z,e-dk(  r e,«        yy)uÙ   
æ™‹æ±Ÿå¸‚äººæ°‘æ”¿åºœ - ç”Ÿæ€�çŽ¯å¢ƒå±€ - ç»¿è‰²è½¬åž‹ï¼ˆç”Ÿæ€�çŽ¯å¢ƒä¿�æŠ¤ï¼‰
CMS: Terton (å¤©æ¶¦) SSP
åˆ—è¡¨: POST /ssp/search/api/v2/external (æ‰¹é‡�100æ�¡)
è¯¦æƒ…: å�– detail HTML æ­£æ–‡ (å�«è¡¨æ ¼ç­‰å®Œæ•´HTML)
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupz--pagesu   é™�åˆ¶é¡µæ•°)ÚtypeÚdefaultÚhelpÚ	SEARCH_DBz/root/search.dbu/   jinjiang.gov.cn-ç»¿è‰²è½¬åž‹-ç”Ÿæ€�çŽ¯å¢ƒä¿�æŠ¤z6https://www.jinjiang.gov.cn/ssp/search/api/v2/externalzhttps://www.jinjiang.gov.cnéd   iG  )Údaysú%Y-%m-%dzCutoff date: éè  zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zapplication/json)ú
User-AgentzContent-Typer   z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36c                 óä   — ddt         | dddigddigdœdœ}	 t        j                  t        |d	¬
«      }d|_        |j                  «       S # t        $ r}t        d| › d|› �«       Y d}~yd}~ww xY w)z%Fetch one page of items from the API.Ú 000000008a939277018a9bf9d5960057ÚWCMDOCAPzdocreltime descÚeqé
   Ú50128)Ú	docstatusÚchnlid)ÚsiteIdÚapiNameÚpageSizeÚpageÚ	sortFieldÚfilteré   )ÚjsonÚtimeoutúutf-8z  [ERR] API page z: N)Ú	PAGE_SIZEÚsessionÚpostÚAPI_URLÚencodingr   Ú	ExceptionÚprint)r   ÚbodyÚrÚes       úcrawl_jinjiang_sthj.pyÚfetch_api_pager,   /   s†   € ð 5ØÜØØ&à ˜*˜Ø˜g�Ð'ñ
ñ
€DðÜ�L‰Lœ t°RˆLÓ8ˆØˆŒ
Ø�v‰v‹xˆøÜò ÜÐ! $  r¨!¨Ð-Ô.Üûðús   ˜2A Á	A/ÁA*Á*A/c                 óœ  — 	 t         j                  | d¬«      }d|_        |j                  dk7  ry	 t        |j                  d	«      }|j                  d
«      }|rt        |«      }|S |j                  d«      }|rt        |«      S |j                  d«      }|rt        |«      S y# t        $ r}t        d| dd › d|› �«       Y d}~yd}~ww xY w)zJFetch detail page and extract content HTML from TRS_Editor / article_area.r   ©r   r    éÈ   Nz  [ERR] detail fetch: éP   z - zhtml.parserz.TRS_Editorz.article_areaz.content, #zoom, .g-detailbox)
Úsession_detailÚgetr%   Ústatus_coder&   r'   r   ÚtextÚ
select_oneÚstr)Úurlr)   r*   ÚsoupÚtrsÚcontentÚareaÚcontent_divs           r+   Úfetch_detailr=   E   sÚ   € ðÜ×Ñ˜s¨BÐÓ/ˆØˆŒ
Ø�=‰=˜CÒØð  ô ˜Ÿ™ Ó/€Dð �/‰/˜-Ó
(€CÙ
Ü�c“(ˆØˆð �?‰?˜?Ó+€DÙÜ�4‹yÐð —/‘/Ð"AÓB€KÙÜ�;ÓÐàøô- ò ÜÐ& s¨3¨B x j°°A°3Ð7Ô8Üûðús   ‚-B$ Â$	CÂ-CÃCc                 óv   — 	 t        | «      dz  }t        j                  |«      j                  d«      S #  Y yxY w)Nr   r   Ú )Úintr   ÚfromtimestampÚstrftime)Úts_msÚtss     r+   Ú
ts_to_daterE   e   s;   € ðÜ�‹Z˜$ÑˆÜ×%Ñ% bÓ)×2Ñ2°:Ó>Ð>øðÙús   ‚14 ´8c                  ó	  — dd l } | j                  t        d¬«      }|j                  d«       |j                  d«       |j	                  «       }t        d„ |j                  dt        f«      j                  «       D «       «      }t        dt        |«      › d	�«       d}d}d}d}d
}t        dkD  rt        nd}	||	k  �rìt        d|› d|	› d�«       t        |«      }
|
r|
j                  d«      dk7  rt        d|› d�«       �n¨|
j                  di «      }t        |j                  dd«      «      }t        |j                  dt        «      «      }||z   d
z
  |z  }	t        dkD  rt        |	t        «      }	|j                  dg «      }|s�n"t        dt        |«      › d|› d|	› d�«       d}d}|D �]9  }|j                  dd«      }|j                  dd«      }|r|sŒ-|j                  dd«      }t!        |«      }|rt        |«      t"        k  r|d
z  }|d
z  }Œi||v r|d
z  }|d
z  }Œxt%        |«      }|s,t        d|d d  › d!�«       |j                  d"«      xs d}|d
z  }|rt        |j'                  d#d«      «      nd}	 |j                  d$t        |||||d%f«       |j(                  dkD  r#|d
z  }|d
z  }|d&z  dk(  r|j+                  «        n|d
z  }t/        j0                  d(«       �Œ< |j+                  «        t        d)|› d*|› d+|› d,|› d�	«       |rvt        |d-   j                  dd«      «      t"        k  rQ|t        |«      d.z  kD  r@|D �cg c]'  }t        |j                  dd«      «      t"        k\  sŒ&|‘Œ) }}|st        d/«       n!|d
z  }t/        j0                  d(«       ||	k  r�Œìt        d0d1› �«       t        d2|› d+|› d3|› d4|› d5�	«       |dkD  rQt        d6«       	 |j                  d7t        › d8�«       |j                  d9«       |j+                  «        t        d:«       |j3                  «        t        d<«       y # t,        $ r}t        d'|› �«       Y d }~�ŒŽd }~ww xY wc c}w # t,        $ r}t        d;|› �«       Y d }~Œbd }~ww xY w)=Nr   é<   r.   zPRAGMA journal_mode=WALzPRAGMA busy_timeout=60000c              3   ó&   K  — | ]	  }|d    –— Œ y­w)r   N© )Ú.0r)   s     r+   ú	<genexpr>zmain.<locals>.<genexpr>u   s   è ø€ ò T˜Q�A�a•Dñ Tùs   ‚z.SELECT page_url FROM gov_raw WHERE site_name=?u     [DB] å·²å…¥åº“ u    æ�¡ï¼Œé‡�å¤�URLè·³è¿‡è¯¦æƒ…é   iç  z

--- Page ú/z ---Únumr/   z  [SKIP] page z
 API errorÚdataÚtotalr   Úresultz  API: z items (total: z	, pages: ú)Ú	docpuburlr?   ÚdoctitleÚpubdatez  [WARN] no content for é(   z, using plain textÚ
doccontentú-zƒINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category) VALUES (?, ?, ?, ?, ?, ?, ?)Úhjxxr   z  [ERR] insert: g333333Ó?z  Page z: +z new, z skip (total +éÿÿÿÿé   z/
  All remaining pages before cutoff, stopping.ú
z2==================================================zFinal: z dup, z skip, z failz
Rebuilding FTS index...z(DELETE FROM gov_search WHERE site_name='ú'z4INSERT INTO gov_search(gov_search) VALUES('rebuild')z  FTS index rebuilt.z  [ERR] FTS rebuild: zDone.)Úsqlite3Úconnectr	   ÚexecuteÚcursorÚsetÚ	SITE_NAMEÚfetchallr'   ÚlenÚ_MAX_PAGES_ARGr,   r2   r@   r!   ÚminrE   Ú	CUTOFF_TSr=   ÚreplaceÚrowcountÚcommitr&   ÚtimeÚsleepÚclose)r^   ÚconnÚcurÚ	_existingÚ	total_newÚ
total_skipÚ	total_dupÚ
total_failr   Útotal_pagesÚresrO   rP   Ú	page_sizerQ   Úpage_newÚ	page_skipÚitemrS   rT   Ú
pubdate_tsÚdate_strÚcontent_htmlÚ	date_rankr*   r)   Ú	unskippeds                              r+   Úmainr�   m   s�  € Ûà�?‰?œ9¨bˆ?Ó1€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø
�+‰+‹-€Cäñ T #§+¡+Ø8¼9¸,ó#HßHPÉË
ôTó T€Iä	Ðœc )›nÐ-Ð-IÐ
JÔKà€IØ€JØ€IØ€JØ€DÜ$2°QÒ$6•.¸C€Kà
�+Ó
Ü�˜D˜6  ; -¨tÐ4Ô5Ü˜TÓ"ˆÙ�c—g‘g˜e“n¨Ò+Ü�N 4 &¨
Ð3Ô4Ùà�w‰w�v˜rÓ"ˆÜ�D—H‘H˜W aÓ(Ó)ˆÜ˜Ÿ™ ¬YÓ7Ó8ˆ	Ø˜yÑ(¨1Ñ,°Ñ:ˆÜ˜AÒÜ˜k¬>Ó:ˆKØ—‘˜( BÓ'ˆáÙä�œ˜F›�} O°E°7¸)ÀKÀ=ÐPQÐRÔSàˆØˆ	Øó -	ˆDØŸ™ ¨bÓ1ˆIØ—x‘x 
¨BÓ/ˆHÙ¡HØàŸ™ )¨QÓ/ˆJÜ! *Ó-ˆHáœC 
›O¬iÒ7Ø˜Q‘�	Ø˜a‘�
Øà˜IÑ%Ø˜Q‘�	Ø˜a‘�
Øô (¨	Ó2ˆLÙÜÐ0°¸#¸2°°Ð?QÐRÔSØ#Ÿx™x¨Ó5Ò;¸�Ø˜a‘�
á:Bœ˜H×,Ñ,¨S°"Ó5Ô6ÈˆIð.Ø—‘ð3ô  ¨)°X¸|ÈYÐX^Ð_ô	ð —<‘< !Ò#Ø ‘M�HØ ‘N�IØ "‘}¨Ò)ØŸ™�à ‘N�Iô �J‰J�sŽOð[-	ð^ 	�‰ŒÜ�˜�v˜S  
¨&°°¸>È)ÈÐTUÐVÔWá”c˜& ™*Ÿ.™.¨°AÓ6Ó7¼)ÒCØœ3˜v›;¨™?Ò*Ø(.ÖX 1´#°a·e±e¸IÀqÓ6IÓ2JÌiÓ2WšQÐX�	ÐXÙ ÜÐLÔMØà�‰	ˆÜ�
‰
�3Œða �+Ô
ôd 
ˆBˆvˆhˆ-ÔÜ	ˆG�I�;˜f Y K¨v°j°\ÀÈÈÐTYÐ
ZÔ[à�1‚}ÜÐ)Ô*ð	/Ø�K‰KÐBÄ9À+ÈQÐOÔPØ�K‰KÐNÔOØ�K‰KŒMÜÐ(Ô)ð 	‡J�J„LÜ	ˆ'…NøôA ò .ÜÐ(¨¨Ð,×-Ò-ûð.üò Yøô$ ò 	/ÜÐ)¨!¨Ð-×.Ñ.ûð	/ús>   É;AQ Í'Q%Í3Q%ÏAQ* Ñ 	Q"Ñ	QÑQ"Ñ*	RÑ3RÒRÚ__main__).Ú__doc__ÚosÚreÚsysrl   r   Úrequestsr   r   Úbs4r   ÚargparseÚ_APÚArgumentParserÚ
_AP_PARSERÚadd_argumentr@   Úparse_known_argsÚ_AP_ARGSÚ_Úpagesrf   Úgetenvr	   rc   r$   ÚBASE_URLr!   ÚnowrB   ÚCUTOFFr'   ÚstrptimeÚ	timestamprh   ÚHEADERSÚSessionr"   ÚheadersÚupdater1   r,   r=   rE   r�   Ú__name__rI   ó    r+   ú<module>rž      sw  ðñó 
Û 	Û 
Û Û Û ß (Ý ã ØˆS×ÑÓ!€
Ø 
× Ñ ˜	¨°Q¸^Ð Ô LØ×)Ñ)Ó+�€ˆ!Ø—‘€àˆB�I‰I�kÐ#4Ó5€	Ø=€	Ø
B€Ø(€Ø€	Ø
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€Ù ˆ�f�XÐÔ ÙÐ!�×!Ñ! &¨*Ó5×?Ñ?ÓAÀDÑHÓI€	ðGà&ñ€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ Ø!�×!Ñ!Ó#€Ø × Ñ × Ñ ØÐPðô ò
ò,ò@òsðl ˆzÒÙ…Fð r�   