Ë
    ^¨£jÕ  ã                   óÊ  — d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
 ddlZ ej                  «       Zej                  dedd¬«       ej#                  «       \  ZZej(                  Z ej,                  dd	«      Zd
ZdZdZdZ ej8                  «        ed¬«      z
  j;                  d«      Z ede› �«        e ej@                  ed«      jC                  «       dz  «      Z"dddœZ# ejH                  «       Z%e%jL                  jO                  e#«        ejH                  «       Z(e(jL                  jO                  ddi«       d„ Z)d„ Z*d„ Z+d„ Z,e-dk(  r e,«        yy)uÚ   
æ™‹æ±Ÿå¸‚äººæ°‘æ”¿åºœ - ç”Ÿæ€�çŽ¯å¢ƒå±€ å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·
CMS: Terton (å¤©æ¶¦) / SSP æ··å�ˆ
åˆ—è¡¨: POST /ssp/search/api/v2/external (æ‰¹é‡�100æ�¡)
è¯¦æƒ…: å�– detail HTML æ­£æ–‡ (å�«è¡¨æ ¼ç­‰å®Œæ•´HTML)
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupz--pagesu   é™�åˆ¶é¡µæ•°)ÚtypeÚdefaultÚhelpÚ	SEARCH_DBz/root/search.dbu   jinjiang.gov.cn-çŽ¯è¯„ä¿¡æ�¯z6https://www.jinjiang.gov.cn/ssp/search/api/v2/externalzhttps://www.jinjiang.gov.cnéd   iG  )Údaysú%Y-%m-%dzCutoff date: éè  zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zapplication/json)ú
User-AgentzContent-Typer   z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36c                 óä   — ddt         | dddigddigdœdœ}	 t        j                  t        |d	¬
«      }d|_        |j                  «       S # t        $ r}t        d| › d|› �«       Y d}~yd}~ww xY w)z%Fetch one page of items from the API.Ú 000000008fd72c27018fdd69fb8b0002ÚWCMOPENINFOAPIzdocreltime descÚeqé
   Ú38080)Ú	docstatusÚchnlid)ÚsiteIdÚapiNameÚpageSizeÚpageÚ	sortFieldÚfilteré   )ÚjsonÚtimeoutúutf-8z  [ERR] API page z: N)Ú	PAGE_SIZEÚsessionÚpostÚAPI_URLÚencodingr   Ú	ExceptionÚprint)r   ÚbodyÚrÚes       úcrawl_jinjiang.pyÚfetch_api_pager,   /   s†   € ð 5Ø#ÜØØ&à ˜*˜Ø˜g�Ð'ñ
ñ
€DðÜ�L‰Lœ t°RˆLÓ8ˆØˆŒ
Ø�v‰v‹xˆøÜò ÜÐ! $  r¨!¨Ð-Ô.Üûðús   ˜2A Á	A/ÁA*Á*A/c                 óØ  — 	 t         j                  | d¬«      }d|_        |j                  dk7  ry	 t        |j                  d	«      }|j                  d
«      }|rt        |«      }|S |j                  d«      }|r)|j                  d«      }|rt        |«      S t        |«      S |j                  d«      }|rt        |«      S y# t        $ r}t        d| dd › d|› �«       Y d}~yd}~ww xY w)z=Fetch detail page and extract content HTML from article_area.r   ©r   r    éÈ   Nz  [ERR] detail fetch: éP   z - zhtml.parserz.TRS_Editorz.article_areazdiv[class*=font]z.content, #zoom, .g-detailbox)
Úsession_detailÚgetr%   Ústatus_coder&   r'   r   ÚtextÚ
select_oneÚstr)	Úurlr)   r*   ÚsoupÚtrsÚcontentÚareaÚfont_divÚcontent_divs	            r+   Úfetch_detailr>   E   sö   € ðÜ×Ñ˜s¨BÐÓ/ˆØˆŒ
Ø�=‰=˜CÒØð  ô ˜Ÿ™ Ó/€Dð �/‰/˜-Ó
(€CÙ
ä�c“(ˆØˆð �?‰?˜?Ó+€DÙà—?‘?Ð#5Ó6ˆÙÜ�x“=Ð Ü�4‹yÐð —/‘/Ð"AÓB€KÙÜ�;ÓÐàøô7 ò ÜÐ& s¨3¨B x j°°A°3Ð7Ô8Üûðús   ‚-C Ã	C)ÃC$Ã$C)c                 óv   — 	 t        | «      dz  }t        j                  |«      j                  d«      S #  Y yxY w)Nr   r   Ú )Úintr   ÚfromtimestampÚstrftime)Úts_msÚtss     r+   Ú
ts_to_daterF   j   s;   € ðÜ�‹Z˜$ÑˆÜ×%Ñ% bÓ)×2Ñ2°:Ó>Ð>øðÙús   ‚14 ´8c                  óB	  — dd l } | j                  t        d¬«      }|j                  d«       |j                  d«       |j	                  «       }t        d„ |j                  dt        f«      j                  «       D «       «      }t        dt        |«      › d	�«       d}d}d}d}d
}t        dkD  rt        nd}	||	k  �rÿt        d|› d|	› d�«       t        |«      }
|
r|
j                  d«      dk7  rt        d|› d�«       �n»|
j                  di «      }t        |j                  dd«      «      }t        |j                  dt        «      «      }||z   d
z
  |z  }	t        dkD  rt        |	t        «      }	|j                  dg «      }|s�n5t        dt        |«      › d|› d|	› d�«       d}d}|D �]L  }|j                  dd«      }|j                  dd«      }|r|sŒ-|j                  dd«      }t!        |«      }|rt        |«      t"        k  r|d
z  }|d
z  }Œi||v r|d
z  }|d
z  }Œxt%        |«      }|s?t        d|d d  › d!�«       |j                  d"«      xs |j                  d#«      xs d}|d
z  }|rt        |j'                  d$d«      «      nd}	 |j                  d%t        |||||d&f«       |j(                  dkD  r#|d
z  }|d
z  }|d'z  dk(  r|j+                  «        n|d
z  }t/        j0                  d)«       �ŒO |j+                  «        t        d*|› d+|› d,|› d-|› d�	«       |rvt        |d.   j                  dd«      «      t"        k  rQ|t        |«      d/z  kD  r@|D �cg c]'  }t        |j                  dd«      «      t"        k\  sŒ&|‘Œ) }}|st        d0«       n!|d
z  }t/        j0                  d)«       ||	k  r�Œÿt        d1d2› �«       t        d3|› d,|› d4|› d5|› d6�	«       |dkD  rQt        d7«       	 |j                  d8t        › d9�«       |j                  d:«       |j+                  «        t        d;«       |j3                  «        t        d=«       y # t,        $ r}t        d(|› �«       Y d }~�ŒŽd }~ww xY wc c}w # t,        $ r}t        d<|› �«       Y d }~Œbd }~ww xY w)>Nr   é<   r.   zPRAGMA journal_mode=WALzPRAGMA busy_timeout=60000c              3   ó&   K  — | ]	  }|d    –— Œ y­w)r   N© )Ú.0r)   s     r+   ú	<genexpr>zmain.<locals>.<genexpr>z   s   è ø€ ò T˜Q�A�a•Dñ Tùs   ‚z.SELECT page_url FROM gov_raw WHERE site_name=?u     [DB] å·²å…¥åº“ u    æ�¡ï¼Œé‡�å¤�URLè·³è¿‡è¯¦æƒ…é   iç  z

--- Page ú/z ---Únumr/   z  [SKIP] page z
 API errorÚdataÚtotalr   Úresultz  API: z items (total: z	, pages: ú)Ú	docpuburlr@   ÚdoctitleÚpubdatez  [WARN] no content for é(   z, using plain textÚ
doccontentÚ_doccontentú-zƒINSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category) VALUES (?, ?, ?, ?, ?, ?, ?)Úhjxxr   z  [ERR] insert: g333333Ó?z  Page z: +z new, z skip (total +éÿÿÿÿé   z/
  All remaining pages before cutoff, stopping.ú
z2==================================================zFinal: z dup, z skip, z failz
Rebuilding FTS index...z(DELETE FROM gov_search WHERE site_name='ú'z4INSERT INTO gov_search(gov_search) VALUES('rebuild')z  FTS index rebuilt.z  [ERR] FTS rebuild: zDone.)Úsqlite3Úconnectr	   ÚexecuteÚcursorÚsetÚ	SITE_NAMEÚfetchallr'   ÚlenÚ_MAX_PAGES_ARGr,   r2   rA   r!   ÚminrF   Ú	CUTOFF_TSr>   ÚreplaceÚrowcountÚcommitr&   ÚtimeÚsleepÚclose)r`   ÚconnÚcurÚ	_existingÚ	total_newÚ
total_skipÚ	total_dupÚ
total_failr   Útotal_pagesÚresrP   rQ   Ú	page_sizerR   Úpage_newÚ	page_skipÚitemrT   rU   Ú
pubdate_tsÚdate_strÚcontent_htmlÚ	date_rankr*   r)   Ú	unskippeds                              r+   Úmainrƒ   r   s¬  € Ûà�?‰?œ9¨bˆ?Ó1€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø
�+‰+‹-€Cäñ T #§+¡+Ø8¼9¸,ó#HßHPÉË
ôTó T€Iä	Ðœc )›nÐ-Ð-IÐ
JÔKà€IØ€JØ€IØ€JØ€DÜ$2°QÒ$6•.¸C€Kà
�+Ó
Ü�˜D˜6  ; -¨tÐ4Ô5Ü˜TÓ"ˆÙ�c—g‘g˜e“n¨Ò+Ü�N 4 &¨
Ð3Ô4Ùà�w‰w�v˜rÓ"ˆÜ�D—H‘H˜W aÓ(Ó)ˆÜ˜Ÿ™ ¬YÓ7Ó8ˆ	Ø˜yÑ(¨1Ñ,°Ñ:ˆÜ˜AÒÜ˜k¬>Ó:ˆKØ—‘˜( BÓ'ˆáÙä�œ˜F›�} O°E°7¸)ÀKÀ=ÐPQÐRÔSàˆØˆ	Øó -	ˆDØŸ™ ¨bÓ1ˆIØ—x‘x 
¨BÓ/ˆHÙ¡HØàŸ™ )¨QÓ/ˆJÜ! *Ó-ˆHáœC 
›O¬iÒ7Ø˜Q‘�	Ø˜a‘�
Øà˜IÑ%Ø˜Q‘�	Ø˜a‘�
Øô (¨	Ó2ˆLÙÜÐ0°¸#¸2°°Ð?QÐRÔSØ#Ÿx™x¨Ó5ÒV¸¿¹À-Ó9PÒVÐTV�Ø˜a‘�
á:Bœ˜H×,Ñ,¨S°"Ó5Ô6ÈˆIð.Ø—‘ð3ô  ¨)°X¸|ÈYÐX^Ð_ô	ð —<‘< !Ò#Ø ‘M�HØ ‘N�IØ "‘}¨Ò)ØŸ™�à ‘N�Iô �J‰J�sŽOð[-	ð^ 	�‰ŒÜ�˜�v˜S  
¨&°°¸>È)ÈÐTUÐVÔWá”c˜& ™*Ÿ.™.¨°AÓ6Ó7¼)ÒCØœ3˜v›;¨™?Ò*Ø(.ÖX 1´#°a·e±e¸IÀqÓ6IÓ2JÌiÓ2WšQÐX�	ÐXÙ ÜÐLÔMØà�‰	ˆÜ�
‰
�3Œða �+Ô
ôd 
ˆBˆvˆhˆ-ÔÜ	ˆG�I�;˜f Y K¨v°j°\ÀÈÈÐTYÐ
ZÔ[à�1‚}ÜÐ)Ô*ð	/Ø�K‰KÐBÄ9À+ÈQÐOÔPØ�K‰KÐNÔOØ�K‰KŒMÜÐ(Ô)ð 	‡J�J„LÜ	ˆ'…NøôA ò .ÜÐ(¨¨Ð,×-Ò-ûð.üò Yøô$ ò 	/ÜÐ)¨!¨Ð-×.Ñ.ûð	/ús>   ÊAQÍ'Q8ÎQ8Ï2AQ= Ñ	Q5ÑQ0Ñ0Q5Ñ=	RÒRÒRÚ__main__).Ú__doc__ÚosÚreÚsysrn   r   Úrequestsr   r   Úbs4r   ÚargparseÚ_APÚArgumentParserÚ
_AP_PARSERÚadd_argumentrA   Úparse_known_argsÚ_AP_ARGSÚ_Úpagesrh   Úgetenvr	   re   r$   ÚBASE_URLr!   ÚnowrC   ÚCUTOFFr'   ÚstrptimeÚ	timestamprj   ÚHEADERSÚSessionr"   ÚheadersÚupdater1   r,   r>   rF   rƒ   Ú__name__rJ   ó    r+   ú<module>r       sw  ðñó 
Û 	Û 
Û Û Û ß (Ý ã ØˆS×ÑÓ!€
Ø 
× Ñ ˜	¨°Q¸^Ð Ô LØ×)Ñ)Ó+�€ˆ!Ø—‘€àˆB�I‰I�kÐ#4Ó5€	Ø*€	Ø
B€Ø(€Ø€	Ø
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€Ù ˆ�f�XÐÔ ÙÐ!�×!Ñ! &¨*Ó5×?Ñ?ÓAÀDÑHÓI€	ðGà&ñ€ð ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ Ø!�×!Ñ!Ó#€Ø × Ñ × Ñ ØÐPðô ò
ò,"òJòsðl ˆzÒÙ…Fð rŸ   