Ë
    $¸£j  ã                   ó(  — d Z ddlZddlZddlZddlZddlZddlZddlm	Z	m
Z
 ddlZddlmZ dZddiZdZ ej"                  d	d
«      ZddlmZmZ  ej*                  «        ed¬«      z
  j-                  d«      Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uÑ   
crawl_gxq_yl_gov.py â€” æ¦†æž—é«˜æ–°æŠ€æœ¯äº§ä¸šå¼€å�‘åŒº - é€šçŸ¥å…¬å‘Š
TRS CMS, static HTML pagination (index.html, index_1.html, ..., index_19.html)
Detail: meta ArticleTitle, PubDate, TRS_UEDITOR content
é    N)ÚThreadPoolExecutorÚas_completed)ÚBeautifulSoupzhttps://gxq.yl.gov.cn/xwzx/tzggz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36u.   æ¦†æž—é«˜æ–°æŠ€æœ¯äº§ä¸šå¼€å�‘åŒº-é€šçŸ¥å…¬å‘ŠÚ	SEARCH_DBz/root/search.db)ÚdatetimeÚ	timedeltaiG  )Údaysz%Y-%m-%dc                 óô  — | dk(  r
t         › d�}nt         › d| › d�}	 t        j                  |t        dd¬«      }d|_        t        |j                  d«      }g }|j                  dt        j                  d«      ¬«      D ]Í  }|j                  dd¬«      }|sŒ|j                  dd«      }|j                  d«      rd|vrŒA|j                  dd«      xs |j                  d¬«      }	|	sŒjt        j                   j#                  t         dz   |«      }
|j                  dd¬«      }|r|j                  d¬«      nd}|j%                  |	|
|f«       ŒÏ |S # t
        $ r}t        d	| › d
|› �«       g cY d}~S d}~ww xY w)z1Fetch list page, return [(title, url, date), ...]r   z/index.htmlz/index_z.htmlé   F©ÚheadersÚtimeoutÚverifyúutf-8z  [ERROR] fetch list page ú: Núhtml.parserÚliz^(odd|even)$©Úclass_ÚaT)Úhrefr   Ú Út202Útitle)Ústripú/ÚspanÚdate)ÚBASE_URLÚrequestsÚgetÚHEADERSÚencodingÚ	ExceptionÚprintr   ÚtextÚfind_allÚreÚcompileÚfindÚendswithÚget_textÚurllibÚparseÚurljoinÚappend)Úpage_idxÚurlÚrespÚeÚsoupÚitemsr   Úa_tagr   r   Úfull_urlÚ	date_spanÚdate_strs                úcrawl_gxq_yl_tzgg.pyÚ
fetch_listr<      su  € à�1‚}Ü�
˜+Ð&‰ä�
˜' ( ¨5Ð1ˆðÜ�|‰|˜C¬¸"ÀUÔKˆØˆŒô
 ˜Ÿ™ MÓ2€DØ€Eà�m‰m˜D¬¯©°OÓ)DˆmÓEò 2ˆØ—‘˜ $�Ó'ˆÙØà�y‰y˜ Ó$ˆØ�}‰}˜WÔ%¨°tÑ);Øà—	‘	˜' 2Ó&ÒD¨%¯.©.¸t¨.Ó*DˆÙØô —<‘<×'Ñ'¬°3©¸Ó=ˆð —G‘G˜F¨6�GÓ2ˆ	Ù5>�9×%Ñ%¨DÐ%Ô1ÀBˆà�‰�e˜X xÐ0Õ1ð)2ð, €Løô; ò ÜÐ*¨8¨*°B°q°cÐ:Ô;Ø�	ûðús   �$E Å	E7ÅE2Å,E7Å2E7c                 óŒ  — 	 t        j                  | t        dd¬«      }d|_        |j                  }t        |d	«      }d
}|j                  dddi¬«      }|r$|j                  d«      r|d   j                  «       }d
}|j                  dddi¬«      }|r'|j                  d«      r|d   j                  «       dd }d
}	|j                  dddi¬«      }
|
r$|
j                  d«      r|
d   j                  «       }	|j                  dt        j                  d«      ¬«      }|s4|j                  t        j                  dt        j                  «      ¬«      }|s4|j                  t        j                  dt        j                  «      ¬«      }d
}|rt        |«      }||||	fS # t        $ r}t        d| › d|› �«       Y d}~yd}~ww xY w)zAFetch detail page, return (content_html, title, pub_date, source)r   Fr   r   z  [ERROR] fetch detail r   N)r   r   r   r   r   r   ÚmetaÚnameÚArticleTitle)ÚattrsÚcontentÚPubDateé
   ÚContentSourceÚdivz&TRS_UEDITOR|trs_editor_view|TRS_Editorr   zzoom|content|article)Úidzzoom|content|article|maintext)r    r!   r"   r#   r$   r%   r&   r   r*   r   r(   r)   ÚIÚstr)r2   r3   r4   Úhtmlr5   r   Ú
meta_titleÚpub_dateÚ	meta_dateÚsourceÚmeta_sourceÚcontent_divrB   s                r;   Úfetch_detailrQ   @   s±  € ð Ü�|‰|˜C¬¸"ÀUÔKˆØˆŒð
 �9‰9€DÜ˜˜}Ó-€Dð €EØ—‘˜6¨&°.Ð)A�ÓB€JÙ�j—n‘n YÔ/Ø˜9Ñ%×+Ñ+Ó-ˆð €HØ—	‘	˜&¨°Ð(;�	Ó<€IÙ�Y—]‘] 9Ô-Ø˜YÑ'×-Ñ-Ó/°°Ð4ˆð €FØ—)‘)˜F¨6°?Ð*C�)ÓD€KÙ�{—‘ yÔ1Ø˜YÑ'×-Ñ-Ó/ˆð —)‘)˜E¬"¯*©*Ð5^Ó*_�)Ó`€KÙà—i‘i¤2§:¡:Ð.EÄrÇtÁtÓ#L�iÓMˆÙØ—i‘i¤r§z¡zÐ2RÔTV×TXÑTXÓ'Y�iÓZˆà€GÙÜ�kÓ"ˆà�U˜H fÐ-Ð-øôK ò  ÜÐ'¨ u¨B¨q¨cÐ2Ô3Üûð ús   ‚$F Æ	GÆ(F>Æ>Gc                 ó°  — ddl }|j                  t        d¬«      }|j                  d«       |j                  d«       |j	                  «       }d}| D ]:  \  }}}}}		 |j                  dt
        |||||	df«       |j                  dkD  r|d	z  }Œ< |j                  «        |j                  «        |S # t        $ r}
t        d
|› d|
› �«       Y d}
~
Œ~d}
~
ww xY w)zBulk insert into gov_rawr   Né   )r   zPRAGMA journal_mode=WALzPRAGMA busy_timeout=5000zÇ
                INSERT OR IGNORE INTO gov_raw 
                    (site_name, title, page_url, publish_date, source_url, content, category)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            u   é€šçŸ¥å…¬å‘Šé   z  [DB ERROR] r   )Úsqlite3ÚconnectÚDB_PATHÚexecuteÚcursorÚ	SITE_NAMEÚrowcountr$   r%   ÚcommitÚclose)r6   rU   ÚconnÚcurÚ	new_countr   r2   rL   rN   rB   r4   s              r;   Úpush_to_searchdbra   l   sâ   € ãØ�?‰?œ7¨Bˆ?Ó/€DØ‡L�LÐ*Ô+Ø‡L�LÐ+Ô,Ø
�+‰+‹-€Cà€IØ16ò 
.Ñ-ˆˆs�H˜f gð		.Ø�K‰Kð ô ˜U C¨°6¸7ÀNÐSô	Uð
 �|‰|˜aÒØ˜Q‘�	øð
.ð 	‡K�K„MØ‡J�J„LØÐøô ò 	.Ü�M #  b¨¨Ð,×-Ñ-ûð	.ús   Á1B1Â1	CÂ:CÃCc                  ó¤  — dd l } | j                  dd¬«       t        dt        j                  «       j                  d«      › dt        › �«       g }t        «       }d}t        |«      D ]á  }t        |«      }|st        d	|d
z   › d�«        nÀg }|D ]I  \  }}}	|	r4|	t        k\  r+||vsŒ|j                  |«       |j                  |||	f«       Œ=|	sŒ@|	t        k  sŒJŒK |j                  |«       t        d	|d
z   › dt        |«      › dt        |«      › d�«       |r|d   d   t        k  rt        d«        nt        j                   d«       Œã t        dt        |«      › �«       g }
t#        d¬«      5 }|D ���	ci c]   \  }}}	|j%                  t&        |«      |||	f“Œ" }}}}	t)        t+        |«      d
«      D ]—  \  }}||   \  }}}		 |j-                  «       \  }}}}|xs |}|xs |	}|rD|
j                  |||||f«       t        d|› dt        |«      › d|d d › dt        |«      › d�	«       n!t        d|› dt        |«      › d|d d › d�«       Œ™ 	 d d d «       d}|
rt1        |
«      }t        d|› d�«       |S t        d «       |S c c}	}}w # t.        $ r.}t        d|› dt        |«      › d|d d › d|› �«       Y d }~�Œ	d }~ww xY w# 1 sw Y   ŒwxY w)!Nr   ÚignorezUnverified HTTPS request)Úmessageú[z%H:%M:%Sz] é   z  Page rT   z: 0 items (stopping)r   z items, z within 3yréÿÿÿÿé   z2  -> Reached items older than 3yr cutoff, stoppingg      à?z  Total within 3 years: é   )Úmax_workersz  [r   é2   z... OK (zB)z... no contenté(   z... ERROR: z
  === Done === New: z, Errors: 0z
  === Done === No new items)ÚwarningsÚfilterwarningsr%   r   ÚnowÚstrftimerZ   ÚsetÚranger<   ÚCUTOFFÚaddr0   ÚextendÚlenÚtimeÚsleepr   ÚsubmitrQ   Ú	enumerater   Úresultr$   ra   )rm   Ú	all_itemsÚ	seen_urlsÚtotal_pagesr1   r6   Ú
page_itemsr   r2   r:   ÚdetailsÚexecutorÚ
future_mapÚiÚfuturerB   Ú	det_titlerL   rN   Úfinal_titleÚ
final_dater4   r`   s                          r;   Úmainrˆ   …   s9  € ÛØ×Ñ˜HÐ.HÐÔIä	ˆAŒh�l‰l‹n×%Ñ% jÓ1Ð2°"´Y°KÐ
@ÔAð €IÜ“€Ið €Kä˜+Ó&ò ˆÜ˜8Ó$ˆÙÜ�G˜H Q™J˜<Ð';Ð<Ô=Ùð ˆ
Ø$)ò 	Ñ ˆE�3˜Ù˜H¬Ò.Ø˜iÒ'Ø—M‘M #Ô&Ø×%Ñ% u¨c°8Ð&<Õ=Ú˜h¬Ó/Øð	ð 	×Ñ˜Ô$Ü�˜ ™
�| 2¤c¨%£j \°¼#¸j»/Ð9JÈ+ÐVÔWñ �U˜2‘Y˜q‘\¤FÒ*ÜÐFÔHÙä�
‰
�3�ð1ô4 
Ð$¤S¨£^Ð$4Ð
5Ô6ð €GÜ	¨Ô	*ð N¨hð )2÷
ð 
á$��s˜Hð �O‰OœL¨#Ó.°¸¸XÐ0FÑFð
ˆ
ò 
ô
 #¤<°
Ó#;¸QÓ?ò 	N‰IˆAˆvØ#-¨fÑ#5Ñ ˆE�3˜ðNØ7=·}±}³Ñ4�˜ H¨fØ'Ò0¨5�Ø%Ò1¨�
áØ—N‘N K°°jÀ&È'Ð#RÔSÜ˜C ˜s !¤C¨	£NÐ#3°2°kÀ#À2Ð6FÐ5GÀxÔPSÐT[ÓP\È~Ð]_Ð`Õaä˜C ˜s !¤C¨	£NÐ#3°2°kÀ#À2Ð6FÐ5GÀ~ÐVÔWøñ	N÷Nð, €IÙÜ$ WÓ-ˆ	ÜÐ& y k°Ð=Ô>ð Ðô 	Ð-Ô/àÐùô9
øô" ò NÜ˜˜A˜3˜a¤ I£Ð/¨r°%¸¸°*°¸[ÈÈÐL×MÒMûðNú÷%Nð NúsI   Å0KÅ7%J
Æ(KÇBJÉKÊKÊ	KÊ#J>Ê8KÊ>KËKËKÚ__main__)Ú__doc__r(   ÚsysÚjsonrw   ÚosÚurllib.parser-   Úconcurrent.futuresr   r   r    Úbs4r   r   r"   rZ   ÚgetenvrW   r   r   ro   rp   rs   r<   rQ   ra   rˆ   Ú__name__© ó    r;   ú<module>r•      s    ðñ÷
 -× ,× ,ß ?Û Ý à,€àð  Dð€ð =€	Ø
ˆ"�)‰)�KÐ!2Ó
3€÷ )Ø
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€ò'òR*.òXò2HðT ˆzÒÙ…Fð r”   