Ë
    ¸¬ujº  ã                   ó¬  — d Z ddlZddlZddlZddlZddlZddlmZmZ ddlm	Z	m
Z
 ddlmZ ej                  j                  dej                  j                  ej                  j!                  e«      «      «       ddlZddlmZ dZd„ Zd	Zd
ZddiZ ej4                  «        ed¬«      z
  j7                  d«      Zd„ Zd„ Zd„ Zd„ Z e!dk(  r e «        yy)u€   
æ´¥å¸‚å¸‚äººæ°‘æ”¿åºœ - å…¬ç¤ºå…¬å‘Š çˆ¬è™«
CMS: PowerCMS (static HTML pagination)
URL: https://www.jinshishi.gov.cn/zwxx/gsgg
é    N)ÚdatetimeÚ	timedelta)ÚThreadPoolExecutorÚas_completed)Úurljoin)ÚBeautifulSoupz/root/search.dbc                  óZ   — dd l } | j                  t        «      }|j                  d«       |S )Nr   zPRAGMA journal_mode=WAL)Úsqlite3ÚconnectÚDB_PATHÚexecute)r
   Úconns     ú&/root/gov_crawler/crawl_jinshi_gsgg.pyÚget_connectionr      s%   € ÛØ�?‰?œ7Ó#€DØ‡L�LÐ*Ô+Ø€Kó    u   æ´¥å¸‚å¸‚-å…¬ç¤ºå…¬å‘Šz&https://www.jinshishi.gov.cn/zwxx/gsggz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36iG  )Údaysz%Y-%m-%dc                 ó¤  — | dk(  rt         }nt         › d| › �}	 t        j                  |t        d¬«      }d|_        t        |j                  d«      }|j                  d
d¬«      }|sg S g }|j                  d«      D ]¨  }|j                  d«      }|r|j                  d«      sŒ(|d   j                  «       }	t        t         |	«      }
|j                  d¬«      }t        j                  d|d	d «      r|d	d nd}|r|dd	 j                  «       n|}|j!                  ||
|dœ«       Œª |S # t        $ r}t        d| › d|› �«       g cY d	}~S d	}~ww xY w)z)Crawl a single list page and return itemsé   Ú_é   ©ÚheadersÚtimeoutúutf-8úhtml.parserz[List] Error page ú: NÚulÚnewsList)Úclass_ÚliÚaÚhrefT)Ústripz\d{4}-\d{2}-\d{2}é
   Ú )ÚtitleÚurlÚdate_str)ÚBASE_URLÚrequestsÚgetÚHEADERSÚencodingr   ÚtextÚ	ExceptionÚprintÚfindÚfind_allr#   r   Úget_textÚreÚmatchÚappend)Úpage_numr'   ÚrespÚsoupÚer   Úitemsr    r!   r"   Úfull_urlÚtxtr(   r&   s                 r   Úcrawl_list_pager>   %   sU  € à�1‚}Ü‰ä�
˜!˜H˜:Ð&ˆðÜ�|‰|˜C¬¸"Ô=ˆØˆŒÜ˜TŸY™Y¨Ó6ˆð
 
�‰�4 
ˆÓ	+€BÙØˆ	à€EØ�k‰k˜$Óò ˆØ�G‰G�C‹LˆÙ˜Ÿ™˜fœØà�‰y�‰Ó ˆÜœ8 TÓ*ˆØ�k‰k ˆkÓ%ˆô  "Ÿx™xÐ(<¸cÀ#À2¸hÔG�3�s˜‘8ÈRˆÙ$,��B�C�—‘Ô °#ˆà�‰ØØØ ñ
õ 	ðð& €Løô9 ò ÜÐ" 8 *¨B¨q¨cÐ2Ô3Ø�	ûðús   ™9D) Ä)	EÄ2E
ÅEÅ
Ec                 óÎ  — | d   }	 t        j                  |t        d¬«      }d|_        t	        |j
                  d«      }| d	   }|j                  d
ddi¬«      }|r$|j                  d«      r|d   j                  «       }| d   }|j                  d
ddi¬«      }|r'|j                  d«      r|d   j                  «       dd }d}	dD ]"  }
|j                  |
«      }|sŒt        |«      }	 n |	s|j                  d«      }|rt        |«      }	d}|	r"t	        |	d«      }|j                  dd¬«      dd }t        |||||	|dœS # t        $ r}t        d|› d|› �«       Y d}~yd}~ww xY w)z'Fetch a detail page and extract contentr'   r   r   r   r   z[Detail] Error r   Nr&   ÚmetaÚnameÚArticleTitle)ÚattrsÚcontentr(   ÚPubDater$   r%   )zarticle.articleConzdiv.printAreaz#contentú T)Ú	separatorr#   i,  )Ú	site_namer&   Úpage_urlÚpublish_dateÚ
source_urlrD   Úsummary)r*   r+   r,   r-   r   r.   r/   r0   r1   r#   Ú
select_oneÚstrr3   Ú	SITE_NAME)Úitemr'   r8   r9   r:   r&   Úmtr(   ÚpdÚ	body_htmlÚselÚdivÚcontent_divÚsummary_textÚtxt_soups                  r   Úcrawl_detailrY   O   s–  € à
ˆu‰+€CðÜ�|‰|˜C¬¸"Ô=ˆØˆŒÜ˜TŸY™Y¨Ó6ˆð �‰M€EØ	�‰�6 &¨.Ð!9ˆÓ	:€BÙ	ˆb�f‰f�YÔØ�9‘×#Ñ#Ó%ˆð �JÑ€HØ	�‰�6 &¨)Ð!4ˆÓ	5€BÙ	ˆb�f‰f�YÔØ�i‘=×&Ñ&Ó(¨¨"Ð-ˆð €IØ6ò ˆØ�o‰o˜cÓ"ˆÚÜ˜C›ˆIÙð	ñ à—o‘o jÓ1ˆÙÜ˜KÓ(ˆIð €LÙÜ  ¨MÓ:ˆØ×(Ñ(°3¸dÐ(ÓCÀDÀSÐIˆô ØØØ ØØØñð øôI ò Ü� ˜u B q cÐ*Ô+Üûðús   ‡9E  Å 	E$Å	EÅE$c                 ót  — t        «       }|j                  «       }d}| D ]R  }	 |j                  d|d   |d   |d   |d   |d   |d   |j                  d	d
«      f«       |j                  dkD  r|dz  }ŒT |j                  «        |j                  «        |S # t
        $ r}t        d|› �«       Y d }~Œ“d }~ww xY w)Nr   z±
                INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, source_url, content, summary)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            rH   r&   rI   rJ   rK   rD   rL   r%   r   z[DB] Error: )	r   Úcursorr   r+   Úrowcountr/   r0   ÚcommitÚclose)Úrecordsr   r[   ÚinsertedÚrecr:   s         r   Ú
save_to_dbrb   †   sÒ   € ÜÓ€DØ�[‰[‹]€FØ€HØò &ˆð	&Ø�N‰Nð ð �KÑ  # g¡,°°J±Ø�NÑ# S¨Ñ%6¸¸I¹Ø—‘˜	 2Ó&ðôð �‰ Ò"Ø˜A‘�øð&ð 	‡K�K„MØ‡J�J„LØ€Oøô	 ò 	&Ü�L  Ð$×%Ñ%ûð	&ús   £ABÂ	B7ÂB2Â2B7c                  ór  — t        dt        › d�«       t        dt        › �«       t        d«       g } t        dd«      D ]Ÿ  }t	        |«      }|st        d|› d�«        n�|D �cg c]  }|d	   t        k\  sŒ|‘Œ }}| j                  |«       t        d|› d
t        |«      › dt        | «      › d�«       |rt        d|d   d	   › �«       t        j                  d«       Œ¡ t        dt        | «      › �«       | st        d«       y t        dt        | «      › d�«       g }d}t        d¬«      5 }| D �ci c]  }|j                  t        |«      |“Œ }}t        t        |«      «      D ]X  \  }}	|	j                  «       }
|
r|j                  |
«       n|dz  }|dz   dz  dk(  sŒ;t        d|dz   › dt        | «      › d�«       ŒZ 	 d d d «       t        dt        |«      › d|› d�«       t        d«       t!        |«      }t        d|› d t        |«      › d�«       t        d!«       y c c}w c c}w # 1 sw Y   ŒkxY w)"Nz=== z ===u   3å¹´æˆªæ­¢: z
--- Step 1: List pages ---r   é,   z[List] Page z: empty, stoppingr(   r   z items (total: ú)z
  Oldest: éÿÿÿÿg333333Ó?z
Total items within 3 years: zNo items, exitingz
--- Step 2: z detail pages ---r   é   )Úmax_workersé   z	[Detail] ú/z donez	
Detail: z
 success, z errorsz
--- Step 3: Save to DB ---z	Inserted z new (total fetched: z
=== Done ===)r0   rO   ÚTHREE_YEARS_AGOÚranger>   ÚextendÚlenÚtimeÚsleepr   ÚsubmitrY   Ú	enumerater   Úresultr6   rb   )Ú	all_itemsÚpr;   Úitr_   ÚerrorsÚexecutorÚfut_mapÚiÚfutra   r`   s               r   Úmainr|   �   sB  € Ü	ˆD”�˜4Ð
 Ô!Ü	ˆLœÐ)Ð
*Ô+ô 
Ð
(Ô)Ø€IÜ�1�b‹\ò ˆÜ Ó"ˆÙÜ�L  Ð#4Ð5Ô6Ùà#ÖI˜ r¨*¡~¼Ó'H’ÐIˆÐIØ×Ñ˜ÔÜ�˜Q˜C˜r¤# e£* ¨_¼SÀ»^Ð<LÈAÐNÔOÙÜ�J˜u R™y¨Ñ4Ð5Ð6Ô7Ü�
‰
�3�ðô 
Ð*¬3¨y«>Ð*:Ð
;Ô<áÜÐ!Ô"Øô 
ˆNœ3˜y›>Ð*Ð*;Ð
<Ô=Ø€GØ€Fä	¨Ô	*ð 	?¨hØCLÖM¸R�8—?‘?¤<°Ó4°bÑ8ÐMˆÐMÜ¤¨WÓ 5Ó6ò 	?‰FˆAˆsØ—*‘*“,ˆCÙØ—‘˜sÕ#à˜!‘�Ø�A‘˜‰|˜qÓ Ü˜	 ! A¡#  a¬¨I«Ð'7°uÐ=Õ>ñ	?÷	?ô 
ˆJ”s˜7“|�n J¨v¨h°gÐ
>Ô?ô 
Ð
(Ô)Ü˜'Ó"€HÜ	ˆI�h�ZÐ4´S¸³\°NÀ!Ð
DÔEÜ	Ð
ÕùòG Jùò& N÷	?ð 	?ús1   Á$H#Á6H#Ä.H-Ä3H(ÅAH-Æ" H-È(H-È-H6Ú__main__)"Ú__doc__ÚsysÚosr4   ro   Újsonr   r   Úconcurrent.futuresr   r   Úurllib.parser   ÚpathÚinsertÚdirnameÚabspathÚ__file__r*   Úbs4r   r   r   rO   r)   r,   ÚnowÚstrftimerk   r>   rY   rb   r|   Ú__name__© r   r   ú<module>rŽ      sË   ðñó Û 	Û 	Û Û ß (ß ?Ý  à ‡�‡���2—7‘7—?‘? 2§7¡7§?¡?°8Ó#<Ó=Ô >Û Ý à
€òð %€	Ø3€àð  Dð€ð  �8—<‘<“>¡I°5Ô$9Ñ9×CÑCÀJÓO€ò'òT4ònò.0ðf ˆzÒÙ…Fð r   