Ë
    V_Šj6   ã                   ó    — d Z ddlZddlZddlZddlZddlZddlZddlmZ dZ	dZ
dZddd	d
œZdZdd„Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uP   
æ™®å®šåŽ¿äººæ°‘æ”¿åºœ - é€šçŸ¥å…¬å‘Š çˆ¬è™«
https://www.aspd.gov.cn/xwzx/tzgg/
é    N)ÚBeautifulSoupu   æ™®å®šåŽ¿-é€šçŸ¥å…¬å‘Šu   è´µå·žz"https://www.aspd.gov.cn/xwzx/tzgg/zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagez/root/search.dbc           
      óf  — t        |«      D ]y  }	 t        j                  | t        d¬«      }d|_        |j
                  dk(  r|j                  c S t        d|j
                  › d| › d|dz   › �«       t        j                  d
«       Œ{ y # t        $ r}t        d	|› d|dz   › �«       Y d }~Œ:d }~ww xY w)Né   )ÚheadersÚtimeoutzutf-8éÈ   z  HTTP z for z, retry é   z	  Error: é   )ÚrangeÚrequestsÚgetÚHEADERSÚencodingÚstatus_codeÚtextÚprintÚ	ExceptionÚtimeÚsleep)ÚurlÚmax_retriesÚattemptÚrÚes        ú/root/gov_crawler/crawl_aspd.pyÚfetchr      s¬   € Ü˜Ó%ò 	ˆð	6Ü—‘˜S¬'¸2Ô>ˆAØ ˆAŒJØ�}‰} Ò#Ø—v‘v’Ü�G˜AŸM™M˜?¨%°¨u°H¸WÀQ¹Y¸KÐHÔIô 	�
‰
�1�ð	ð øô ò 	6Ü�I˜a˜S ¨°©¨Ð4×5Ñ5ûð	6ús   �=B	Á!B	Â		B0ÂB+Â+B0c                 óh  — g }t        | d«      }|j                  d¬«      }|st        d«       |S |j                  dd¬«      D ]l  }|j	                  d¬«      }|d	   }|sŒ|sŒ d
|v sŒ%|j                  d«      s$t        j                  j                  t        |«      }|j                  ||f«       Œn |S )Núhtml.parserÚNewsList©Úclass_z  WARNING: NewsList not foundÚaT©Úhref©Ústripr%   z/tzgg/Úhttp)r   Úfindr   Úfind_allÚget_textÚ
startswithr   ÚcompatÚurljoinÚBASE_URLÚappend)ÚhtmlÚitemsÚsoupÚ	news_listr#   Útitler%   s          r   Úparse_list_pager6   %   s±   € Ø€EÜ˜˜}Ó-€DØ—	‘	 �	Ó,€IÙÜÐ-Ô.ØˆØ×Ñ ¨$ÐÓ/ò (ˆØ—
‘
 �
Ó&ˆØ�‰yˆÚ’T˜h¨$Ò.Ø—?‘? 6Ô*Ü—‘×.Ñ.¬x¸Ó>�Ø�L‰L˜% ˜Õ'ð(ð €Ló    c                 ó6  — t        | d«      }|j                  d¬«      }|r|j                  d¬«      nd}|s%|j                  d«      }|r|j                  d¬«      }t        j                  d| «      }|r|j                  d	«      nd}|j                  d
„ ¬«      }|s|j                  d¬«      }d}	g }
|�rx|j                  dd¬«      D ]š  }|d   }t        j                  d|t        j                  «      sŒ.|j                  d«      s t        j                  j                  ||«      }|j                  d¬«      xs |j                  d«      d   }|
j                  ||dœ«       Œœ |j                  ddg«      D ]  }|j                  «        Œ g }|j                  D �]}  }|j                  d¬«      }|sŒ|j                   dk(  r|j                  |«       Œ:|j                   dv rd|j                  dd¬«      D ](  }|j                  d¬«      }|sŒ|j                  |«       Œ* |j                  dd¬«      rŒš|j                  |«       Œ¬|j                   dk(  r“|j                  d«      }g }|D ]U  }|j                  ddg«      D �cg c]  }|j                  d¬«      ‘Œ }}|sŒ6|j                  dj#                  |«      «       ŒW |s�Œ,|j                  dj#                  |«      «       �ŒN|j                   €|j                  |«       �Œm|j                  |«       �Œ€ d j#                  |«      }	t%        |	j'                  «       «      d!k  r)d"|› d#|› d$�}	|
r|	d%dj#                  d&„ |
D «       «      z   z  }	|||	|
rt)        j*                  |
d¬'«      |d(œS d|d(œS c c}w ))Nr   ÚArticleTitler!   Tr&   Ú r5   z.var pubdata='(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2})r
   c                 ó   — | xr d| v S )NÚtrs_editor_view© )Úcs    r   ú<lambda>zparse_detail.<locals>.<lambda>B   s   € ¨AÒ,HÐ2CÀqÐ2H€ r7   Únryr#   r$   r%   z\.(doc|docx|pdf|xls|xlsx|xlsm)$r(   ú/éÿÿÿÿ)r5   r   ÚscriptÚstyleÚp)ÚdivÚsectionF)Ú	recursiveÚtableÚtrÚtdÚthz | ú
z

é   z<p><a href="z">z</a></p>u   

é™„ä»¶ï¼š
c              3   ó:   K  — | ]  }d |d   › d|d   › d�–— Œ y­w)ú[r5   z](r   ú)Nr=   ©Ú.0r#   s     r   ú	<genexpr>zparse_detail.<locals>.<genexpr>x   s)   è ø€ Ò4hÐWX°q¸¸7¹¸ÀBÀqÈÁxÀjÐPQÔ5RÑ4hùs   ‚)Úensure_ascii)r5   Úpublish_dateÚcontentÚattachmentsÚpage_url)r   r)   r+   ÚreÚsearchÚgroupr*   ÚIr,   r   r-   r.   Úsplitr0   Ú	decomposeÚchildrenÚnameÚjoinÚlenr'   ÚjsonÚdumps)r1   r   r3   Útitle_elr5   ÚtÚ
date_matchrV   Ú
content_elrW   rX   r#   r%   Úattach_titleÚtagÚpartsÚchildÚ
child_textÚinner_pÚ
inner_textÚrowsÚtable_linesÚrowÚcellÚcellss                            r   Úparse_detailrv   5   sq  € Ü˜˜}Ó-€Dà�y‰y ˆyÓ/€HÙ-5ˆH×Ñ DÐÔ)¸2€EÙØ�I‰I�gÓˆÙØ—J‘J T�JÓ*ˆEä—‘ÐLÈdÓS€JÙ*4�:×#Ñ# AÔ&¸"€Là—‘Ñ"H�ÓI€JÙØ—Y‘Y e�YÓ,ˆ
à€GØ€KÚØ×$Ñ$ S¨tÐ$Ó4ò 	IˆAØ�V‘9ˆDÜ�y‰yÐ;¸TÄ2Ç4Á4ÕHØ—‘ vÔ.Ü#Ÿ?™?×2Ñ2°3¸Ó=�DØ Ÿz™z°˜zÓ5ÒL¸¿¹ÀC»ÈÑ9L�Ø×"Ñ"¨\À$Ñ#GÕHð	Ið ×&Ñ&¨°'Ð':Ó;ò 	ˆCØ�M‰M�Oð	ð ˆØ×(Ñ(ó 	-ˆEØŸ™¨d˜Ó3ˆJÚØ—:‘: Ò$Ø—L‘L Õ,Ø—Z‘ZÐ#5Ñ5à#(§>¡>°#À >Ó#Gò 5˜Ø%,×%5Ñ%5¸DÐ%5Ó%A˜
Ú%Ø!ŸL™L¨Õ4ð5ð !Ÿ>™>¨#¸˜>Õ?ØŸ™ ZÕ0Ø—Z‘Z 7Ò*à Ÿ>™>¨$Ó/�DØ"$�KØ#ò B˜ØGJÇ|Á|ÐUYÐ[_ÐT`ÓGaÖ b¸t §¡°T Õ!:Ð b˜Ð bÚ Ø'×.Ñ.¨u¯z©z¸%Ó/@ÕAðBó #ØŸ™ T§Y¡Y¨{Ó%;Ö<Ø—Z‘ZÐ'à—L‘L Ö,à—L‘L Ö,ð7	-ð8 —+‘+˜eÓ$ˆô ˆ7�=‰=‹?Ó˜bÒ Ø    R¨ w¨hÐ7ˆÙØÐ(¨4¯9©9Ñ4hÐ\gÔ4hÓ+hÑhÑhˆGð Ø$ØÙFQ”t—z‘z +¸EÔBØñð ð XZØñð ùò% !cs   ÊNc                 ó’  — g }t        | |dz   «      D �]1  }|dk(  rt        }nt        › d|dz
  › d�}t        d|› d|› d|› �«       t        |«      }|st        d|› �«       ŒOt	        |«      }|st        d|› d	�«        |S t        d
t        |«      › d�«       t        |«      D ]‰  \  }\  }}	t        d|dz   › dt        |«      › d|d d › d�«       t        |	«      }
|
st        d«       ŒFt        |
|	«      }t        |d<   t        |d<   |j                  |«       t        j                  d«       Œ‹ t        j                  d«       �Œ4 |S )Nr
   Úindex_z.htmlz  Page rA   z: z  FAILED to fetch page z  No items on page z
, stoppingz  Found z itemsz    [ú] é2   z...z      FAILED detailÚ	site_namer\   )r   r/   r   r   r6   rc   Ú	enumeraterv   Ú	SITE_NAMEÚGROUPr0   r   r   )Ú
start_pageÚend_pageÚ	all_itemsÚpager   r1   r2   Úir5   Úitem_urlÚdetail_htmlÚdetails               r   Úcrawl_pagesr‡   ‚   sn  € Ø€IÜ�j (¨Q¡,Ó/ó !ˆØ�1Š9Ü‰Cä�J˜f T¨!¡V H¨EÐ2ˆCä�˜�v˜Q˜x˜j¨¨3¨%Ð0Ô1Ü�S‹zˆÙÜÐ+¨D¨6Ð2Ô3Øä Ó%ˆÙÜÐ'¨ v¨ZÐ8Ô9Øð( Ðô% 	�œ˜U›˜ FÐ+Ô,ä$-¨eÓ$4ò 	Ñ ˆAÑ ��xÜ�E˜!˜A™#˜˜a¤ E£
˜|¨2¨e°C°R¨j¨\¸Ð=Ô>Ü ›/ˆKÙÜÐ+Ô-Øä! +¨xÓ8ˆFÜ"+ˆF�;ÑÜ#ˆF�7‰Oà×Ñ˜VÔ$Ü�J‰J�q�Mð	ô 	�
‰
�1ŽðC!ðF Ðr7   c                 ó  — dd l }|j                  t        «      }|j                  d«       |j	                  «       }d}d}| D ]€  }	 |j                  d|d   |d   |d   |j                  dd«      d|j                  d	d«      |j                  d
d«      |j                  dd«      f«       |j                  dkD  r|dz  }n|dz  }Œ‚ |j                  «        |j                  «        ||fS # t        $ r}t        d|› �«       Y d }~ŒÃd }~ww xY w)Nr   zPRAGMA busy_timeout=5000zÐ
                INSERT OR IGNORE INTO gov_raw 
                (site_name, page_url, title, publish_date, summary, content, category, attachments)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
            r{   rY   r5   rV   r:   rW   r\   rX   r
   z    DB Error: )Úsqlite3ÚconnectÚDB_PATHÚexecuteÚcursorr   Úrowcountr   r   ÚcommitÚclose)r2   r‰   Úconnr>   ÚinsertedÚskippedÚitemr   s           r   Ú
save_to_dbr•   ©   s  € ÛØ�?‰?œ7Ó#€DØ‡L�LÐ+Ô,Ø�‰‹€Aà€HØ€GØò (ˆð	(Ø�I‰Ið ð
 �[Ñ!Ø�ZÑ Ø�W‘Ø—‘˜¨Ó,ØØ—‘˜ BÓ'Ø—‘˜ "Ó%Ø—‘˜¨Ó+ð	ô	ð �z‰z˜AŠ~Ø˜A‘‘à˜1‘�øð'(ð. 	‡K�K„MØ‡J�J„LØ�WÐÐøô ò 	(Ü�N 1 #Ð&×'Ñ'ûð	(ús   ÁA=C(Ã(	D	Ã1DÄD	c            
      ó  — t        j                  dt        › �¬«      } | j                  ddd¬«       | j                  ddd¬«       | j                  d	t        d
d¬«       | j                  «       }|j                  s|j                  rd}|j                  rdnd}n|j                  }d|z  }t        d|› dt        › d�«       t        d|«      }|st        d«       y t        dt        |«      › d�«       t        |«      \  }}t        d|› d|› �«       t        d«       |d d
 D ]d  }t        d|d   › d|d   d d › �«       |d   sŒ#t        j                  |d   «      }|sŒ>t        d d!j                  d"„ |d d# D «       «      › �«       Œf y )$Nu   çˆ¬å�–)Údescriptionz--testÚ
store_trueu   æµ‹è¯•æ¨¡å¼�ï¼šçˆ¬ç¬¬1é¡µ)ÚactionÚhelpz--incrementalu   å¢žé‡�æ¨¡å¼�ï¼šå�ªçˆ¬ç¬¬1é¡µz--max-pagesé   u   æœ€å¤§é¡µæ•°)ÚtypeÚdefaultrš   r
   u   æµ‹è¯•u   å¢žé‡�u   å…¨é‡�(å‰�%dé¡µ)z=== u   æ¨¡å¼�: z ===zNo items collectedu   
å…±èŽ·å�– u
    æ�¡æ•°æ�®u   å…¥åº“: æ–°å¢ž u	   , è·³è¿‡ u   
=== å‰�5æ�¡é¢„è§ˆ ===z  [rV   ry   r5   é<   rX   u       é™„ä»¶: z, c              3   ó&   K  — | ]	  }|d    –— Œ y­w)r5   Nr=   rR   s     r   rT   zmain.<locals>.<genexpr>ë   s   è ø€ Ò.L¸a¨q°­zÑ.Lùs   ‚é   )ÚargparseÚArgumentParserr}   Úadd_argumentÚintÚ
parse_argsÚtestÚincrementalÚ	max_pagesr   r‡   rc   r•   rd   Úloadsrb   )	ÚparserÚargsÚendÚmoder2   r’   r“   r”   Úattss	            r   Úmainr¯   Ì   s“  € Ü×$Ñ$°6¼)¸Ð1EÔF€FØ
×Ñ˜¨Ð<WÐÔXØ
×Ñ˜°ÐCaÐÔbØ
×Ñ˜¬C¸ÀÐÔPØ×ÑÓ€Dà‡y‚y�D×$Ò$ØˆØŸ9š9‰x¨(‰à�n‰nˆØ! CÑ'ˆä	ˆD���hœy˜k¨Ð
.Ô/Ü˜˜3Ó€EáÜÐ"Ô#Øä	ˆLœ˜U›˜ JÐ
/Ô0Ü" 5Ó)Ñ€HˆgÜ	ˆO˜H˜: Y¨w¨iÐ
8Ô9ä	Ð#Ô%Ø�b�q�	ò PˆÜ��D˜Ñ(Ð)¨¨D°©M¸#¸2Ð,>Ð+?Ð@ÔAØ�ÓÜ—:‘:˜d =Ñ1Ó2ˆDÚÜ˜ T§Y¡YÑ.LÀ4ÈÈÀ8Ô.LÓ%LÐ$MÐNÕOñPr7   Ú__main__)r    )Ú__doc__rZ   Úsysrd   r   r¡   r   Úbs4r   r}   r~   r/   r   r‹   r   r6   rv   r‡   r•   r¯   Ú__name__r=   r7   r   ú<module>rµ      s‚   ðñó 
Û 
Û Û Û Û Ý à$€	Ø€Ø/€ð DØOØ0ñ€ð
 €óòò KòZ%òN!òFPðB ˆzÒÙ…Fð r7   