Ë
    ê}j–  ã                   ó¾   — d dl Zd dlZd dlZd dlZd dlZd dlZd dlZ ej                  dd«      Z	dZ
dZdZdZdZd	Zdd
dddœZdd„Zd„ Zdd„Zd„ Zedk(  r e«        yy)é    NÚ	SEARCH_DBz/root/search.dbu   ä¸°åŸŽå¸‚ä¹¡é•‡å›žåº”zwww.jxfc.gov.cni»  z
/queryListz
2023-06-01Úxzhyz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36zapplication/json)ÚHostz
User-AgentzContent-TypeÚAcceptc                 óR  — t         j                  j                  t        t        d¬«      }|j                  dt        t        j                  t        g|dg| dœ«      t        ¬«       t        j                  |j                  «       j                  «       j                  d«      «      d   S )	Né   )ÚtimeoutÚPOSTÚfcsrmzf)ÚchannelCodeÚpageSizeÚwebSiteCodeÚcurrent)ÚbodyÚheaderszutf-8Údata)ÚhttpÚclientÚHTTPSConnectionÚAPI_HOSTÚAPI_PORTÚrequestÚAPI_PATHÚjsonÚdumpsÚCHANNELÚHEADERSÚloadsÚgetresponseÚreadÚdecode)ÚpnÚpsÚconns      ú'/root/gov_crawler/crawl_fcsrmzf_xzhy.pyÚ
fetch_pager&      sŽ   € Ü�;‰;×&Ñ&¤x´À2Ð&ÓF€DØ‡L�L�œ¬¯
©
Ä7À)ÐWYÐirÐhsð  Añ  4Bó  )Cô  MT€Lô  UÜ�:‰:�d×&Ñ&Ó(×-Ñ-Ó/×6Ñ6°wÓ?Ó@ÀÑHÐHó    c                 óX  — | s| S t        j                  dd| t         j                  t         j                  z  ¬«      } t        j                  dd| t         j                  t         j                  z  ¬«      } t        j                  dd| t         j                  ¬«      } | j	                  «       S )Nz<script[^>]*>.*?</script>Ú )Úflagsz<style[^>]*>.*?</style>zwidth\s*:\s*\d+pxzmax-width:100%)ÚreÚsubÚDOTALLÚIÚstrip)Úhtmls    r%   Úclean_contentr1      sq   € Ù˜�Ü�6‰6Ð.°°DÄÇ	Á	Ì"Ï$É$ÁÔO€DÜ�6‰6Ð,¨b°$¼b¿i¹iÌÏÉ¹nÔM€DÜ�6‰6Ð&Ð(8¸$ÄbÇdÁdÔK€DØ�:‰:‹<Ðr'   c                 ó„  — 	 t        | t        «      rt        j                  | «      }n| }|rt        |t        «      sydg}|D ]Ì  }t        |t
        «      sŒ|j                  dd«      xs |j                  dd«      xs d}|j                  dd«      xs d}|j                  dd«      xs |}|rL|j                  d«      r||z   }n|j                  d«      r|}n|dz   |z   }|j                  d	|› d
|› d�«       Œ¸|j                  d|› d�«       ŒÎ |j                  d«       dj                  |«      S # t        $ r Y yw xY w)Nr)   u4   <div class="article-attachments"><h3>é™„ä»¶</h3><ul>ÚfileNameÚnameÚfilePathÚ
domainNameú/r   z<li><a href="z" target="_blank" download>z	</a></li>z<li>z</li>z</ul></div>ú
)Ú
isinstanceÚstrr   r   ÚlistÚdictÚgetÚ
startswithÚappendÚjoinÚ	Exception)	Úarticle_filesÚdomainÚfilesÚpartsÚfÚfnameÚfpathÚdomain_nameÚurls	            r%   Úbuild_attachment_htmlrK   !   sC  € ðÜ�m¤SÔ)Ü—J‘J˜}Ó-‰Eà!ˆEÙœJ u¬dÔ3ØØMÐNˆØò 	2ˆAÜ˜a¤Ô&ØØ—E‘E˜* bÓ)ÒD¨Q¯U©U°6¸2Ó->ÒDÀ"ˆEØ—E‘E˜* bÓ)Ò/¨RˆEØŸ%™% ¨bÓ1Ò;°VˆKÙØ×#Ñ# CÔ(Ø%¨Ñ-‘CØ×%Ñ% fÔ-Ø‘Cà  3™,¨Ñ.�CØ—‘˜}¨S¨EÐ1LÈUÈGÐS\Ð]Õ^à—‘˜t E 7¨%Ð0Õ1ð	2ð  	�‰�]Ô#Ø�y‰y˜ÓÐøÜò Ùðús   ‚:D3 ½C5D3 Ä3	D?Ä>D?c                  óH  — t        dt        › d�«       t        dd«      } | d   }|dz   dz  }t        d|› d|› �«       t        j                  t
        «      }|j                  «       }d	\  }}}d
}t        j                  «       }	t        d|dz   «      D �]O  }
	 t        |
d«      } | j                  dg «      }|s �n+d
}|D �]Ñ  }|j                  di «      }|j                  dd«      xs dj                  «       }|j                  dd«      xs dd d }|r|t        k  r|dz  }|dz  }Œi|dz  }	 t        j                  |j                  dd«      «      }|j                  dd«      xs d}|j                  di «      xs i }d}t!        |t"        «      r|j                  dd«      xs d}nt!        |t$        «      r|}t'        |«      }|s-|j                  dd«      }|r|dk7  rt)        |«      }|r|}|dz  }|j                  d«      xs |j                  dd«      }|r|s|dz  }�Œe	 |j+                  dt-        |«      t        |rd|› �nd|rd|› �nd||||rt-        |j/                  dd«      «      nd
f«       |j0                  d
kD  r|dz  }n|dz  }�ŒÔ t        j                  «       |	z
  }t        d|
› d |› d!|› d"|› d#|› d$|d%›d&�«       |d
k  s�Œ>|
dkD  s�ŒEt        d'«        n |j3                  «        t        d(|› �«       t        j                  «       |	z
  }t        d)|d%›d*|› d+|› d,|› �«       y # t        $ r1}t        d|
› d|› �«       t        j                  d«       Y d }~�ŒÖd }~ww xY w#  d}Y �ŒxY w# t        $ r}t        d|› �«       |dz  }Y d }~�ŒÞd }~ww xY w)-Nz=== z ===é   Útotalé   é   zTotal: z, Max pages: )r   r   r   r   z  [ERROR] Page z: é   ÚresultsÚsourceÚtitler)   ÚpubDateé
   Úurlsz{}ÚpcÚcontentÚarticleFilesz[{}]Úidz|INSERT OR IGNORE INTO gov_raw(id,site_name,source_url,page_url,title,publish_date,content,date_rank) VALUES(?,?,?,?,?,?,?,?)úhttp://www.jxfc.gov.cnú-z  [DB ERR] z  Page r7   z: +z (-z, >3yz) [z.0fzs]z  (Reached cutoff, stopping)z
Attachment content filled: z
=== Done (zs) === Inserted z
, Skipped z, Old )ÚprintÚ	SITE_NAMEr&   Úsqlite3ÚconnectÚDB_PATHÚcursorÚtimeÚrangerA   Úsleepr=   r/   ÚCUTOFF_DATEr   r   r9   r<   r:   r1   rK   ÚexecuteÚintÚreplaceÚrowcountÚcommit)ÚdrN   Útotal_pagesr$   ÚcÚinsertedÚskippedÚoldÚcontent_filledÚ
start_timeÚpgÚerR   Úpage_okÚartÚsrcrT   Úpub_daterW   Úpage_urlÚcoÚrawÚcontent_htmlÚafÚattachment_htmlÚaidÚelapseds                              r%   Úmainrƒ   ?   sà  € Ü	ˆD”�˜4Ð
 Ô!ô 	�1�aÓ€AØˆg‰J€EØ˜2‘: "Ñ$€KÜ	ˆG�E�7˜-¨ }Ð
5Ô6ä�?‰?œ7Ó#€DØ�‰‹€AØ$Ñ€Hˆg�sØ€NÜ—‘“€Jô �A�{ Q‘Ó'ó =ˆð	Ü˜2˜rÓ"ˆAð
 —%‘%˜	 2Ó&ˆÙÚàˆØó )	ˆCØ—'‘'˜( BÓ'ˆCØ—W‘W˜W bÓ)Ò/¨R×6Ñ6Ó8ˆEØŸ™ 	¨2Ó.Ò4°"°c°rÐ:ˆHÙ˜H¤{Ò2Ø�q‘�Ø˜1‘�ØØ�q‰LˆGðÜ—z‘z #§'¡'¨&°$Ó"7Ó8�ØŸ8™8 D¨"Ó-Ò3°�ð —‘˜ BÓ'Ò-¨2ˆBØˆCÜ˜"œdÔ#¨2¯6©6°)¸RÓ+@Ò+FÀB¡SÜ˜B¤Ô$¨B cÜ(¨Ó-ˆLÙØ—W‘W˜^¨RÓ0�Ù˜" š,Ü&;¸BÓ&?�OÙ&Ø'6˜Ø&¨!Ñ+˜Ø—'‘'˜$“-Ò4 3§7¡7¨4°Ó#4ˆCÙ¡Ø˜1‘�ÙðØ—	‘	ð  YÜ˜s›8¤YÙBJÐ3°H°:Ñ>ÐPRÙBJÐ3°H°:Ñ>ÐPRØ  (¨LÙ<Dœ3˜x×/Ñ/°°BÓ7Ô8È!ð	MôNð —:‘: ’> 8¨q¡=¡8Ø ‘l�gùðM)	ôV —)‘)“+ 
Ñ*ˆÜ�˜�t˜1˜[˜M¨¨X¨J°c¸'¸À%ÈÀuÈCÐPWÐX[È}Ð\^Ð_Ô`ð �aŒ<˜B œFÜÐ0Ô2Ùð{=ð~ 	‡K�K„MÜ	Ð)¨.Ð)9Ð
:Ô;Ü�i‰i‹k˜JÑ&€GÜ	ˆL˜ ˜Ð%5°h°Z¸zÈ'ÈÐRXÐY\ÐX]Ð
^Õ_øô ò 	Ü�O B 4 r¨!¨Ð-Ô.Ü�J‰J�qŒMÝûð	ûð*Ø“ûô4 ò Ü˜ A 3Ð'Ô(Ø˜1‘–ûðús=   ÂL3Ä-;M0È$A,M:Ì3	M-Ì<&M(Í(M-Í0M7Í:	N!ÎNÎN!Ú__main__)rP   )r\   )Úhttp.clientr   r   r`   r+   Úsysrd   ÚosÚgetenvrb   r_   r   r   r   rg   r   r   r&   r1   rK   rƒ   Ú__name__© r'   r%   ú<module>r‹      s…   ð÷ 1× 0× 0Û 	à
ˆ"�)‰)�KÐ!2Ó
3€Ø8€	Ø€Ø€Ø€Ø€Ø
€ð ØPØ&Ø ñ	€óIò
óò<R`ðh ˆzÒÙ…Fð r'   