Ë
    ê}jÒ  ã                   ó¶   — d dl Zd dlZd dlZd dlZd dlZd dlZ ej                  dd«      ZdZ	dZ
dZdZdZd	Zdd
dddœZdd„Zd„ Zdd„Zd„ Zedk(  r e«        yy)é    NÚ	SEARCH_DBz/root/search.dbu   ä¸°åŸŽå¸‚é‡�å¤§å»ºè®¾é¡¹ç›®zwww.jxfc.gov.cni»  z
/queryListz
2023-06-01Úzdjsxmz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36zapplication/json)ÚHostz
User-AgentzContent-TypeÚAcceptc                 óR  — t         j                  j                  t        t        d¬«      }|j                  dt        t        j                  t        g|dg| dœ«      t        ¬«       t        j                  |j                  «       j                  «       j                  d«      «      d   S )	Né   )ÚtimeoutÚPOSTÚfcsrmzf)ÚchannelCodeÚpageSizeÚwebSiteCodeÚcurrent)ÚbodyÚheaderszutf-8Údata)ÚhttpÚclientÚHTTPSConnectionÚAPI_HOSTÚAPI_PORTÚrequestÚAPI_PATHÚjsonÚdumpsÚCHANNELÚHEADERSÚloadsÚgetresponseÚreadÚdecode)ÚpnÚpsÚconns      ú)/root/gov_crawler/crawl_fcsrmzf_zdjsxm.pyÚ
fetch_pager&      sŽ   € Ü�;‰;×&Ñ&¤x´À2Ð&ÓF€DØ‡L�L�œ¬¯
©
Ä7À)ÐWYÐirÐhsð  Añ  4Bó  )Cô  MT€Lô  UÜ�:‰:�d×&Ñ&Ó(×-Ñ-Ó/×6Ñ6°wÓ?Ó@ÀÑHÐHó    c                 óX  — | s| S t        j                  dd| t         j                  t         j                  z  ¬«      } t        j                  dd| t         j                  t         j                  z  ¬«      } t        j                  dd| t         j                  ¬«      } | j	                  «       S )Nz<script[^>]*>.*?</script>Ú )Úflagsz<style[^>]*>.*?</style>zwidth\s*:\s*\d+pxzmax-width:100%)ÚreÚsubÚDOTALLÚIÚstrip)Úhtmls    r%   Úclean_contentr1      sq   € Ù˜�Ü�6‰6Ð.°°DÄÇ	Á	Ì"Ï$É$ÁÔO€DÜ�6‰6Ð,¨b°$¼b¿i¹iÌÏÉ¹nÔM€DÜ�6‰6Ð&Ð(8¸$ÄbÇdÁdÔK€DØ�:‰:‹<Ðr'   c                 ó„  — 	 t        | t        «      rt        j                  | «      }n| }|rt        |t        «      sydg}|D ]Ì  }t        |t
        «      sŒ|j                  dd«      xs |j                  dd«      xs d}|j                  dd«      xs d}|j                  dd«      xs |}|rL|j                  d«      r||z   }n|j                  d«      r|}n|dz   |z   }|j                  d	|› d
|› d�«       Œ¸|j                  d|› d�«       ŒÎ |j                  d«       dj                  |«      S # t        $ r Y yw xY w)zAGenerate HTML snippet for file attachments when content is empty.r)   u4   <div class="article-attachments"><h3>é™„ä»¶</h3><ul>ÚfileNameÚnameÚfilePathÚ
domainNameú/r   z<li><a href="z" target="_blank" download>z	</a></li>z<li>z</li>z</ul></div>ú
)Ú
isinstanceÚstrr   r   ÚlistÚdictÚgetÚ
startswithÚappendÚjoinÚ	Exception)	Úarticle_filesÚdomainÚfilesÚpartsÚfÚfnameÚfpathÚdomain_nameÚurls	            r%   Úbuild_attachment_htmlrK   !   sC  € ðÜ�m¤SÔ)Ü—J‘J˜}Ó-‰Eà!ˆEÙœJ u¬dÔ3ØØMÐNˆØò 	2ˆAÜ˜a¤Ô&ØØ—E‘E˜* bÓ)ÒD¨Q¯U©U°6¸2Ó->ÒDÀ"ˆEØ—E‘E˜* bÓ)Ò/¨RˆEØŸ%™% ¨bÓ1Ò;°VˆKÙØ×#Ñ# CÔ(Ø%¨Ñ-‘CØ×%Ñ% fÔ-Ø‘Cà  3™,¨Ñ.�CØ—‘˜}¨S¨EÐ1LÈUÈGÐS\Ð]Õ^à—‘˜t E 7¨%Ð0Õ1ð	2ð  	�‰�]Ô#Ø�y‰y˜ÓÐøÜò Ùðús   ‚:D3 ½C5D3 Ä3	D?Ä>D?c                  óT  — t        dt        › d�«       t        dd«      } | d   }|dz   dz  }t        d|› d|› �«       t        j                  t
        «      }|j                  «       }d	\  }}}d
}d
}	t        d|dz   «      D ]c  }
t        |
d«      } | j                  dg «      }|s nA|D �cg c]  }|d   j                  dd«      d d ‘Œ }}|rt        |«      t        k  r|
}	 n|
}	Œe t        d|	› d�«       t        d|	dz   «      D �]  }
	 t        |
d«      } | j                  dg «      }|s �nê|D �]Ç  }|j                  di «      }|j                  dd«      xs dj                  «       }|j                  dd«      xs dd d }|r|t        k  r|dz  }Œd	 t        j                  |j                  dd«      «      }|j                  dd«      xs d}|j                  di «      xs i }d}t        |t         «      r|j                  dd«      xs d}nt        |t"        «      r|}t%        |«      }|s-|j                  dd«      }|r|dk7  rt'        |«      }|r|}|dz  }|j                  d«      xs |j                  dd«      }|r|s|dz  }�Œ[	 |j)                  dt+        |«      t        |rd|› �nd|rd|› �nd||||rt+        |j-                  dd«      «      nd
f«       |j.                  d
kD  r|dz  }n|dz  }�ŒÊ t        d |
› d!|	› d"|› d#|› d$|› �
«       �Œ |j1                  «        t        d%|› �«       t        d&|› d'|› d(|› �«       y c c}w # t        $ r}t        d|
› d|› �«       Y d }~�Œjd }~ww xY w#  d}Y �Œ¯xY w# t        $ r}t        d|› �«       |dz  }Y d }~�Œtd }~ww xY w))Nz=== z (v2 - with attachments) ===é   Útotalé   é   zTotal: z	, Pages: )r   r   r   r   ÚresultsÚsourceÚpubDater)   é
   z
Fetching: z pagesz  [ERROR] Page z: ÚtitleÚurlsz{}ÚpcÚcontentÚarticleFilesz[{}]Úidz|INSERT OR IGNORE INTO gov_raw(id,site_name,source_url,page_url,title,publish_date,content,date_rank) VALUES(?,?,?,?,?,?,?,?)úhttp://www.jxfc.gov.cnú-z  [DB ERR] z  Page r7   z: +z, -z, outz
Attachment content filled: z
=== Done === Inserted z
, Skipped z, Old )ÚprintÚ	SITE_NAMEr&   Úsqlite3ÚconnectÚDB_PATHÚcursorÚranger=   ÚminÚCUTOFF_DATErA   r/   r   r   r9   r<   r:   r1   rK   ÚexecuteÚintÚreplaceÚrowcountÚcommit)ÚdrN   Útotal_pagesr$   ÚcÚinsertedÚskippedÚoldÚcontent_filledÚmax_pgÚpgrQ   ÚrÚdatesÚeÚartÚsrcrU   Úpub_daterV   Úpage_urlÚcoÚrawÚcontent_htmlÚafÚattachment_htmlÚaids                              r%   Úmainr�   @   sÿ  € Ü	ˆD”�Ð7Ð
8Ô9Ü�1�aÓ€AØˆg‰J€EØ˜2‘: "Ñ$€KÜ	ˆG�E�7˜) K =Ð
1Ô2ä�?‰?œ7Ó#€DØ�‰‹€AØ$Ñ€Hˆg�sØ€NØ€FÜ�A�{ Q‘Ó'ò 	ˆÜ�r˜2ÓˆØ—%‘%˜	 2Ó&ˆÙÙØ>EÖF¸��8‘—‘ ¨BÓ/°°Ò4ÐFˆÐFÙ”S˜“Z¤+Ò-ØˆFÙØ‰ð	ô 
ˆJ�v�h˜fÐ
%Ô&ä�A�v ‘zÓ"ó 4Jˆð	Ü˜2˜rÓ"ˆAð —%‘%˜	 2Ó&ˆÙÚØó *	ˆCØ—'‘'˜( BÓ'ˆCØ—W‘W˜W bÓ)Ò/¨R×6Ñ6Ó8ˆEØŸ™ 	¨2Ó.Ò4°"°c°rÐ:ˆHÙ˜H¤{Ò2Ø�q‘�ØðÜ—z‘z #§'¡'¨&°$Ó"7Ó8�ØŸ8™8 D¨"Ó-Ò3°�ð —‘˜ BÓ'Ò-¨2ˆBØˆCÜ˜"œdÔ#¨2¯6©6°)¸RÓ+@Ò+FÀB¡SÜ˜B¤Ô$¨B cÜ(¨Ó-ˆLñ  Ø—W‘W˜^¨RÓ0�Ù˜" š,Ü&;¸BÓ&?�OÙ&Ø'6˜Ø&¨!Ñ+˜à—'‘'˜$“-Ò4 3§7¡7¨4°Ó#4ˆCÙ¡Ø˜1‘�ÙðØ—	‘	ð  YÜ˜s›8¤YÙBJÐ3°H°:Ñ>ÐPRÙBJÐ3°H°:Ñ>ÐPRØ  (¨LÙ<Dœ3˜x×/Ñ/°°BÓ7Ô8È!ð	MôNð —:‘: ’> 8¨q¡=¡8Ø ‘l�gùðO*	ôV 	�˜�t˜1˜V˜H C¨ z°°W°I¸UÀ3À%ÐHÖIði4Jðl 	‡K�K„MÜ	Ð)¨.Ð)9Ð
:Ô;Ü	Ð$ X J¨j¸¸	ÀÈÀuÐ
MÕNùò Gøô ò 	Ü�O B 4 r¨!¨Ð-Ô.Ýûð	ûð Ø“ûô: ò Ü˜ A 3Ð'Ô(Ø˜1‘–ûðúsC   Â)M	Ä	MÆ;M6Ê
A,N Í	M3ÍM.Í.M3Í6M=Î 	N'Î	N"Î"N'Ú__main__)rP   )r[   )Úhttp.clientr   r   r_   r+   ÚsysÚosÚgetenvra   r^   r   r   r   re   r   r   r&   r1   rK   r�   Ú__name__© r'   r%   ú<module>r‰      s…   ð÷ +× *Ó *Û 	à
ˆ"�)‰)�KÐ!2Ó
3€ØD€	Ø€Ø€Ø€Ø€Ø
€ð ØPØ&Ø ñ	€óIò
óò>POðd ˆzÒÙ…Fð r'   