Ë
    ¸¬uj´  ã                   óú  — d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
mZmZ ddlmZmZ ddlmZ dZdZdZd	Zd
ZdZdZdZ ej2                  dd«      ZdZ ej8                  «       Zd"d„Zd„ Zd„ Z d#d„Z!d„ Z"d$d„Z#d„ Z$e%dk(  rl ejL                  d¬«      Z'e'jQ                  de)dd¬«       e'jQ                  de)dd ¬«       e'jU                  «       Z+ e$e+jX                  e+jZ                  ¬!«       yy)%uU   Crawler: yfyunchengqu.gov.cn - è…°å�¤é•‡æ”¿åºœä¿¡æ�¯å…¬å¼€å¹³å�° (å¹¿ä¸œç»Ÿä¸€gkmlpt)é    N)ÚRequestÚurlopen)Ú	HTTPErrorÚURLError)Údatetimeu   äº‘æµ®å¸‚äº‘åŸŽåŒºè…°å�¤é•‡u   å¹¿ä¸œu   æ”¿åºœå…¬å‘Šzwww.yfyunchengqu.gov.cnz113.104.21.155ÚyfycygÚ766048Ú3637Ú	SEARCH_DBz/mnt/data/search.dbg      à?c           	      óT  — t        |«      D ]c  }	 t        | |xs ddj                  t        t        «      dœ¬«      }t        |dt        ¬«      }|j                  «       j                  dd¬	«      c S  y # t        $ r+}||d
z
  k  rt        j                  d|z  «       Y d }~Œ•‚ d }~ww xY w)Nz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36zhttp://{}/{}/gkmlpt/)z
User-AgentÚReferer)Úheadersé   )ÚtimeoutÚcontextzutf-8Úreplace)Úerrorsé   é   )Úranger   ÚformatÚDOMAINÚSITE_PREFIXr   Ú_CTXÚreadÚdecodeÚ	ExceptionÚtimeÚsleep)Úurlr   ÚretriesÚattemptÚreqÚrespÚes          ú'/root/gov_crawler/crawl_yfyunchengqu.pyÚhttp_getr'      s¥   € Ü˜“>ò ˆð	Ü˜# wò (Ø\Ø1×8Ñ8¼ÄÓMñ3ô ˆCô ˜3¨´DÔ9ˆDØ—9‘9“;×%Ñ% g°iÐ%Ó@Ò@ñøô ò 	Ø˜ 1™Ò$Ü—
‘
˜1 ™<Ô(ÜØûð		ús   �AA3Á3	B'Á< B"Â!B"Â"B'c           	      óÖ  — | syt        | «      j                  «       } t        j                  d| «      }|r>t	        |j                  d«      «      }t        j                  |«      j                  d«      S t        j                  d| «      }|r[dj                  t	        |j                  d«      «      t	        |j                  d«      «      t	        |j                  d«      «      «      S | d	d
 S )zFParse date from gkmlpt API format (Unix timestamp or formatted string)Ú z
^(\d{10})$r   z%Y-%m-%dz(\d{4})-(\d{2})-(\d{2})z{}-{:02d}-{:02d}r   é   Né
   )
ÚstrÚstripÚreÚmatchÚintÚgroupr   ÚfromtimestampÚstrftimer   )ÚrawÚmÚtss      r&   Ú
parse_dater7   %   s±   € áØÜ
ˆc‹(�.‰.Ó
€Cä
�‰� Ó$€AÙÜ�—‘˜“‹_ˆÜ×%Ñ% bÓ)×2Ñ2°:Ó>Ð>ä
�‰Ð+¨SÓ1€AÙØ!×(Ñ(¬¨Q¯W©W°Q«Z«¼#¸a¿g¹gÀa»j»/Ì3ÈqÏwÉwÐWXËzË?Ó[Ð[Øˆs�ˆ8€Oó    c                 ó†  — ddl m}  || d«      }d}|j                  dddi¬«      }|r$|j                  d	«      r|d	   j	                  «       }|sN|j                  d
«      }|r$t        |«      dk\  r|d   j                  d¬«      }n|r|d   j                  d¬«      }d}|j                  dddi¬«      }|r|j                  d	«      rt        |d	   «      }d}|j                  dd¬«      }|s|j                  dd	¬«      }|r g }	|j                  ddgd¬«      D ]u  }
|
j                  dk(  r:|
j                  d«      rŒ$|
j                  dd¬«      }|sŒ:|	j                  |«       ŒL|
j                  dk(  sŒ\|	j                  t        |
«      «       Œw dj                  |	«      }|r|dd j                  dd«      n|dd }||||fS )z(Extract article content from detail pager   )ÚBeautifulSoupzhtml.parserr)   ÚmetaÚnameÚArticleTitle)ÚattrsÚcontentÚh1r   r   T)r-   ÚPubDateÚdivzarticle-content)Úclass_ÚpÚtable)Ú	recursivez

Ni,  ú
ú )Úbs4r:   ÚfindÚgetr-   Úfind_allÚlenÚget_textr7   r<   Úfind_parentÚappendr,   Újoinr   )Úhtmlr:   ÚsoupÚtitler;   Úh1sÚpub_dater?   Úcontent_divÚpartsÚelemÚtextÚsummarys                r&   Úextract_contentr\   6   sÉ  € å!Ù˜˜}Ó-€Dð €EØ�9‰9�V F¨NÐ#;ˆ9Ó<€DÙ�—‘˜Ô#Ø�Y‘×%Ñ%Ó'ˆÙØ�m‰m˜DÓ!ˆÙ”3�s“8˜q’=Ø˜‘F—O‘O¨$�OÓ/‰EÙØ˜‘F—O‘O¨$�OÓ/ˆEð €HØ�9‰9�V F¨IÐ#6ˆ9Ó7€DÙ�—‘˜Ô#Ü˜d 9™oÓ.ˆð €GØ—)‘)˜EÐ*;�)Ó<€KÙØ—i‘i ¨i�iÓ8ˆÙØˆØ×(Ñ(¨#¨w¨À4Ð(ÓHò 	(ˆDØ�y‰y˜CÒØ×#Ñ# GÔ,ØØ—}‘} R¨t�}Ó4�ÚØ—L‘L Õ&Ø—‘˜gÓ%Ø—‘œS ›YÕ'ð	(ð —+‘+˜eÓ$ˆñ 3:ˆg�d�sˆm×#Ñ# D¨#Ô.¸uÀTÀc¸{€Gà�(˜G WÐ,Ð,r8   é   c                 ó\  — g }t        d| dz   «      D ]è  }dj                  t        t        t        |t
        «      }	 t        |«      }t        j                  |«      }|j                  dg «      }|j                  dd«      }|st        d|› d|› d�«        |S |j                  |«       t        d|› d	t        |«      › d
t        |«      › d|› d�	«       t        |«      |k\  r |S t        j                  t        «       Œê |S # t         $ r#}t        d|› d|› �«       |dk(  r‚ Y d}~ |S d}~ww xY w)z"Fetch article list from gkmlpt APIr   z-http://{}/{}/gkmlpt/api/all/{}?page={}&sid={}ÚarticlesÚtotalr   z  [PAGE z] No articles (total=ú)z] Got z articles (running: ú/z	] ERROR: N)r   r   r   r   Ú	COLUMN_IDÚSIDr'   ÚjsonÚloadsrK   ÚprintÚextendrM   r   r   ÚDELAYr   )	Ú	max_pagesÚall_articlesÚpageÚapi_urlÚdataÚresultr_   r`   r%   s	            r&   Úfetch_articlesrp   e   sJ  € à€Lä�a˜ Q™Ó'ò ˆØA×HÑHÜ”K¤¨D´#ó7ˆð	Ü˜GÓ$ˆDÜ—Z‘Z Ó%ˆFØ—z‘z *¨bÓ1ˆHØ—J‘J˜w¨Ó*ˆEáÜ˜  Ð&;¸E¸7À!ÐDÔEØð  Ðð ×Ñ Ô)Ü�H˜T˜F &¬¨X«¨Ð7KÌCÐP\ÓL]ÐK^Ð^_Ð`eÐ_fÐfgÐhÔiä�<Ó  EÒ)Øð Ðô �J‰J”uÕð'ð6 Ðøô ò 	Ü�H˜T˜F )¨A¨3Ð/Ô0Ø�qŠyØÜàÐûð	ús%   »AC?ÂA	C?Ã"C?Ã?	D+ÄD&Ä&D+c           	      óä  — | j                  dd«      }|sy| j                  dd«      j                  «       }t        | j                  d«      xs% | j                  d«      xs | j                  dd«      «      }	 t        |«      }t	        |«      \  }}}}|r|}|r|}|s|st        d|dd	 › �«       y|||||t        t        t        dœS # t        $ r}	t        d
|dd › d|	› �«       Y d}	~	yd}	~	ww xY w)z"Fetch detail page and extract infor    r)   NrT   ÚdateÚcreate_timeÚ
created_atz    [SKIP] No content: é2   z    [ERROR] Detail: é<   z - )rT   r    rV   r?   r[   Ú	site_nameÚ
group_nameÚindustry)
rK   r-   r7   r'   r\   rg   r   Ú	SITE_NAMEÚGROUPÚINDUSTRY)
Úarticler    rT   rV   rR   Údetail_titleÚdetail_dater?   r[   r%   s
             r&   Úprocess_articler€   ‡   s  € à
�+‰+�e˜RÓ
 €CÙØà�K‰K˜ Ó$×*Ñ*Ó,€Eä˜'Ÿ+™+ fÓ-Òl°·±¸]Ó1KÒlÈwÏ{É{Ð[gÐikÓOlÓm€HðÜ˜‹}ˆÜ6EÀdÓ6KÑ3ˆ�k 7¨GáØ ˆEÙØ"ˆHá™wÜÐ+¨E°#°2¨J¨<Ð8Ô9Øð ØØØØÜÜÜñ	ð 	øô	 ò ÜÐ$ S¨¨" X J¨c°!°Ð5Ô6Üûðús   Á88C Ã	C/ÃC*Ã*C/c                 ó¶  — | st        d«       yg }| D ]Š  }d„ } ||d   «      } ||d   «      } ||d   «      } ||d   «      } ||d   «      }	 ||d	   «      }
 ||d
   «      } ||d   «      }|j                  d|› d|› d|› d|› d|› d|	› d|
› d|› d|› d�«       ŒŒ dj                  |«      }t        d«      D ]Y  }t	        j
                  dt        g|ddd¬«      }|j                  dk(  r nRd|j                  v rddl	} |j                  d«       ŒY n+ t        dj                  j                  «       dd › �«       yt        dt        | «      › d�«       y)z?Write to search.db - gov_raw only (FTS sync handled separately)z  [SKIP] no dataNc                 ó@   — | rt        | «      j                  dd«      S dS )Nú'z''r)   )r,   r   )Úss    r&   ú<lambda>zpush_to_db.<locals>.<lambda>·   s   € ±Qœ˜A›Ÿ™ s¨DÓ1€ ¸B€ r8   r    rT   r?   r[   rw   rx   ry   rV   z|INSERT OR IGNORE INTO gov_raw(source_url,page_url,title,content,summary,site_name,group_name,industry,publish_date) VALUES('z','z');rG   r*   Úsqlite3Té   )ÚinputÚcapture_outputrZ   r   r   Úlockedz  [DB ERROR] éÈ   z  [DB] z records written)rg   rP   rQ   r   Ú
subprocessÚrunr   Ú
returncodeÚstderrr   r   r-   rM   )ÚitemsÚbatch_labelÚlinesÚitemÚescr    rT   r?   r[   ÚsiteÚgroup_ry   rV   Úall_sqlr"   Úrr   s                    r&   Ú
push_to_dbr™   ¯   s¤  € áÜÐ Ô!Øà€EØò 
ˆÙ>ˆÙ�$�u‘+ÓˆÙ�D˜‘MÓ"ˆÙ�d˜9‘oÓ&ˆÙ�d˜9‘oÓ&ˆÙ�4˜Ñ$Ó%ˆÙ�T˜,Ñ'Ó(ˆÙ�t˜JÑ'Ó(ˆÙ�t˜JÑ'Ó(ˆØ�‰ðØ�e˜3˜s˜e 3 u g¨S°°	¸¸W¸IÀSÈÈÈcÐRXÐQYÐY\Ð]eÐ\fÐfiÐjrÐisÐsvðxõ	
ð
ð �i‰i˜Ó€GÜ˜“8ò ˆÜ�N‰N˜I¤yÐ1¸Ø)-°DÀ"ôFˆà�<‰<˜1ÒÙØ�q—x‘xÑÛ˜˜Ÿ™ AœØÙðô 	�˜aŸh™hŸn™nÓ.¨t°Ð4Ð5Ð6Ô7Øä	ˆG”C˜“J�<Ð/Ð
0Õ1r8   c           	      ó$  — t        d| › d�«       t        | ¬«      }t        dt        |«      › d�«       |r!|t        |«      k  r|d | }t        d|› �«       g }d\  }}t        |«      D ]†  \  }}t        d|d	z   › d
t        |«      › d�d¬«       t	        |«      }|r+|j                  |«       |d	z  }t        d|d   d d › �«       n|d	z  }t        d«       t        j                  t        «       Œˆ t        d|› d|› d�«       t        |d¬«       y )Nu#   [è…°å�¤é•‡gkmlpt] Start (max_pages=ra   )rj   z  API returned z	 articlesz  Limited to )r   r   z  [r   rb   z] Processing...rH   )ÚendzOK: rT   ru   ÚSKIPz
Done: z OK, z skippedÚyfyunchengqu)r‘   )
rg   rp   rM   Ú	enumerater€   rP   r   r   ri   r™   )	rj   Úlimitr_   r�   ÚokÚerrÚir}   r“   s	            r&   Úmainr£   Ö   s  € Ü	Ð/°	¨{¸!Ð
<Ô=ä¨	Ô2€HÜ	ˆOœC ›M˜?¨)Ð
4Ô5á�œ˜X›Ò&Ø˜F˜UÐ#ˆÜ�˜e˜WÐ%Ô&à€EØ�G€BˆÜ Ó)ò 
‰
ˆˆ7Ü��A�a‘C�5˜œ#˜h›-˜¨Ð8¸cÕBÜ˜wÓ'ˆÙØ�L‰L˜ÔØ�!‰GˆBÜ�D˜˜g™ s¨Ð+Ð,Ð-Õ.à�1‰HˆCÜ�&ŒMÜ�
‰
”5Õð
ô 
ˆH�R�D˜˜c˜U (Ð
+Ô,Üˆu .Ö1r8   Ú__main__u   è…°å�¤é•‡gkmlptçˆ¬è™«)Údescriptionz--pagesu   æœ€å¤§é¡µæ•°)ÚtypeÚdefaultÚhelpz--limitu   é™�åˆ¶æ�¡æ•°)rj   rŸ   )Nr*   )r]   )Úcrawl).Ú__doc__Úargparsere   Úosr.   ÚsysrŒ   Ússlr   Ú	tracebackÚurllib.requestr   r   Úurllib.errorr   r   r   rz   r{   r|   r   ÚIPr   rd   rc   Úgetenvr   ri   Úcreate_default_contextr   r'   r7   r\   rp   r€   r™   r£   Ú__name__ÚArgumentParserÚparserÚadd_argumentr0   Ú
parse_argsÚargsÚpagesrŸ   © r8   r&   ú<module>r½      s  ðá [ß D× D× D× DÓ Dß +ß ,Ý à)€	Ø€Ø€Ø	"€Ø€Ø€Ø€Ø€	ØˆB�I‰I�kÐ#8Ó9€	Ø€Ø!€s×!Ñ!Ó#€óò ò",-ó^òD%óP$2òN2ð8 ˆzÒØ$ˆX×$Ñ$Ð1HÔI€FØ
×Ñ˜	¨°Q¸^ÐÔLØ
×Ñ˜	¨°Q¸^ÐÔLØ×ÑÓ€DÙ�4—:‘: T§Z¡ZÖ0ð r8   