Ë
    ¸¬uj;  ã                   ó¨  — d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl	m	Z	m
Z
 dZ ej                  dd«      ZdZ ej                  «       Zej"                  j%                  d	d
i«       de_         ej(                  «         e	j*                  «        e
d¬«      z
  j-                  d«      ZdZdZdZdd„Zd„ Zd„ Zd„ Zd„ Ze dk(  r e«        yy)uœ   
Crawl è¡¡æ°´é«˜æ–°åŒº - ç”Ÿæ€�çŽ¯å¢ƒ (hskfq.gov.cn/col/col11730/)
Data loaded via AJAX proxy: /module/web/jpage/dataproxy.jsp
Detail: div.neirong / div.nr
é    N)ÚBeautifulSoup)ÚdatetimeÚ	timedeltazhttp://www.hskfq.gov.cnÚ	SEARCH_DBz/root/search.dbu   è¡¡æ°´é«˜æ–°åŒº - ç”Ÿæ€�çŽ¯å¢ƒz
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36FiG  )Údaysz%Y-%m-%dz/module/web/jpage/dataproxy.jspzVpage={}&appid=1&webid=15&path=/&columnid=11730&unitid=4063&webname={}&permissiontype=0u6   æ²³åŒ—è¡¡æ°´é«˜æ–°æŠ€æœ¯äº§ä¸šå¼€å�‘åŒºç®¡ç�†å§”å‘˜ä¼šc                 ó”  — ddl }t        j                  | |j                  j	                  t
        «      «      }t        › t        › d|› �}	 t        j                  |d¬«      }|j                  dk7  ry|j                  }ddlmc m} |j                  |«      }t!        |j#                  dd	«      «      }t!        |j#                  d
d	«      «      }	g }
|j%                  d«      D ]>  }|j                  r|j                  j'                  «       nd}|sŒ.|
j)                  |«       Œ@ |
||	fS # t*        $ r}t-        d|› �«       Y d}~yd}~ww xY w)zCall AJAX proxy, return itemsr   Nú?é   ©ÚtimeoutéÈ   )Nr   r   ÚtotalrecordÚ0Ú	totalpagez	.//recordÚ z  AJAX error: )Úurllib.parseÚAJAX_PARAMSÚformatÚparseÚquoteÚWEBNAMEÚBASE_URLÚAJAX_URLÚsessionÚgetÚstatus_codeÚtextÚxml.etree.ElementTreeÚetreeÚElementTreeÚ
fromstringÚintÚfindtextÚfindallÚstripÚappendÚ	ExceptionÚprint)ÚpageÚurllibÚparamsÚurlÚrespÚxmlÚETÚrootÚtotal_recordsÚtotal_pagesÚrecordsÚrecordÚcdataÚes                 ú /root/gov_crawler/crawl_hskfq.pyÚfetch_list_pager8      s   € ãÜ×Ñ  f§l¡l×&8Ñ&8¼Ó&AÓB€FÜˆJ”x�j  & Ð
*€CðÜ�{‰{˜3¨ˆ{Ó+ˆØ×Ñ˜sÒ"ØØ�i‰iˆß*Ð*Ø�}‰}˜SÓ!ˆÜ˜DŸM™M¨-¸Ó=Ó>ˆÜ˜$Ÿ-™-¨°SÓ9Ó:ˆØˆØ—l‘l ;Ó/ò 	&ˆFØ+1¯;ª;�F—K‘K×%Ñ%Ô'¸BˆEÚØ—‘˜uÕ%ð	&ð ˜ {Ð2Ð2øÜò Ü�˜q˜cÐ"Ô#Üûðús%   Á
&D& Á1BD& ÄD& Ä&	EÄ/EÅEc                 óö  — g }| D ]ñ  }t        |d«      }|j                  d«      }|sŒ#|j                  dd«      }|j                  dd«      xs |j                  d¬«      }|r|sŒ`|j	                  d«      r
t
        |z   }n,|j	                  d	«      st
        dz   |j                  d«      z   }|j                  d
«      }|r!|j                  d¬«      j                  d«      nd}|j                  |||dœ«       Œó |S )zParse CDATA HTML itemsúhtml.parserÚaÚhrefr   ÚtitleT©r%   ú/ÚhttpÚspanz[])r=   r,   Údate)	r   Úfindr   Úget_textÚ
startswithr   Úlstripr%   r&   )	Ú
cdata_listr3   r5   Úsoupr;   r<   r=   rA   Údate_strs	            r7   Úparse_list_itemsrJ   6   sî   € à€GØò HˆÜ˜U MÓ2ˆØ�I‰I�c‹NˆÙØØ�u‰u�V˜RÓ ˆØ—‘�g˜rÓ"Ò< a§j¡j°t jÓ&<ˆÙ™5ØØ�?‰?˜3ÔÜ˜d‘?‰DØ—‘ Ô(Ü˜c‘> D§K¡K°Ó$4Ñ4ˆDà�y‰y˜Ó ˆÙ<@�4—=‘= t�=Ó,×2Ñ2°4Ô8ÀbˆØ�‰ ¨t¸XÑFÕGð!Hð" €Nó    c                 óÐ  — 	 t         j                  | d¬«      }d|_        t	        |j
                  d«      }|j                  dd¬«      }|s|j                  dd	¬«      }|rt        |«      nd
}d
}|j                  dddi¬«      }|r'|j                  d«      r|d   j                  «       d d }d
}|j                  d«      }|r|j                  d¬«      }|||fS # t        $ r Y yw xY w)Né   r   zutf-8)r   r   r   r:   ÚdivÚnr)Úclass_Úneirongr   ÚmetaÚnameÚPubDate)ÚattrsÚcontenté
   Úh1Tr>   )
r   r   Úencodingr'   r   r   rC   Ústrr%   rD   )	r,   r-   rH   Úcontent_divÚcontent_htmlrI   rR   Ú
full_titlerX   s	            r7   Úfetch_detailr^   L   sð   € ðÜ�{‰{˜3¨ˆ{Ó+ˆØˆŒô ˜Ÿ™ MÓ2€DØ—)‘)˜E¨$�)Ó/€KÙØ—i‘i ¨i�iÓ8ˆÙ'2”3�{Ô#¸€LØ€HØ�9‰9�V F¨IÐ#6ˆ9Ó7€DÙ�—‘˜Ô#Ø˜	‘?×(Ñ(Ó*¨3¨BÐ/ˆØ€JØ	�‰�4‹€BÙ	Ø—[‘[ t�[Ó,ˆ
Ø˜ :Ð-Ð-øô ò Ùðús   ‚C Ã	C%Ã$C%c                 óŽ  — |j                  «       }d}d}| D ]u  }|d   r|d   t        k  r|dz  }Œ	 |j                  d|d   t        |d   d|j	                  dd«      |j	                  dd«      t
        f«       |j                  dkD  r|dz  }Œw |j                  «        ||fS # t        $ r}t        d	|› �«       Y d }~Œ¨d }~ww xY w)
Nr   rB   é   zÂ
                INSERT OR IGNORE INTO gov_raw 
                (page_url, source_url, title, summary, content, publish_date, site_name)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            r,   r=   r   rV   z  DB error: )
ÚcursorÚTHREE_YEARS_AGOÚexecuter   r   Ú	SITE_NAMEÚrowcountr'   r(   Úcommit)r3   Úconnra   ÚinsertedÚskippedÚrecr6   s          r7   Úinsert_to_dbrk   a   så   € Ø�[‰[‹]€FØ€HØ€GØò &ˆØˆvŠ;˜3˜v™;¬Ò8Ø�q‰LˆGØð		&Ø�N‰Nð ð �e‘*œh¨¨G©°b¸#¿'¹'À)ÈRÓ:PÐRU×RYÑRYÐZ`ÐbdÓReÔgpÐqô	sð
 �‰ Ò"Ø˜A‘�øð&ð 	‡K�K„MØ�WÐÐøô ò 	&Ü�L  Ð$×%Ñ%ûð	&ús   ²AB#Â#	CÂ,B?Â?Cc            	      óÞ  — dt         j                  v xs t         j                  d   dk(  } dd l}|j                  t        «      }t        dt        › d�«       t        d«      \  }}}|€t        d«       |j                  «        y t        |«      }t        d	t        |«      › d
|› d|› d�«       | sƒt        d|«      }t        d|dz   «      D ]e  }t        |«      \  }	}
}
|	s nRt        |	«      }|s nCt        d|› dt        |«      › d�«       |j                  |«       t        j                  d«       Œg t        dt        |«      › �«       t        d«       t!        |«      D ]Q  \  }}|dz  dk(  rt        d|› dt        |«      › �d¬«       t#        |d   «      \  }}}|r||d<   |r||d<   |sŒM||d<   ŒS t%        ||«      \  }}|j                  «        t        d«       t        d|› d |› �«       y )!NÚincrementaléÿÿÿÿÚ1r   z=== z ===r`   zFailed to fetch dataz
  Page 1: z records (total: z	, pages: ú)é   é   z  Page z: z recordsg333333Ó?z
Total collected: zFetching details...rW   z  r?   T)Úflushr,   rV   rB   r=   z
=== SUMMARY ===z
Inserted: z, Skipped (old): )ÚsysÚargvÚsqlite3ÚconnectÚDB_PATHr(   rd   r8   ÚcloserJ   ÚlenÚminÚrangeÚextendÚtimeÚsleepÚ	enumerater^   rk   )rm   rv   rg   rG   Útotalr2   Úall_recordsÚ	max_pagesr)   Úcdata_list2Ú_r3   Úirj   rV   Údate_from_detailr]   rh   ri   s                      r7   Úmainrˆ   v   sý  € Ø¤3§8¡8Ð+ÒB¬s¯x©x¸©|¸sÑ/B€KÛØ�?‰?œ7Ó#€DÜ	ˆD”�˜4Ð
 Ô!ô &5°QÓ%7Ñ"€J��{ØÐÜÐ$Ô%Ø�
‰
ŒØä" :Ó.€KÜ	ˆJ”s˜;Ó'Ð(Ð(9¸%¸À	È+ÈÐVWÐ
XÔYñ Ü˜˜;Ó'ˆ	Ü˜!˜Y¨™]Ó+ò 		ˆDÜ /°Ó 5ÑˆK˜˜AÙÙÜ& {Ó3ˆGÙÙÜ�G˜D˜6 ¤C¨£L >°Ð:Ô;Ø×Ñ˜wÔ'Ü�J‰J�s�Oð		ô 
Ð¤ KÓ 0Ð1Ð
2Ô3Ü	Ð
Ô Ü˜KÓ(ò 	&‰ˆˆ3Øˆr‰6�QŠ;Ü�B�q�c˜œ3˜{Ó+Ð,Ð-°TÕ:Ü0<¸SÀ¹ZÓ0HÑ-ˆÐ! :ÙØ$ˆC�	‰NÙØ*ˆC�‰KÚØ%ˆC�ŠLð	&ô % [°$Ó7Ñ€HˆgØ‡J�J„LÜ	ÐÔÜ	ˆJ�x�jÐ 1°'°Ð
;Õ<rK   Ú__main__)r`   )!Ú__doc__rt   Úrer~   ÚosÚrequestsÚurllib3Úbs4r   r   r   r   Úgetenvrx   rd   ÚSessionr   ÚheadersÚupdateÚverifyÚdisable_warningsÚnowÚstrftimerb   r   r   r   r8   rJ   r^   rk   rˆ   Ú__name__© rK   r7   ú<module>rš      sÞ   ðñ÷ Ó ß Ó Ý ß (à$€Ø
ˆ"�)‰)�KÐ!2Ó
3€Ø,€	à
ˆ(×
Ñ
Ó
€Ø ‡�× Ñ ØÐPðô ð €„Ø €× Ñ Ô à�8—<‘<“>¡I°5Ô$9Ñ9×CÑCÀJÓO€à,€Øf€Ø
B€óò0ò,.ò*ò*.=ð` ˆzÒÙ…Fð rK   