Ë
    ½ÞŽjû  ã                   óˆ  — d dl Z 	 d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl	m
Z
 d dlZ eej                  «      dkD  r2ej                  d   j                  «       r eej                  d   «      ndZe� ed ee«      z   «        e j&                  dd«      ZdZd	Zd
ddœZdZedz   ZdZd„ Zd„ Zd„ Zedk(  r e«        yy)é    N)ÚurlopenÚRequest)Ú	urlencodeé   z[AutoPg] max_pages=Ú	SEARCH_DBz/root/search.dbzhttp://www.qidong.gov.cnu(   å�¯ä¸œå¸‚äººæ°‘æ”¿åºœ-éƒ¨é—¨å…¬å‘Šå…¬ç¤ºzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z3http://www.qidong.gov.cn/qdsrmzf/bmgggs/bmgggs.html)z
User-AgentÚRefererz$0106496c-0949-485f-ba9e-e5c67592bc9az(/truecms/messageController/getMessage.doé   c                 óŠ  — | |dt         dœ}t        › dt        |«      › d�}t        |t        ¬«      }t        |d¬«      }|j                  «       j                  dd	¬
«      }||j                  d«      |j                  d«      dz    }t        j                  |«      }|j                  dd«      }	t        j                  d|	«      }
|
S )z'Fetch a group of articles from the API.é
   )ÚstartrecordÚ	endrecordÚperpageÚcolumnIdú?z&callback=cb©Úheadersr	   ©Útimeoutúutf-8Úreplace©Úerrorsú{ú}r   ÚresultÚ zG<a[^>]*href="([^"]+)"[^>]*>(.*?)</a>\s*<span>(\d{4}-\d{2}-\d{2})</span>)Ú	COLUMN_IDÚAPI_URLr   r   ÚHEADERSr   ÚreadÚdecodeÚindexÚrindexÚjsonÚloadsÚgetÚreÚfindall)r   r   ÚparamsÚurlÚreqÚrespÚbodyÚjson_strÚdataÚxmlÚrecordss              ú!/root/gov_crawler/crawl_qidong.pyÚ
fetch_listr3       sÃ   € ð #ØØÜñ	€Fô ˆI�Q”y Ó(Ð)¨Ð
6€CÜ
�#œwÔ
'€CÜ�3 Ô#€DØ�9‰9‹;×Ñ˜g¨iÐÓ8€Dð �D—J‘J˜s“O D§K¡K°Ó$4°QÑ$6Ð7€HÜ�:‰:�hÓ€DØ
�(‰(�8˜RÓ
 €Cô �j‰jØRØó€Gð €Nó    c                 ó¦  — t        | t        ¬«      }t        |d¬«      }|j                  «       j	                  dd¬«      }t        j                  d|t
        j                  «      }|�rj|j                  d«      }d}d	}t        |«      D ]œ  \  }}	|	d
k(  sŒ|j                  d|«      }
|
dk(  r nz||dz   |
 j                  «       }|j                  «       d	   }|j                  d«      r|dz  }|d	k(  r2|
dz   } n/|j                  d«      s|dvr|j                  d«      s|dz  }|
}Œž |d	kD  r|d|dz
   j                  «       n|j                  «       }t        j                  dd|t
        j                  t
        j                   z  ¬«      }t        j                  dd|t
        j                  t
        j                   z  ¬«      }|j                  «       S y)zFetch detail page content.r   r	   r   r   r   r   z-<div[^>]*class=[\"\']content[\"\']?[^>]*>(.*)r   r   ú<ú>éÿÿÿÿú/z!--)ÚbrÚhrÚimgÚinputÚmetaÚlinkz!DOCTYPENz<script[^>]*>.*?</script>r   )Úflagsz<style[^>]*>.*?</style>)r   r   r   r    r!   r'   ÚsearchÚDOTALLÚgroupÚ	enumerateÚfindÚstripÚsplitÚ
startswithÚendswithÚsubÚI)r*   r+   r,   ÚhtmlÚcÚrestÚdepthÚposÚiÚchÚtag_endÚtagÚtag_nameÚcontents                 r2   Úfetch_contentrW   :   s¶  € ä
�#œwÔ
'€CÜ�3 Ô#€DØ�9‰9‹;×Ñ˜g¨iÐÓ8€Dô 	�	‰	ÐBÀDÌ"Ï)É)ÓT€AÚØ�w‰w�q‹zˆàˆØˆÜ˜t“_ò 	‰EˆAˆrØ�S‹yØŸ)™) C¨Ó+�Ø˜b’=ÙØ˜1˜Q™3˜wÐ'×-Ñ-Ó/�ØŸ9™9›; q™>�Ø—>‘> #Ô&Ø˜Q‘J�EØ ’zØ%¨™k˜ÙØŸ™¨Ô.°8ÐC{Ñ3{ð  EH÷  EQñ  EQð  RUô  EVØ˜Q‘J�EØ‘ð	ð +.°ª'�$�v˜˜A™�,×$Ñ$Ô&°t·z±z³|ˆÜ—&‘&Ð5°r¸7Ì"Ï)É)ÔVX×VZÑVZÑJZÔ[ˆÜ—&‘&Ð3°R¸ÌÏ	É	ÔTV×TXÑTXÑHXÔYˆØ�}‰}‹ÐØr4   c                  ó   — t        j                   «       } t        j                  t        «      }|j	                  d«       t        dt        «      }t        dt        › d�«       d}d}d}t        t        |t        xs |«      «      D �]  }|t        z  dz   }t        |t        z   dz
  d«      }t        d|dz   › d	|› d
|› d|› d�	«       t        ||«      }|D ]�  \  }	}
}t        j                  dd|
«      j                  «       }|	j                  d	«      r	t        |	z   n|	}|dz  }d}	 t!        |«      }t        j"                  d«       |j	                  d|||||t&        f«       |dz  }Œ’ |j)                  «        t        d|› d�«       t        j"                  d«       �Œ t        j                   «       | z
  }t        d|› d	|› d|d›d�«       |j+                  «        y # t$        $ r}t        d|› �«       Y d }~Œ´d }~ww xY w)NzPRAGMA busy_timeout=5000r   z,[qidong] Total: 3300 records (110 groups of ú)r   én   iä  z[qidong] Group r9   z
 (records ú-z)...z<[^>]+>r   gš™™™™™¹?z  [warn] Detail failed: zxINSERT OR REPLACE INTO gov_raw (title, content, publish_date, page_url, source_url, site_name) VALUES (?, ?, ?, ?, ?, ?)z[qidong] Progress: z/3300 savedg333333Ó?z[qidong] Done: z
 saved in z.1fÚs)ÚtimeÚsqlite3ÚconnectÚDB_PATHÚexecuter3   Ú
GROUP_SIZEÚprintÚrangeÚminÚ_MAX_PGr'   rJ   rF   rH   ÚBASErW   ÚsleepÚ	ExceptionÚ	SITE_NAMEÚcommitÚclose)ÚstartÚconnr1   Útotal_savedÚtotal_articlesÚtotal_groupsrC   ÚstartrecÚendrecÚhrefÚ	title_rawÚdateÚtitleÚarticle_urlrV   ÚeÚelapseds                    r2   Úmainr{   ]   sû  € Ü�I‰I‹K€EÜ�?‰?œ7Ó#€DØ‡L�LÐ+Ô,ô ˜œJÓ'€GÜ	Ð8¼¸ÀAÐ
FÔGà€KØ€NØ€Lä”s˜<¬Ò)@°LÓAÓBó ˆØœ:Ñ%¨Ñ)ˆÜ�X¤
Ñ*¨QÑ.°Ó5ˆä�  a¡˜y¨¨,¨°zÀ(ÀÈ1ÈVÈHÐTXÐYÔZÜ˜X vÓ.ˆà%,ò 	Ñ!ˆD�)˜TÜ—F‘F˜: r¨9Ó5×;Ñ;Ó=ˆEØ)-¯©¸Ô)=œ$ š+À4ˆKà˜aÑˆNð ˆGð6Ü'¨Ó4�Ü—
‘
˜3”ð �L‰Lð KØ˜  {°KÄÐKôð ˜1Ñ‰Kð%	ð( 	�‰ŒÜÐ# K =°Ð<Ô=Ü�
‰
�3Žð;ô> �i‰i‹k˜EÑ!€GÜ	ˆO˜K˜=¨¨.Ð)9¸ÀGÈCÀ=ÐPQÐ
RÔSØ‡J�J…Løô ò 6ÜÐ0°°Ð4×5Ñ5ûð6ús   Ä$ GÇ	G=Ç%G8Ç8G=Ú__main__)Úosr'   Úsysr]   r$   r^   Úurllib.requestr   r   Úurllib.parser   Ú_SYSÚlenÚargvÚisdigitÚintrf   rc   ÚstrÚgetenvr`   rg   rj   r   r   r   rb   r3   rW   r{   Ú__name__© r4   r2   ú<module>rŠ      sÛ   ðã 	ð÷ $× #Ó #ß +Ý "ã Ù" 4§9¡9›~°Ò1°d·i±iÀ±l×6JÑ6JÔ6L‰#ˆd�i‰i˜‰lÔ
ÐRV€Ø
ÐÙ	Ð
¡# g£,Ñ
.Ô/ð ˆ"�)‰)�KÐ!2Ó
3€Ø!€Ø6€	ð DØDñ€ð 3€	Ø
Ð;Ñ
;€Ø€
òò4 òF.ðb ˆzÒÙ…Fð r4   