Ë
    ¡Š£jI.  ã            	       óÜ  — d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	 ddl
mZ ddlZ ej                  ej                  j                  «       dZedz   ZdZdZd	Zd
ZdZ ej.                  dd«      ZddiZ ej4                  «       Zej8                  j;                  e«       de_        ddedz   ddddddœZedz   Z h d£Z!d„ Z"d„ Z#d"d„Z$d„ Z%d„ Z&d„ Z'd„ Z(d „ Z)e*d!k(  r e)«        yy)#ud  
æ­¦å¨�å¸‚äººæ°‘æ”¿åºœ - å�„éƒ¨é—¨æ”¿ç­–æ–‡ä»¶
URL: https://www.gswuwei.gov.cn/col/col1003/index.html?number=WWSA1533
CMS: å¤§æ±‰ç½‘ç»œ (Hanweb) + JPage AJAXåˆ†é¡µ
List API: POST /module/web/jpage/dataproxy.jsp?startrecord=N&endrecord=M&perpage=15
  Body: col=1&webid=1&path=...&columnid=1003&unitid=5827&webname=æ­¦å¨�å¸‚äººæ°‘æ”¿åºœ
Detail: div#Zoom > p
é    N)ÚBeautifulSoup)Úurljoinzhttps://www.gswuwei.gov.cnz'/col/col1003/index.html?number=WWSA1533u+   æ­¦å¨�å¸‚äººæ°‘æ”¿åºœ-å�„éƒ¨é—¨æ”¿ç­–æ–‡ä»¶u   å�„éƒ¨é—¨æ”¿ç­–æ–‡ä»¶u   ç”˜è‚ƒé   é-   Ú	SEARCH_DBz/root/search.dbz
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36Fé   ú/ië  Ú5827u   æ­¦å¨�å¸‚äººæ°‘æ”¿åºœ)ÚcolÚwebidÚpathÚcolumnidÚsourceContentTypeÚunitidÚwebnameÚpermissiontypez/module/web/jpage/dataproxy.jsp>)   ÚpÚddÚdlÚdtÚh1Úh2Úh3Úh4Úh5Úh6ÚhrÚliÚolÚtdÚthÚtrÚulÚdivÚnavÚpreÚformÚmainÚasideÚtableÚtbodyÚtfootÚtheadÚcenterÚdialogÚfigureÚfooterÚheaderÚhgroupÚaddressÚarticleÚcaptionÚdetailsÚsectionÚfieldsetÚ
blockquoteÚ
figcaptionc                 óÒ   ‡‡‡— | €yddl }ddlmŠ g Šˆˆˆfd„Š ‰| «       dj                  ‰«      }|j	                  dd|«      }|j	                  dd	|«      }|j                  «       S )
ul   å�—çº§è¾¹ç•Œå‡ºæ�¢è¡Œã€�è¡Œå†…èŠ‚ç‚¹ç›´æŽ¥æ‹¼æŽ¥ã€�<br> å‡ºæ�¢è¡Œï¼ˆâ‰ˆ æµ�è§ˆå™¨çœ‹åˆ°çš„æ�¢è¡Œç»“æž„ï¼‰ã€‚NÚ r   )ÚNavigableStringc                 óF  •— | j                   D ]‘  }t        |‰«      r‰j                  t        |«      «       Œ*t	        |dd «      dk(  r‰j                  d«       ŒLt	        |dd «      t
        v r+‰j                  d«        ‰|«       ‰j                  d«       ŒŠ ‰|«       Œ“ y )NÚnameÚbrú
)ÚchildrenÚ
isinstanceÚappendÚstrÚgetattrÚ_BLOCK_TAGS)ÚnodeÚchr>   ÚoutÚwalks     €€€ú /root/gov_crawler/crawl_wuwei.pyrL   zbody_text.<locals>.walkA   s„   ø€ Ø—-‘-ò 
	ˆBÜ˜"˜oÔ.Ø—
‘
œ3˜r›7Õ#Ü˜˜V TÓ*¨dÒ2Ø—
‘
˜4Õ Ü˜˜V TÓ*¬kÑ9Ø—
‘
˜4Ô Ù�R”Ø—
‘
˜4Õ á�R•ñ
	ó    z[ \t\r\f\v]*\n[ \t\r\f\v]*rB   z\n{3,}ú

)ÚreÚbs4r>   ÚjoinÚsubÚstrip)ÚelÚ_reÚtr>   rK   rL   s      @@@rM   Ú	body_textrX   9   sb   ú€ à	€zØÛÝ#Ø
€Cöñ 	ˆ„HØ
�‰�‹€AØ�‰Ð-¨t°QÓ7€AØ�‰�	˜6 1Ó%€AØ�7‰7‹9ÐrN   c                 ó<   — t        | t        j                  d¬«       y )NT)ÚfileÚflush)ÚprintÚsysÚstderr)Úmsgs    rM   Úlogr`   T   s   € Ü	ˆ#”C—J‘J dÖ+rN   c           
      ó   — t        |«      D ].  }	 t        j                  | d¬«      }d|_        |j                  c S  y	# t
        $ r6}t        d|dz   › d|› d|› �«       t        j                  d«       Y d }~Œld }~ww xY w)
Né   ©Útimeoutúutf-8u     [WARN] è¯·æ±‚å¤±è´¥ (å°�è¯• r   r	   z): é   r=   )	ÚrangeÚsessionÚgetÚencodingÚtextÚ	Exceptionr`   ÚtimeÚsleep)ÚurlÚretriesÚattemptÚrÚes        rM   Úfetchrt   X   s€   € Ü˜“>ò ˆð	Ü—‘˜C¨�Ó,ˆAØ ˆAŒJØ—6‘6ŠMð	ð øô ò 	ÜÐ0°¸±°¸1¸W¸IÀSÈÈÐLÔMÜ�J‰J�q�M‰Mûð	ús   �)>¾	A=Á,A8Á8A=c                 ó\  — t         › d| › d|› dt        › �}dt        dœ}t        d«      D ]A  }	 t        j                  |t        |d¬«      }d	|_        |j                  r|j                  c S ŒC y# t        $ r3}t        d
|dz   › d|› �«       t        j                  d«       Y d}~Œ|d}~ww xY w)u!   è°ƒç”¨JPage APIèŽ·å�–åˆ—è¡¨æ•°æ�®z?startrecord=z&endrecord=z	&perpage=ÚXMLHttpRequest)zX-Requested-WithÚRefereré   rb   )ÚdataÚheadersrd   re   u#     [WARN] JPageè¯·æ±‚å¤±è´¥ (å°�è¯• r   z/3): rf   Nr=   )Ú	JPAGE_URLÚPER_PAGEÚLIST_URLrg   rh   ÚpostÚJPAGE_PARAMSrj   rk   rl   r`   rm   rn   )ÚstartrecordÚ	endrecordro   rz   rq   rr   rs   s          rM   Úfetch_jpager‚   d   s²   € äˆK�} [ M°¸Y¸KÀyÔQYÐPZÐ
[€CØ#3ÄÑI€GÜ˜“8ò ˆð	Ü—‘˜S¤|¸WÈb�ÓQˆAØ ˆAŒJØ�vŠvØ—v‘v’ð ð	ð øô ò 	ÜÐ5°g¸a±i°[ÀÀaÀSÐIÔJÜ�J‰J�q�M‰Mûð	ús   ®;A/Á/	B+Á8)B&Â&B+c                 ó2  — g }t        j                  d| t         j                  «      }|D ]ê  }t        j                  d|«      }t        j                  d|«      }t        j                  d|«      }|sŒH|sŒK|j	                  d«      }|j	                  d«      j                  «       }t        j                  dd|«      j                  «       }|r|j	                  d«      nd}	|j                  |j                  d«      r|nt        t        |«      ||	d	œ«       Œì |S )
u'   ä»ŽJPage APIè¿”å›žçš„XMLä¸­è§£æž�æ�¡ç›®z<!\[CDATA\[(.*?)\]\]>zhref='([^']+)'ztitle='([^']+)'z$<span[^>]*>(\d{4}[-/]\d{2}[-/]\d{2})r   u   ^Â·\s*|^&middot;\s*|^\s*r=   Úhttp)ro   ÚtitleÚdate)rP   ÚfindallÚDOTALLÚsearchÚgrouprT   rS   rE   Ú
startswithr   ÚBASE)
Úxml_textÚitemsÚrecordsÚrecÚhref_mÚtitle_mÚdate_mÚhrefr…   Údate_strs
             rM   Úparse_list_xmlr–   t   së   € à€EÜ�j‰jÐ1°8¼R¿Y¹YÓG€GØò ˆÜ—‘Ð,¨cÓ2ˆÜ—)‘)Ð.°Ó4ˆÜ—‘ÐBÀCÓHˆÚ’gØ—<‘< “?ˆDØ—M‘M !Ó$×*Ñ*Ó,ˆEä—F‘FÐ6¸¸EÓB×HÑHÓJˆEÙ*0�v—|‘| A”°bˆHØ�L‰LØ#Ÿ™¨vÔ6‘t¼GÄDÈ$Ó<OØØ ñõ ðð €LrN   c           	      ó”  — t        | «      }|s|d|g fS t        |d«      }|}|j                  dddi¬«      }|r%|j                  d«      r|d   j	                  «       }n,|j                  dd	¬
«      }|r|j                  dd¬«      }|r|}t        j                  dd|«      j	                  «       }d}|j                  dddi¬«      }	|	r|	j                  d«      r|	d   dd }|s¼t        j                  d| «      }
|
rA|
j                  d«      dd › d|
j                  d«      dd › d|
j                  d«      dd › �}nct        j                  d| «      }|rK|j                  d«      › dt        |j                  d«      «      d›dt        |j                  d«      «      d›�}|j                  dd¬«      }d}g }|rÓg }|j                  g d¢d¬«      D ]£  }|j                  dk(  rŒ|j                  dd¬«      }|rt        |«      dkD  r|j                  |«       |j                  dk(  sŒW|j                  d «      D ]9  }|j                  d!d«      }|sŒ|j                  d"t        t         |«      d#œ«       Œ; Œ¥ |rd$j#                  |«      nd}n|rt%        |«      nd}|so|r|j                  d%«      nd}|rX|j                  d!d«      }|rD|j'                  d&«      st        t         |«      }|j                  d'|d#œ«       d(t        |«      › d)�}|s|rd*t        |«      › d+�}n%|r#|j'                  d,«      s|d-t        |«      › d+�z  }||||fS ).u   æ��å�–è¯¦æƒ…é¡µæ­£æ–‡r=   zhtml.parserÚmetar@   ÚArticleTitle)ÚattrsÚcontentr$   Úcon_tit)Úclass_T)rT   u   ^Â·\s*ÚpubdateNé
   zt(\d{8})r   é   ú-é   é   z/(20\d{2})/(\d{1,2})/(\d{1,2})/rf   Ú02drx   ÚZoom)Úid)r   r$   r*   )Ú	recursiver   ÚimgÚsrcu   æ­£æ–‡é™„å›¾)r@   ro   rO   Úiframer„   u   æ­£æ–‡é™„ä»¶(PDF)u"   [æœ¬å…¬å‘Šæ­£æ–‡ä¸ºPDFé™„ä»¶ï¼Œå…±u(   ä¸ªé™„ä»¶ï¼Œè¯·ç‚¹å‡»é™„ä»¶æŸ¥çœ‹å…¨æ–‡]u"   [æœ¬å…¬å‘Šä¸ºå›¾ç‰‡æ ¼å¼�ï¼Œé™„ä»¶u   å¼ ]u   [æœ¬å…¬å‘Šæ­£æ–‡ä¸ºPDFu   

[å…¬å‘Šé™„å›¾)rt   r   Úfindri   rT   Úget_textrP   rS   r‰   rŠ   ÚintÚfind_allr@   ÚlenrE   r   rŒ   rR   rX   r‹   )Ú
detail_urlÚ
list_titleÚhtmlÚsoupÚ
full_titleÚ
meta_titleÚtit_divrW   r•   Ú	meta_dateÚm_uÚm_u2Úzoomr›   ÚattachmentsÚpartsÚchildrk   r¨   r©   rª   s                        rM   Úfetch_detailr¾   Š   s\  € ä�Ó€DÙØ˜2˜z¨2Ð-Ð-Ü˜˜}Ó-€Dð €JØ—‘˜6¨&°.Ð)A�ÓB€JÙ�j—n‘n YÔ/Ø 	Ñ*×0Ñ0Ó2‰
à—)‘)˜E¨)�)Ó4ˆÙØ× Ñ  ¨4Ð Ó0ˆAÙØ�
Ü—‘˜	 2 zÓ2×8Ñ8Ó:€Jð €HØ—	‘	˜&¨°Ð(;�	Ó<€IÙ�Y—]‘] 9Ô-Ø˜YÑ'¨¨Ð,ˆáÜ�i‰i˜ ZÓ0ˆÙØŸ)™) A›, r¨Ð*Ð+¨1¨S¯Y©Y°q«\¸!¸AÐ->Ð,?¸qÀÇÁÈ1ÃÈaÐPQÐARÐ@SÐT‰Hä—9‘9Ð?ÀÓLˆDÙØ"Ÿj™j¨›m˜_¨A¬c°$·*±*¸Q³-Ó.@ÀÐ-EÀQÄsÈ4Ï:É:ÐVWË=ÓGYÐZ]ÐF^Ð_�ð �9‰9�U˜vˆ9Ó&€DØ€GØ€KÙØˆØ—]‘]Ò#8ÀD�]ÓIò 	`ˆEà�z‰z˜VÒ#ØØ—>‘> "¨D�>Ó1ˆDÙœ˜D›	 AšØ—‘˜TÔ"à�z‰z˜SÓ Ø Ÿ>™>¨%Ó0ò `�CØŸ'™' %¨Ó,�CÚØ#×*Ñ*°NÌ7ÔSWÐY\ÓK]Ñ+^Õ_ñ`ð	`ñ ).�&—+‘+˜eÔ$°2‰á%)”)˜D”/¨rˆñ Ù(,�—‘˜8Ô$°$ˆÙØ—*‘*˜U BÓ'ˆCÙØ—~‘~ fÔ-Ü!¤$¨Ó,�CØ×"Ñ"Ð,?ÈÑ#LÔMØ>¼sÀ;Ó?OÐ>PÐPxÐy�ñ ‘{Ø6´s¸;Ó7GÐ6HÈÐM‰Ù	˜W×/Ñ/Ð0HÔIØÐ&¤s¨;Ó'7Ð&8¸Ð=Ñ=ˆà�w ¨+Ð5Ð5rN   c           
      óX  — 	 t        j                  t        d¬«      }| j                  d«      xs dd d }| j                  d«      xs dd d }| j                  d«      xs d}| j                  d	«      xs dd d
 }| j                  d«      xs dd d }t	        j
                  | j                  d«      xs g d¬«      }|j                  d|f«      j                  «       }|D ]  \  }	|j                  d|	f«       Œ |j                  d|||||||f«       |j                  d«      j                  «       d   }
|r|d d n|d d }|j                  d|
|||f«       |j                  «        |j                  «        t        d|d d › �«       y# t        $ r}t        d|› �«       Y d }~yd }~ww xY w)NrŸ   rc   r…   r=   iô  Úpage_urliè  r›   Úpublish_dateé   Ú	site_nameÚunknownéd   r»   F)Úensure_asciiz,SELECT rowid FROM gov_raw WHERE page_url = ?z&DELETE FROM gov_search WHERE rowid = ?zšINSERT OR REPLACE INTO gov_raw (title, page_url, content, publish_date, site_name, source_url, status, attachments) VALUES (?, ?, ?, ?, ?, ?, 'synced', ?)zSELECT last_insert_rowid()r   zWINSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)u     âœ… rb   Tz  [ERR] DB import failed: )Úsqlite3ÚconnectÚDB_PATHri   ÚjsonÚdumpsÚexecuteÚfetchallÚfetchoneÚcommitÚcloser`   rl   )ÚrecordÚdbr…   rÀ   r›   rÁ   rÃ   Úattachments_strÚ
old_rowidsÚridÚ	new_rowidÚsummaryrs   s                rM   Úimport_to_dbrØ   Ø   sµ  € ðÜ�_‰_œW¨bÔ1ˆØ—‘˜GÓ$Ò*¨¨D¨SÐ1ˆØ—J‘J˜zÓ*Ò0¨b°%°4Ð8ˆØ—*‘*˜YÓ'Ò-¨2ˆØŸ
™
 >Ó2Ò8°b¸#¸2Ð>ˆØ—Z‘Z Ó,Ò9°	¸4¸CÐ@ˆ	ÜŸ*™* V§Z¡Z°Ó%>Ò%DÀ"ÐSXÔYˆà—Z‘ZÐ NÐQYÐP[Ó\×eÑeÓgˆ
Ø ò 	I‰FˆSØ�J‰JÐ?À#ÀÕHð	Ið 	�
‰
ð iØ�H˜g |°YÀÈ/ÐZô	
ð —J‘JÐ;Ó<×EÑEÓGÈÑJˆ	Ù#*�'˜$˜3‘-°°d°s°ˆØ
�
‰
ÐlØ˜u i°Ð9ô	;à
�	‰	ŒØ
�‰Œ
Üˆf�U˜3˜B�Z�LÐ!Ô"ØøÜò ÜÐ(¨¨Ð,Ô-Üûðús   ‚FF Æ	F)ÆF$Æ$F)c                  óH  — dd l } | j                  «       }|j                  dt        dd¬«       |j	                  «       }|j
                  t        z  }|t        z   dz
  t        z  }t        d|j
                  › d|› d	|› d
�«       g }t        d«       t        t        «      }|rjt        j                  d|t        j                  «      }|rCt        |j                  d«      «      }t        dt!        |«      › d�«       |j#                  |«       t%        d|«      D ]¨  }	|	t        z  dz   }
|
t        z   dz
  }|
dkD  r n‰t        d|	dz   › d|
› d|› �«       t'        |
|«      }|s nbt        |«      }t        dt!        |«      › d�«       |j#                  |«       t!        |«      t        dz  k  r nt)        j*                  d«       Œª |d | }t        dt!        |«      › d�«       d}t-        |d«      D ]�  \  }}|d   }|d   }|d   }t        d|› dt!        |«      › d|j/                  d«      d   xs |dd  › �«       t1        ||«      \  }}}}|||xs |||t2        t4        t6        d œ}t9        |«      }|r|dz  }t)        j*                  d«       ŒŸ t        d!t2        › d"|› d�«       y )#Nr   z--pagesé   u1   çˆ¬å�–å‰�Né¡µï¼ˆæ¯�é¡µ15æ�¡ï¼Œé»˜è®¤5é¡µ=75æ�¡ï¼‰)ÚtypeÚdefaultÚhelpr   u   ðŸ“‹ ç›®æ ‡: u    é¡µ = u
    æ�¡, éœ€ u    ç»„u    ðŸ“„ èŽ·å�–é¦–é¡µåˆ�å§‹æ•°æ�®...z<datastore>(.*?)</datastore>u     â†’ é¦–é¡µèŽ·å�– u    æ�¡iÿ  u   ðŸ“„ JPageç»„ z
: records r¡   u     â†’ rf   g      à?u   
ðŸ“‹ å…±èŽ·å�– u    æ�¡å¾…çˆ¬å�–è¯¦æƒ…ro   r…   r†   z  (r	   u
   ) è¯¦æƒ…: éÿÿÿÿiâÿÿÿ)r…   rÀ   rÁ   r›   r»   rÃ   ÚcolumnÚprovinceu   
âœ… u    çˆ¬å�–å®Œæˆ�ï¼Œå…±å…¥åº“ )ÚargparseÚArgumentParserÚadd_argumentr­   Ú
parse_argsÚpagesr|   Ú
GROUP_SIZEr`   rt   r}   rP   r‰   rˆ   r–   rŠ   r¯   Úextendrg   r‚   rm   rn   Ú	enumerateÚsplitr¾   ÚSITEÚCOLUMNÚPROVINCErØ   )rá   ÚparserÚargsÚtotal_neededÚgroups_neededÚ	all_itemsr²   Ú	xml_matchrŽ   ÚgÚstartÚendr�   ÚcountÚidxÚitemro   r±   Ú	list_dater´   r›   Údetail_dater»   rÑ   Úoks                            rM   r(   r(   ÷   s»  € ÛØ×$Ñ$Ó&€FØ
×Ñ˜	¬°QØPð ô Rà×ÑÓ€Dà—:‘:¤Ñ(€LØ!¤JÑ.°Ñ2´zÑA€Mäˆ-˜Ÿ
™
�| 7¨<¨.¸
À=À/ÐQUÐVÔWà€Iô Ð
*Ô,Ü”‹?€DÙÜ—I‘IÐ=¸tÄRÇYÁYÓOˆ	ÙÜ" 9§?¡?°1Ó#5Ó6ˆEÜÐ%¤c¨%£j \°Ð6Ô7Ø×Ñ˜UÔ#ô �1�mÓ$ò ˆØ”J‘ Ñ"ˆØ”jÑ  1Ñ$ˆØ�4Š<ÙÜˆn˜Q˜q™S˜E ¨E¨7°!°C°5Ð9Ô:Ü˜u cÓ*ˆÙÙÜ˜xÓ(ˆÜˆf”S˜“Z�L Ð%Ô&Ø×Ñ˜ÔÜˆu‹:œ
 a™Ò'ÙÜ�
‰
�3�ðð" ˜-˜<Ð(€IÜÐ
œC 	›NÐ+Ð+>Ð?Ô@à€EÜ˜y¨!Ó,ò ‰	ˆˆTØ�5‰kˆØ˜'‘]ˆ
Ø˜‘Lˆ	Üˆc�#��aœ˜I›Ð' z°#·)±)¸C³.ÀÑ2DÒ2QÈÈCÈDÈ	Ð1RÐSÔTÜ8DÀSÈ*Ó8UÑ5ˆ
�G˜[¨+ð  ØØ'Ò4¨9ØØ&ÜÜÜ ñ	
ˆô ˜&Ó!ˆÙØ�Q‰JˆEÜ�
‰
�1�ð)ô, ˆ&”�Ð0°°°tÐ<Õ=rN   Ú__main__)rx   )+Ú__doc__r]   ÚosrP   rÊ   rm   rÇ   ÚrequestsrQ   r   Úurllib.parser   Úurllib3Údisable_warningsÚ
exceptionsÚInsecureRequestWarningrŒ   r}   rê   rë   rì   r|   ræ   ÚgetenvrÉ   ÚHEADERSÚSessionrh   rz   ÚupdateÚverifyr   r{   rH   rX   r`   rt   r‚   r–   r¾   rØ   r(   Ú__name__© rN   rM   ú<module>r     s&  ðñ÷ (× '× 'Û Ý Ý  Û Ø €× Ñ ˜×+Ñ+×BÑBÔ Cà#€ØÐ;Ñ;€Ø4€Ø	 €Ø€Ø€Ø€
à
ˆ"�)‰)�KÐ!2Ó
3€àÐWÐ
X€Ø
ˆ(×
Ñ
Ó
€Ø ‡�× Ñ �wÔ Ø€„ð ØØ�3‰JØØØØ&Øñ	€ð Ð4Ñ4€	ò*€òò6,ó	òò ò,K6ò\ò>D>ðN ˆzÒÙ…Fð rN   