Ë
    ¸¬uj"  ã                   óä   — d Z ddlZddlZddlZddlZddlZddlmZ ddlZ ej                  dd«      Z
dZe› d�Ze› d�Zd	d
dddddœZdZdZdZdddœZdd„Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u¬   æœˆæ¹–åŒº-å…¬å‘Šå…¬ç¤ºçˆ¬è™«
ç«™ç‚¹: http://www.yuehu.gov.cn
æ �ç›®: å…¬å‘Šå…¬ç¤º (col3794)
åˆ†é¡µ: JPage POST /module/web/jpage/dataproxy.jsp, 45æ�¡/é¡µ
CMS: Hanweb JPage
é    N)ÚBeautifulSoupÚ	SEARCH_DBz/mnt/data/search.dbzhttp://www.yuehu.gov.cnz/col/col3794/index.htmlz/module/web/jpage/dataproxy.jspÚ34zhttp://www.yuehu.gov.cn/Ú3794Ú35403u   æœˆæ¹–åŒºäººæ°‘æ”¿åºœç½‘ç«™Ú0)ÚwebidÚpathÚcolumnidÚunitidÚwebnameÚpermissiontypeu   æœˆæ¹–åŒº-å…¬å‘Šå…¬ç¤ºu   æ”¿åºœå…¬å‘Šu   æ±Ÿè¥¿z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36zzh-CN,zh;q=0.9)z
User-AgentzAccept-Languagec                 ó|  — t        |«      D ]S  }	 |rt        j                  | |t        d¬«      }nt        j                  | t        d¬«      }d|_        |j                  c S  y # t        $ rO}||dz
  k  rt        j                  d«       n't        d| › d|› �t        j                  ¬	«       Y d }~ y Y d }~Œªd }~ww xY w)
Né   )ÚdataÚheadersÚtimeout)r   r   zutf-8é   é   z
  [ERROR] z: ©Úfile)ÚrangeÚrequestsÚpostÚHEADERSÚgetÚencodingÚtextÚ	ExceptionÚtimeÚsleepÚprintÚsysÚstderr)ÚurlÚretriesÚ	post_dataÚiÚrÚes         ú%/root/gov_crawler/crawl_yuehu_gggs.pyÚfetchr,   #   sŸ   € Ü�7‹^ò ˆð	ÙÜ—M‘M #¨I¼wÐPRÔS‘ä—L‘L ¬g¸rÔB�Ø ˆAŒJØ—6‘6ŠMñøô ò 	Ø�7˜Q‘;ŠÜ—
‘
˜1•ä˜
 3 % r¨!¨Ð-´C·J±JÕ?Ýô ûð	ús   �AA#Á#	B;Á,?B6Â6B;c                 ó$  — g }t        j                  d| t         j                  «      }|D ]ã  }t        j                  d|«      }|sŒ|j	                  d«      j                  «       }|j	                  d«      j                  «       }|r|rd|vrŒc|j                  d«      s	t        |z   }|j                  d«      }|j                  d«      s|dz  }d	}t        j                  d
|«      }|r|j	                  d«      }|j                  |||dœ«       Œå |S )u    ä»ŽHTML/APIå“�åº”ä¸­æ��å�–è®°å½•z<record>(.*?)</record>zhref="([^"]*)"[^>]*>([^<]*)</a>r   r   Úart_Úhttpú.z.htmlÚ z(\d{4}-\d{1,2}-\d{1,2}))Úpage_urlÚtitleÚdate)ÚreÚfindallÚDOTALLÚsearchÚgroupÚstripÚ
startswithÚBASE_URLÚrstripÚendswithÚappend)	ÚhtmlÚitemsÚrecordsÚrecÚmÚhrefr3   Údate_strÚdms	            r+   Úparse_recordsrH   4   s   € à€Eä�j‰jÐ2°D¼"¿)¹)ÓD€GØò ˆä�I‰IÐ8¸#Ó>ˆÙØØ�w‰w�q‹z×ÑÓ!ˆØ—‘˜“
× Ñ Ó"ˆÙ™5 F°$Ñ$6ØØ�‰˜vÔ&Ü˜d‘?ˆDà�{‰{˜3ÓˆØ�}‰}˜WÔ%Ø�G‰OˆDð ˆÜ�Y‰YÐ1°3Ó7ˆÙØ—x‘x “{ˆHà�‰ØØØñ
õ 	ð-ð6 €Ló    c                 óˆ  — t        | d«      }d}d}d}|j                  d«      }|r>|j                  d¬«      }dD ]#  }|j                  |«      sŒ|t	        |«      d } n |r|}|j                  «       }	t        j                  d|	«      }
|
r|
j                  d	«      }|j                  d
«      }|r|j                  d«      nd}|s|j                  d«      }|rgg }|j                  dd¬«      D ]:  }|j                  d«      rŒ|j                  d¬«      }|sŒ*|j                  |«       Œ< |rdj                  |«      nd}|||fS )u   è§£æž�è¯¦æƒ…é¡µzhtml.parserr1   r3   T)r:   )u)   æœˆæ¹–åŒºäººæ°‘æ”¿åºœç½‘ç«™ å…¬å‘Šå…¬ç¤º u(   æœˆæ¹–åŒºäººæ°‘æ”¿åºœç½‘ç«™ å…¬å‘Šå…¬ç¤ºNu-   å�‘å¸ƒæ—¶é—´[ï¼š:]?\s*(\d{4}-\d{1,2}-\d{1,2})r   zdiv.bfr_article_contentzdiv.captionÚp)Ú	recursiveÚtablez

)r   ÚfindÚget_textr;   Úlenr5   r8   r9   Ú
select_oneÚfind_allÚfind_parentr?   Újoin)r@   r%   Úsoupr3   Úpublish_dateÚcontentÚ	title_tagÚtÚprefixÚ	body_textrG   Úcontent_divÚcaptionÚ
paragraphsrK   r   s                   r+   Úparse_detailr_   W   sV  € ä˜˜}Ó-€DØ€EØ€LØ€Gð —	‘	˜'Ó"€IÙØ×Ñ TÐÓ*ˆàoò 	ˆFØ�|‰|˜FÕ#Ø”c˜&“k�l�O�Ùð	ñ ØˆEð —‘“€IÜ	�‰ÐCÀYÓ	O€BÙ	Ø—x‘x “{ˆð —/‘/Ð";Ó<€KÙ7Bˆk×$Ñ$ ]Ô3È€GÙØ—/‘/ -Ó0ˆÙØˆ
Ø×!Ñ! #°Ð!Ó6ò 	(ˆAØ�}‰}˜WÔ%ØØ—:‘: D�:Ó)ˆDÚØ×!Ñ! $Õ'ð	(ñ .8�&—+‘+˜jÔ)¸Rˆà�, Ð'Ð'rI   c                 óV  — d„ }d || «      › d�}t        j                  dt        |gddd¬«      }|j                  j	                  «       ry|r|d d	 nd
}d || «      › d ||«      › d ||«      › d ||«      › d |t
        «      › dt        › d ||«      › d�}t        j                  dt        |gddd¬«      }|j                  dk7  r7d|j                  vr)t        d|j                  › �t        j                  ¬«       yd || «      › d�}	t        j                  dt        |	gddd¬«       y)Nc                 ó.   — | r| j                  dd«      S dS )Nú'z''r1   )Úreplace)Úss    r+   Úesczsave_to_db.<locals>.escƒ   s   € Ù'(ˆq�y‰y˜˜dÓ#Ð0¨bÐ0rI   z,SELECT rowid FROM gov_raw WHERE page_url = 'rb   Úsqlite3Té
   )Úcapture_outputr   r   ÚdupéÈ   r1   zdINSERT INTO gov_raw (page_url, title, publish_date, content, site_name, industry, summary)
VALUES ('z', 'z')r   ÚUNIQUEz  [DB ERROR] r   Úerrorz�INSERT INTO gov_search (rowid, title, site_name, summary)
SELECT rowid, title, site_name, summary FROM gov_raw
WHERE page_url = 'z1' AND rowid NOT IN (SELECT rowid FROM gov_search)Únew)Ú
subprocessÚrunÚDB_PATHÚstdoutr:   Ú	SITE_NAMEÚINDUSTRYÚ
returncoder$   r"   r#   )
r2   r3   rV   rW   re   Ú	check_sqlÚresultÚsummaryÚsqlÚsync_sqls
             r+   Ú
save_to_dbrz   ‚   sR  € ò1à>¹sÀ8»}¸oÈQÐO€IÜ�^‰^˜Y¬°Ð;ÈDÐW[ÐegÔh€FØ‡}�}×ÑÔØÙ&ˆg�d�s‰m¨B€Gð
Ù
ˆh‹-ˆ˜™S ›Z˜L¨©S°Ó->Ð,?¸tÁCÈÃLÀ>ÐQUÑVYÔZcÓVdÐUeÐeiÔjrÐisÐswÑx{ð  }Dó  yEð  xFð  FHðK€Cä�^‰^˜Y¬°Ð5ÀdÐQUÐ_aÔb€FØ×Ñ˜AÒ (°&·-±-Ñ"?Ü�˜fŸm™m˜_Ð-´C·J±JÕ?Øðá�x“=�/Ð!RðV€Hô ‡N�N�Iœw¨Ð1À$ÈTÐ[]Õ^ØrI   c                  óX  — t        t        j                  «      dkD  rt        j                  d   nd} 	 t        | «      }t        dt        › d|› �d¬«       g }t        t        «      }|r6t        |«      }t        dt        |«      › d	�d¬«       |j                  |«       t        d
|dz   «      D ]·  }t        t        t        |«      ¬«      }t        t        |¬«      }|s nˆt        |«      }|s ny|D �ch c]  }|d   ’Œ	 }	}|D �cg c]  }|d   |	vsŒ|‘Œ }
}t        d|› dt        |«      › dt        |
«      › d�d¬«       |
s n#|j                  |
«       t        |«      dk  sŒ· n t        dt        |«      › �d¬«       dx}x}}|D ]Ÿ  }t        d|d   d d › d�d¬«       t        |d   «      }|s|dz  }Œ0t!        ||d   «      \  }}}|s|j#                  dd«      }|s|j#                  dd«      }|s|dz  }Œs	 t%        |d   |||«      }|dk(  r|dz  }n|dk(  r|dz  }n|dz  }Œ¡ t        dt        › d �d¬«       t        d!|› d"|› d#|› �d¬«       y # t        $ r d}Y �Œ&w xY wc c}w c c}w # t&        $ r.}t        d|› �t        j(                  ¬«       |dz  }Y d }~�Œd }~ww xY w)$Nr   Ú3é   ú[z] Starting crawl, max_pages=T)Úflushz  Page 1 (HTML): z itemsr   )Úpage)r'   r2   z  Page z (API): z items, z newr   zTotal list items: r   z  Fetching: r3   é(   z...r1   r4   rm   ri   z  [ERROR] DB: r   z
=== z	 Done ===zNew: z, Dup: z	, Error: )rP   r#   ÚargvÚintÚ
ValueErrorr"   rr   r,   ÚLIST_URLrH   Úextendr   ÚdictÚJPPARAMSÚstrÚAPI_URLr_   r   rz   r   r$   )Úmax_pages_strÚ	max_pagesÚ	all_itemsr@   rA   r€   r'   Úapi_htmlr(   Úexisting_urlsÚ	new_itemsÚ	new_countÚ	dup_countÚerror_countÚitemÚdetail_htmlr3   rV   rW   rv   r*   s                        r+   Úcrawlr–   —   sä  € Ü#&¤s§x¡x£=°1Ò#4”C—H‘H˜Q’K¸#€MðÜ˜Ó&ˆ	ô 
ˆAŒiˆ[Ð4°Y°KÐ
@ÈÕMØ€Iô ”‹?€DÙÜ˜dÓ#ˆÜÐ!¤# e£* ¨VÐ4¸DÕAØ×Ñ˜Ôô �a˜ Q™Ó'ò ˆÜœ¬¨D«	Ô2ˆ	Üœ¨IÔ6ˆÙÙÜ˜hÓ'ˆÙÙà09Ö:¨1˜˜:›Ð:ˆÐ:Ø %ÖL˜1¨¨:©¸mÒ)K’QÐLˆ	ÐLÜ�˜�v˜X¤c¨%£j \°¼#¸i».Ð9IÈÐNÐVZÕ[ÙÙØ×Ñ˜Ô#Üˆu‹:˜‹?Ùð!ô$ 
Ðœs 9›~Ð.Ð
/°tÕ<à*+Ð+€IÐ+�	˜KØò ˆÜ�˜T '™]¨3¨BÐ/Ð0°Ð4¸DÕAÜ˜D Ñ,Ó-ˆÙØ˜1ÑˆKØÜ'3°KÀÀjÑAQÓ'RÑ$ˆˆ|˜WÙØ—H‘H˜W bÓ)ˆEÙØŸ8™8 F¨BÓ/ˆLÙØ˜1ÑˆKØð
	Ü  ZÑ 0°%¸ÀwÓOˆFØ˜ŠØ˜Q‘‘	Ø˜5’Ø˜Q‘‘	à˜qÑ �øð+ô4 
ˆF”9�+˜YÐ
'¨tÕ4Ü	ˆE�)�˜G I ;¨i¸°}Ð
EÈTÖRøô} ò Ø‹	ðüò. ;ùÚLøôB ò 	Ü�N 1 #Ð&¬S¯Z©ZÕ8Ø˜1ÑŽKûð	ús;   ³I Ã2I(ÄI-ÄI-Ç<,I2ÉI%É$I%É2	J)É;#J$Ê$J)Ú__main__)r}   N)Ú__doc__Úosr5   r#   r    rn   Úbs4r   r   Úgetenvrp   r<   r…   rŠ   rˆ   rr   rs   ÚGROUPr   r,   rH   r_   rz   r–   Ú__name__© rI   r+   ú<module>rŸ      s¹   ðñ÷ %× $Ó $Ý Û à
ˆ"�)‰)�KÐ!6Ó
7€Ø$€ØˆZÐ.Ð/€ØˆJÐ5Ð
6€àØ&ØØØ,Øñ€ð %€	Ø€Ø€ð QØ'ñ€óò" òF((òVò*BSðJ ˆzÒÙ	…Gð rI   