Ë
    »À£j¼  ã                   ó„   — d Z ddlZddlZddlZddlZddlmZ dZdZdZ	dZ
dd	iZd
„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uÇ   
crawl_soochowchem.py - å®�å¤�ä¸œå�´å†œåŒ–è‚¡ä»½æœ‰é™�å…¬å�¸ æ–°é—»/çŽ¯è¯„å…¬ç¤º
URL: http://www.soochowchem.com/news.html
ç»“æž„: å�•é¡µ46æ�¡åˆ—è¡¨ï¼Œ/news_detail/id/N.html è¯¦æƒ…ï¼Œæ­£æ–‡åœ¨ td.zw
é    N)ÚBeautifulSoupz/root/search.dbu   å®�å¤�ä¸œå�´å†œåŒ–-æ–°é—»å…¬ç¤ºu   çŽ¯å¢ƒå…¬ç¤ºzhttp://www.soochowchem.comz
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36c                 ó.   — | xs dj                  dd«      S )zEscape single quotes for SQLiteÚ ú'z'')Úreplace)Úss    úcrawl_soochowchem.pyÚescr
      s   € àŠG�×Ñ˜S $Ó'Ð'ó    c                 óx   — t        j                  d| «      }|r!|j                  d«      j                  dd«      S dS )Nz(\d{4}[-/]\d{1,2}[-/]\d{1,2})é   ú/ú-r   )ÚreÚsearchÚgroupr   )ÚtextÚms     r	   Ú
parse_dater      s5   € Ü
�	‰	Ð2°DÓ9€AÙ+,ˆ1�7‰7�1‹:×Ñ˜c 3Ó'Ð4°"Ð4r   c                 óp  — 	 t        j                  | dt        ¬«      }d|_        |j                  dk7  ry	 t        |j                  d«      }|j                  d«      }|r|j                  d¬«      nd}t        |j                  «      }|j                  dd¬«      }|s|j                  d¬«      }|rt        |d„ ¬«      }n#|j                  d«      }	|	rt        |	d„ ¬«      nd
}d}
|rú|j                  d«      }|rQg }|D ]7  }|j                  d¬«      }|sŒt        |«      dkD  sŒ'|j                  |«       Œ9 |rdj!                  |«      }
|
s9|j                  d«      D ]  }|j#                  d«       Œ |j                  d¬«      }
|r-|
j%                  |«      r|
t        |«      d
 j'                  «       }
d|› �}||
v r#|
j)                  |d«      d   j'                  «       }
|s'|j                  d«      }|r|j                  d¬«      nd}||
|fS # t
        $ r}t        d| › d|› �d¬	«       Y d
}~yd
}~ww xY w)z8Fetch detail page, return (title, content, publish_date)é   ©ÚtimeoutÚheadersúutf-8éÈ   )NNNz  [ERR] z: T©ÚflushNúhtml.parserÚh1©Ústripr   ÚtdÚzw)Úclass_c                 ó8   — t        | j                  d¬«      «      S ©NTr!   ©ÚlenÚget_text©Úxs    r	   ú<lambda>zfetch_detail.<locals>.<lambda>3   ó   € ¤C¨¯
©
¸¨
Ó(>Ó$?€ r   )Úkeyc                 ó8   — t        | j                  d¬«      «      S r'   r(   r+   s    r	   r-   zfetch_detail.<locals>.<lambda>7   r.   r   Úpé   z

Úbrú
u   å�‘å¸ƒæ—¶é—´ï¼šr   éÿÿÿÿÚtitle)ÚrequestsÚgetÚHEADERSÚencodingÚstatus_codeÚ	ExceptionÚprintr   r   Úfindr*   r   Úfind_allÚmaxr)   ÚappendÚjoinÚreplace_withÚ
startswithr"   Úsplit)ÚurlÚrÚeÚsoupr    r6   Úpub_dateÚzwsr$   ÚtdsÚcontentÚpsÚ
paragraphsr1   Útxtr3   Údate_prefixÚ	title_tags                     r	   Úfetch_detailrS      s,  € ð Ü�L‰L˜ b´'Ô:ˆØˆŒ
Ø�=‰=˜CÒØ#ð  ô ˜Ÿ™ Ó/€DØ	�‰�4‹€BÙ')ˆB�K‰K˜dˆKÔ#¨r€Eä˜!Ÿ&™&Ó!€Hà
�-‰-˜ Tˆ-Ó
*€CÙØ�m‰m 4ˆmÓ(ˆÙ
ä�Ñ?Ô@‰ð �m‰m˜DÓ!ˆÙDGŒS�Ñ?Õ@ÈTˆà€GÙ	ð �[‰[˜ÓˆÙØˆJØò +�Ø—j‘j t�jÓ,�Úœ3˜s›8 a›<Ø×%Ñ% cÕ*ð+ñ Ø Ÿ+™+ jÓ1�ÙØ—k‘k $Ó'ò &�Ø—‘ Õ%ð&à—k‘k¨�kÓ-ˆGá�W×'Ñ'¨Ô.Øœc %›j˜kÐ*×0Ñ0Ó2ˆGØ'¨ zÐ2ˆØ˜'Ñ!Ø—m‘m K°Ó3°BÑ7×=Ñ=Ó?ˆGáØ—I‘I˜gÓ&ˆ	Ù2;�	×"Ñ"¨Ð"Ô.Àˆà�'˜8Ð#Ð#øôc ò  Ü�˜˜˜R ˜sÐ#¨4Õ0Üûð ús   ‚2H È	H5ÈH0È0H5c                 óÔ  — |r|dd nd}dt        | «      › dt        |«      › dt        |«      › dt        |«      › dt        t        «      › dt        › dt        |«      › d�}	 t        j                  ddd	t
        |gd
d
d¬«      }|j                  dk7  r7d|j                  vr)t        d|j                  › �t        j                  ¬«       ydt        | «      › d�}	 t        j                  ddd	t
        |gd
d
d¬«       y
# t        j                  $ r! t        d| › �t        j                  ¬«       Y yw xY w# t        j                  $ r! t        d| › �t        j                  ¬«       Y y
w xY w)z%Insert into search.db via sqlite3 CLINr   r   zgINSERT INTO gov_raw (page_url, title, publish_date, content, site_name, industry, summary)
VALUES (
  'z',
  'z'
)Úsqlite3z-cmdz.timeout 60000TéZ   )Úcapture_outputr   r   z  [DB TIMEOUT] ©ÚfileFr   ÚUNIQUEz  [DB ERROR] z�INSERT INTO gov_search (rowid, title, site_name, summary)
SELECT rowid, title, site_name, summary
FROM gov_raw WHERE page_url = 'z1' AND rowid NOT IN (SELECT rowid FROM gov_search)z  [FTS TIMEOUT] )r
   Ú	SITE_NAMEÚINDUSTRYÚ
subprocessÚrunÚDB_PATHÚTimeoutExpiredr=   ÚsysÚstderrÚ
returncode)Úpage_urlr6   rM   rJ   ÚsummaryÚsqlÚresultÚfts_sqls           r	   Ú
insert_oneri   X   s  € á&ˆg�d�s‰m¨B€GðäˆƒM€?ð ÜˆƒJ€<ð ÜˆƒM€?ð ÜˆƒL€>ð ÜŒ	ƒNÐð Ü€:ð ÜˆƒL€>ð ð	€CðÜ—‘Ø˜Ð 0´'¸3Ð?Ø d°Bô
ˆð ×Ñ˜AÒ (°&·-±-Ñ"?Ü�˜fŸm™m˜_Ð-´C·J±JÕ?Øð ä # H£˜Ð._ðc€Gð>Ü�‰Ø˜Ð 0´'¸7ÐCØ d°Bõ	
ð øô% ×$Ñ$ò Ü� ˜zÐ*´·±Õ<Ùðûô  ×$Ñ$ò >ÜÐ   
Ð+´#·*±*Ö=Øð>ús$   Á!"C< Ã"D3 Ã<1D0Ä/D0Ä31E'Å&E'c                  óÈ  — d} t        t        j                  «      dkD  r	 t        t        j                  d   «      } t        dt        › d�d¬«       t        › d�}t        j                  |dt        ¬	«      }d
|_        |j                  dk7  r)t        d|j                  › �t        j                  ¬«       y t        |j                  d«      }|j!                  dd¬«      }g }|D ]N  }|d   }d|v sŒ|j#                  d¬«      }|j%                  d«      r|n	t        › |› �}	|j'                  |	|f«       ŒP t        dt        |«      › d�d¬«       d}
d}d}t)        |«      D ]\  \  }\  }}t        d|dz   › dt        |«      › d|d d › d�d¬«       t+        |«      \  }}}|xs |}t-        ||||«      r|
dz  }
ŒX|dz  }Œ^ t        d|
› d|› d|› �d¬«       y # t        $ r Y �Œ¤w xY w) Nr2   r   ú[z] Starting crawl...Tr   z
/news.htmlr   r   r   r   z[ERR] List page HTTP rX   r   Úa)Úhrefrm   Únews_detailr!   ÚhttpzFound z items on list pager   z  [r   z] é2   z...z
Done! New: z, Duplicates: z
, Errors: )r)   ra   ÚargvÚintÚ
ValueErrorr=   r[   ÚBASE_URLr7   r8   r9   r:   r;   rb   r   r   r?   r*   rD   rA   Ú	enumeraterS   ri   )Ú	max_pagesrF   rG   rI   ÚlinksÚ
news_itemsrl   rm   r6   Úfull_urlÚ	new_countÚ	dup_countÚ	err_countÚiÚ
detail_urlÚ
list_titlerM   rJ   Úfinal_titles                      r	   Úcrawlr�      s   € Ø€IÜ
Œ3�8‰8ƒ}�qÒð	ÜœCŸH™H Q™KÓ(ˆIô 
ˆAŒiˆ[Ð+Ð
,°DÕ9ô ˆJ�jÐ
!€CÜ�‰�S "¬gÔ6€AØ€A„JØ‡}�}˜ÒÜÐ% a§m¡m _Ð5¼C¿J¹JÕGØä˜Ÿ™ Ó/€DØ�M‰M˜# DˆMÓ)€EØ€JØò 1ˆØ�‰yˆØ˜DÒ Ø—J‘J T�JÓ*ˆEØ#Ÿ™¨vÔ6‘t¼x¸jÈÈÐ<OˆHØ×Ñ˜x¨Ð/Õ0ð1ô 
ˆF”3�z“?Ð#Ð#6Ð
7¸tÕDà€IØ€IØ€Iä'0°Ó'<ò Ñ#ˆÑ#ˆJ˜
Ü��A�a‘C�5˜œ#˜j›/Ð*¨"¨Z¸¸¨_Ð,=¸SÐAÈÕNÜ#/°
Ó#;Ñ ˆˆw˜ØÒ)˜zˆÜ�j +¨w¸ÔAØ˜‰N‰Ià˜‰N‰Iðô 
ˆM˜)˜ N°9°+¸ZÈ	À{Ð
SÐ[_Ö`øôM ò 	Úð	ús    G Ç	G!Ç G!Ú__main__)Ú__doc__ra   r   r7   r]   Úbs4r   r_   r[   r\   rt   r9   r
   r   rS   ri   r�   Ú__name__© r   r	   ú<module>r‡      sg   ðñ÷
 %× $Ý à
€Ø-€	Ø€Ø'€àÐPð€ò
(ò
5ò
8$òv$òN+að\ ˆzÒÙ	…Gð r   