Ë
    ¸¬uj`#  ã                   óÜ  — d Z ddlZddlZddlZddlZddlZddlZddlmZ ddl	m	Z	m
Z
  ej                  dd«      ZdZdZd	Zd
ZdZdZ e	j&                  «        e
d¬«      z
  j)                  d«      ZddddœZd„ Zd„ Zdd„Zdd„Zd„ Zd„ Zd d„Zedk(  rXdej>                  v Z dZ!ej>                  D ].  Z"e"jG                  d«      sŒ e$e"jK                  d«      d   «      Z!Œ0  ee e!¬«       yy)!uµ   
crawl_hf_gov.py -- å�ˆè‚¥å¸‚äººæ°‘æ”¿åºœ-çŽ¯è¯„å…¬ç¤º
Site: https://www.hefei.gov.cn/public/column/19081?type=4&catId=6999451&action=list
CMS: Custom, with CloudFlare JS challenge
é    N)ÚBeautifulSoup)ÚdatetimeÚ	timedeltaÚ	SEARCH_DBz/root/search.dbu"   å�ˆè‚¥å¸‚äººæ°‘æ”¿åºœ-çŽ¯è¯„å…¬ç¤ºzwww.hefei.gov.cnzhttps://www.hefei.gov.cnz5/public/column/19081?type=4&catId=6999451&action=listz/tmp/solve_hf_jsl.jsÚnodeiG  )Údaysz%Y-%m-%d)ÚnewÚskipÚerrorsc                  ó2  — 	 t        j                  t        t        gddd¬«      } | j                  j                  «       }t        j                  |«      }d|v rt        d|d   › �«       y|d   |d   d	œS # t        $ r}t        d
|› �«       Y d}~yd}~ww xY w)z7Run Node.js to solve JS challenge, returns cookies dictTé   ©Úcapture_outputÚtextÚtimeoutÚerrorz  [ERROR] JSL solve: NÚjsluidÚ	clearance©Ú
__jsluid_sÚ__jsl_clearance_sz  [ERROR] JSL: )
Ú
subprocessÚrunÚNODE_BINÚ
JSL_SCRIPTÚstdoutÚstripÚjsonÚloadsÚprintÚ	Exception)ÚresultÚoutÚdataÚes       ú!/root/gov_crawler/crawl_hf_gov.pyÚ	solve_jslr'      sœ   € ðÜ—‘¤¬:Ð 6ÀtÐRVÐ`bÔcˆØ�m‰m×!Ñ!Ó#ˆÜ�z‰z˜#‹ˆØ�d‰?ÜÐ)¨$¨w©-¨Ð9Ô:Øà˜x™.Ø!% kÑ!2ñ
ð 	
øô ò Ü� ˜sÐ#Ô$Üûðús   ‚A'A5 Á*
A5 Á5	BÁ>BÂBc                 óö   — 	 t        j                  t        t        | gddd¬«      }|j                  j                  «       }t        j                  |«      }d|v ry|d   |d   dœS # t        $ r
}Y d}~yd}~ww xY w)	z*Solve JS challenge for a specific page URLTr   r   r   Nr   r   r   )	r   r   r   r   r   r   r   r   r!   )Úpage_urlr"   r#   r$   r%   s        r&   Úsolve_jsl_pager*   *   s|   € ðÜ—‘¤¬:°xÐ @ÐQUÐ\`ÐjlÔmˆØ�m‰m×!Ñ!Ó#ˆÜ�z‰z˜#‹ˆØ�d‰?Øà˜x™.Ø!% kÑ!2ñ
ð 	
øô ò Üûðús   ‚AA% Á
A% Á%	A8Á3A8c                 ó˜   — ddl }ddddœ}	 |j                  t        | z   ||xs i dd¬	«      }d
|_        |S # t        $ r
}Y d}~yd}~ww xY w)z!Fetch a URL with optional cookiesr   Nz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languager   F)ÚheadersÚcookiesr   Úverifyzutf-8)ÚrequestsÚgetÚBASE_URLÚencodingr!   )Úpathr.   Úreqr-   Úrr%   s         r&   Úfetch_with_cookiesr7   :   s[   € ãàTØSØ4ñ€Gð
Ø�G‰G”H˜t‘O¨W¸gºmÈÐUWÐ`eˆGÓfˆØˆŒ
ØˆøÜò Üûðús   Œ)6 ¶	A	ÁA	é   c                 ó–  ‡— | dk(  rt         }nd| › �}t        «       }|sg S t        ||«      }|r|j                  dk7  rg S t	        |j
                  d«      }g }h d£Šˆfd„}|j                  d«      D ]±  }|j                  d«      }|j                  d	«      }	|sŒ(|	sŒ+|j                  d
d«      }
|j                  dd«      xs |j                  d¬«      }|	j                  d¬«      } ||
||«      sŒ€|
j                  d«      s	t        |
z   }
|j                  ||
|dd dœ«       Œ³ |s×|j                  d«      D ]Ã  }|j                  d«      }t        |«      dk\  sŒ#|d   j                  d«      }|sŒ:|j                  d
d«      }
|j                  dd«      xs |j                  d¬«      }|d   j                  d¬«      } ||
||«      sŒ’|
j                  d«      s	t        |
z   }
|j                  ||
|dd dœ«       ŒÅ |sÈ|j                  d«      D ]´  }|j                  d«      }|j                  d	«      }|sŒ(|sŒ+|j                  d
d«      }
|j                  dd«      xs |j                  d¬«      }|d   j                  d¬«      } ||
||«      sŒƒ|
j                  d«      s	t        |
z   }
|j                  ||
|dd dœ«       Œ¶ |S )u8   Get list page. catId=6999451 for çŽ¯è¯„, type=4 for listr8   z;/public/column/19081?type=4&catId=6999451&action=list&page=éÈ   úhtml.parser>	   õ   é¦–é¡µõ   æ—¶äº‹æ–°é—»õ   æœºæž„è�Œèƒ½õ   æœºæž„è®¾ç½®õ   ç»„ç»‡æœºæž„õ   ç½‘ç«™åœ°å›¾õ   è�”ç³»æˆ‘ä»¬õ   é¢†å¯¼ä¿¡æ�¯õ   æ”¿åºœæœºæž„ç½‘ç«™c                 óô   •— | r|r| j                  d«      ry|j                  «       ‰v ry|r.|j                  «       ‰v st        |j                  «       «      dk  ryt        |j                  «       «      dk  ryy)z'Filter out navigation/placeholder itemszjavascript:Fé   é   T)Ú
startswithr   Úlen)ÚhrefÚtitleÚdate_strÚSKIP_TITLESs      €r&   Úis_valid_itemz$get_list_page.<locals>.is_valid_itemb   sh   ø€ á™5 D§O¡O°MÔ$BØØ�;‰;‹=˜KÑ'ØÙ˜8Ÿ>™>Ó+¨{Ñ:¼cÀ(Ç.Á.ÓBRÓ>SÐVWÒ>WØÜˆu�{‰{‹}Ó Ò!ØØó    z4ul.list li, ul.news_list li, ul.right-list li, ul liÚaÚspanrJ   Ú rK   T)r   ÚhttpNé
   )rK   ÚurlÚdateztable trÚtdé   r   éÿÿÿÿz+.content-list li, .list-item, .xxgk-list li)ÚLIST_URLr'   r7   Ústatus_coder   r   ÚselectÚfindr1   Úget_textrH   r2   ÚappendÚfind_allrI   )ÚpagerU   r.   r6   ÚsoupÚitemsrN   ÚlirP   rQ   rJ   rK   rL   ÚtrÚtdsÚitem_divÚspansrM   s                    @r&   Úget_list_pageri   J   sÂ  ø€ àˆq‚yÜ‰àKÈDÈ6ÐRˆô ‹k€GÙØˆ	ä˜3 Ó(€AÙ�—‘ Ò$Øˆ	ä˜Ÿ™ Ó/€Dð €Eò [€Kô
ð �k‰kÐPÓQò 
SˆØ�G‰G�C‹LˆØ�w‰w�v‹ˆÚ’Ø—5‘5˜ Ó$ˆDØ—E‘E˜' 2Ó&Ò@¨!¯*©*¸4¨*Ó*@ˆEØ—}‘}¨4�}Ó0ˆHÙ˜T 5¨(Õ3Ø—‘ vÔ.Ü# d™?�DØ—‘ u°TÀ8ÈCÈRÀ=ÑQÕRð
Sñ Ø—+‘+˜jÓ)ò 	[ˆBØ—+‘+˜dÓ#ˆCÜ�3‹x˜1‹}Ø˜‘F—K‘K Ó$�ÚØŸ5™5 ¨Ó,�DØŸE™E '¨2Ó.ÒH°!·*±*À4°*Ó2H�EØ" 2™w×/Ñ/°dÐ/Ó;�HÙ$ T¨5°(Õ;Ø#Ÿ™¨vÔ6Ü#+¨d¡?˜DØŸ™¨u¸TÈ8ÐTWÐUWÈ=Ñ%YÕZð	[ñ ØŸ™Ð$QÓRò 
	WˆHØ—‘˜cÓ"ˆAØ×%Ñ% fÓ-ˆEÚ’UØ—u‘u˜V RÓ(�ØŸ™˜g rÓ*ÒD¨a¯j©j¸t¨jÓ.D�Ø  ™9×-Ñ-°DÐ-Ó9�Ù   u¨hÕ7ØŸ?™?¨6Ô2Ü'¨$™˜Ø—L‘L¨5¸ÀxÐPSÐQSÀ}Ñ!UÕVð
	Wð €LrO   c                 ór  — t        «       }|syt        | j                  t        d«      |«      }|r|j                  dk7  ryt        |j                  d«      }|j                  dddi¬«      }d}|r|j                  d	«      r|d	   d
d }d
}dD ]  }|j                  |«      }|sŒ n d}|rt        |«      }||fS )zFetch detail page content)rR   rR   rR   r:   r;   ÚmetaÚnameÚPubDate)ÚattrsÚcontentNrT   )zdiv#zoomzdiv.contentzdiv.main-contentzdiv.article-contentzdiv.bt-contentzdiv.TRS_UEDITORzdiv.trs_editor_viewzdiv.text-contentzdiv.content-bodyzdiv#contentz.article-contentz.news-content)r'   r7   Úreplacer2   r[   r   r   r]   r1   Ú
select_oneÚstr)	rU   r.   r6   rb   Ú	meta_daterL   Úcontent_divÚselectorÚcontent_htmls	            r&   Úfetch_detailrw   ›   sÑ   € ä‹k€GÙØä˜3Ÿ;™;¤x°Ó4°gÓ>€AÙ�—‘ Ò$Øä˜Ÿ™ Ó/€Dð —	‘	˜&¨°Ð(;�	Ó<€IØ€HÙ�Y—]‘] 9Ô-Ø˜YÑ'¨¨Ð,ˆð €Kð:ò ˆð —o‘o hÓ/ˆÚÙðð €LÙÜ˜;Ó'ˆà˜Ð!Ð!rO   c                 óê  — 	 dd l }|j                  t        «      }|j                  «       }|r t	        |d«      j                  dd¬«      d d nd}|j                  dt        | |||||d	f«       |j                  dkD  rt        d
xx   dz  cc<   nt        dxx   dz  cc<   |j                  «        |j                  «        y # t        $ r)}t        d|› �«       t        dxx   dz  cc<   Y d }~y d }~ww xY w)Nr   r;   ú T)Ú	separatorr   iô  rR   z�INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, source_url, summary, category) VALUES (?, ?, ?, ?, ?, ?, ?, ?)Úhpgsr	   r8   r
   z  [ERROR] DB: r   )Úsqlite3ÚconnectÚDB_PATHÚcursorr   r^   ÚexecuteÚ	SITE_NAMEÚrowcountÚstatsÚcommitÚcloser!   r    )	rK   rU   ro   rL   r|   ÚconnÚcÚsummaryr%   s	            r&   Ú
store_itemr‰   ¾   sÝ   € ðÛØ�‰œwÓ'ˆØ�K‰K‹MˆÙel”- ¨Ó7×@Ñ@È3ÐVZÐ@Ó[Ð\`Ð]`ÑaÐrtˆØ	�	‰	ð  eÜ˜e S¨(°G¸SÀ'È6ÐRô	Tà�:‰:˜Š>Ü�%‹L˜AÑŒLä�&‹M˜QÑ‹MØ�‰ŒØ�
‰
�øÜò Ü�˜q˜cÐ"Ô#Üˆh‹˜1Ñ�ˆûðús   ‚B=C  Ã 	C2Ã	C-Ã-C2c           
      ó  — t        d«       t        d«       t        d«       t        dd«      D �],  }t        d|› d�«       t        |«      }|st        d«        �nt        dt        |«      › d	�«       |rt        d
|d   d   › d|d   d   › �«       d}t	        |«      D ]´  \  }}|d   }|d   }|d   }	|	r/|	t
        k  r&t        d|	› d|d d › �«       t        dxx   dz  cc<   ŒFt        d|dz   › d|d d › d|	› �«       |r||k\  rt        d|› d�«         y t        |«      \  }
}|xs |	}t        |||
|«       |dz  }t        j                  d«       Œ¶ | r�Œ- n t        dt        d   › dt        d   › dt        d    › �«       y )!Nz<============================================================z#Hefei Gov EIA Announcements crawlerr8   é3   z

--- Page z ---z  No items foundz  Got z itemsz	  Range: rY   rV   z ~ r   rK   rU   z  [SKIP] Too old: ry   r   r
   z  [z] é(   z | z
  Test mode: z items doneg      à?z
Done! New: r	   z, Skip: z
, Errors: r   )r    Úrangeri   rI   Ú	enumerateÚTHREE_YEARS_AGOrƒ   rw   r‰   ÚtimeÚsleep)ÚfullÚtest_nra   rc   ÚseenÚidxÚitrK   rU   rL   ro   Údetail_dateÚ
final_dates                r&   Úcrawlr™   Ñ   s»  € Ü	ˆ(„OÜ	Ð
/Ô0Ü	ˆ(„Oä�a˜“ó $ˆÜ�˜D˜6 Ð&Ô'Ü˜dÓ#ˆÙÜÐ$Ô%Úä�”s˜5“z�l &Ð)Ô*ÙÜ�I˜e B™i¨Ñ/Ð0°°E¸!±H¸VÑ4DÐ3EÐFÔGàˆÜ  Ó'ò 	‰GˆC�Ø�w‘KˆEØ�U‘)ˆCØ˜&‘zˆHá˜H¤Ò6ÜÐ*¨8¨*°A°e¸C¸R°j°\ÐBÔCÜ�f“ Ñ"“Øä�C˜˜A™�w˜b  s¨  ¨C°¨zÐ:Ô;á˜$ &š.Ü˜¨ x¨{Ð;Ô<Úô $0°Ó#4Ñ ˆG�[Ø$Ò0¨ˆJÜ�u˜c 7¨JÔ7Ø�A‰IˆDÜ�J‰J�s�Oð+	ó. ÙðI$ôL 
ˆMœ% ™,˜ x´°f±¨¸jÌÈxÉÐHYÐ
ZÕ[rO   Ú__main__z--fullz--test=ú=)r’   r“   )N)r8   )FN)&Ú__doc__ÚsysÚrer�   Úosr   r   Úbs4r   r   r   Úgetenvr~   r�   ÚDOMAINr2   rZ   r   r   ÚnowÚstrftimer�   rƒ   r'   r*   r7   ri   rw   r‰   r™   Ú__name__Úargvr’   r“   ÚargrH   ÚintÚsplit© rO   r&   ú<module>r«      s  ðñ÷ +× *× *Ý ß (à
ˆ"�)‰)�KÐ!2Ó
3€Ø0€	Ø	€Ø%€ØB€Ø#€
Ø€à�8—<‘<“>¡I°5Ô$9Ñ9×CÑCÀJÓO€Ø˜1¨Ñ*€òò$ó ó Nòb "òFó&+\ð\ ˆzÒØ�s—x‘xÐ€DØ€FØ�x‰xò ,ˆØ�>‰>˜)Õ$Ù˜Ÿ™ 3›¨Ñ*Ó+‰Fð,ñ 
ˆt˜FÖ#ð rO   