Ë
    ¸¬uj˜"  ã                   óæ  — d dl Z 	 d dlZd dlZd dlZd dlZd dl Z d dlZd dlmZ d dlmZm	Z	 dZ
dZdZ e j                  dd«      Zd	Ze› d
�ZddddddddœZdeez   dœZ ej&                  «        e	d¬«      z
  j)                  d«      Zd„ Zd„ Zd„ Zd„ Zd„ Zd#d„Zedk(  rRdej:                  v ZdZej:                  d d D ]%  Z e jC                  «       r
 e"e «      Z ne d!k(  sŒ#d Z n  eee¬"«       yy)$é    N)Úurljoin)ÚdatetimeÚ	timedeltau%   é‡‘æ˜Œå¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€-çŽ¯è¯„ä¿¡æ�¯zhttps://sthj.jcs.gov.cnz)/zwgk/fdzdgknr/hjxx/jsxmhjyxpg/index.htmlÚ	SEARCH_DBz/root/search.dbé   z7/api-gateway/jpaas-publish-server/front/page/build/unitÚ 47f74ec6fa054832961ddb79491f6065Ú c3594f9e0ce64fe38759159c9d07d002ÚbulidstaticÚcolumnu+   æ³•å®šä¸»åŠ¨å…¬å¼€å†…å®¹ç¬¬ä¸€ä¸ªæ �ç›®listÚnullÚ a7ce57f894724fb1970fb836e7b631ff)ÚwebIdÚpageIdÚ	parseTypeÚpageTypeÚtagIdÚeditTypeÚtplSetIdz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36)z
User-AgentÚRefereriG  )Údaysz%Y-%m-%dc                 óÂ  — t        t        «      }t        j                  |ddœd¬«      |d<   	 | j	                  t
        |t        d¬«      }|j                  dk7  ry	|j                  «       }|j	                  d
«      sy	|d   d   }t        j                  d|«      }|rt        |j                  d«      «      nd}||fS # t        $ r}t        d|› �«       Y d}~y	d}~ww xY w)z=Fetch one page of list via GET API with paramJson pagination.é   )ÚpageNoÚpageSizeF)Úensure_asciiÚ	paramJsoné   )ÚparamsÚheadersÚtimeoutéÈ   )Nr   ÚsuccessÚdataÚhtmlzcount="(\d+)"é   r   z    API error: N)ÚdictÚ
API_PARAMSÚjsonÚdumpsÚgetÚAPI_URLÚHEADERSÚstatus_codeÚreÚsearchÚintÚgroupÚ	ExceptionÚprint)	ÚsessionÚpager   Úrr#   r$   ÚmÚtotalÚes	            ú#/root/gov_crawler/crawl_jincheng.pyÚ
fetch_listr;   "   sË   € ä”*Ó€FÜŸ*™*°À"Ñ%EÐTYÔZ€Fˆ;ÑðØ�K‰Kœ¨¼ÈˆKÓLˆØ�=‰=˜CÒØØ�v‰v‹xˆØ�x‰x˜	Ô"ØØ�F‰|˜FÑ#ˆä�I‰IÐ&¨Ó-ˆÙ#$”�A—G‘G˜A“J”¨!ˆØ�Uˆ{ÐøÜò Ü� ˜sÐ#Ô$Üûðús#   ®,B= Á!B= Á=?B= Â=	CÃCÃCc                 ób  — g }t        j                  dt         j                  «      }|j                  | «      D ]t  }t	        t
        |j                  d«      «      }|j                  d«      j                  «       }|j                  d«      j                  «       }|j                  |||dœ«       Œv |S )z%Extract article items from list HTML.zc<a\s+class="fl"\s+href="([^"]+)"\s+title="([^"]*)"[^>]*>.*?</a>\s*<span\s+class="fr">([^<]+)</span>r%   é   é   )ÚurlÚtitleÚdate)	r.   ÚcompileÚDOTALLÚfinditerr   ÚBASEr1   ÚstripÚappend)r$   ÚitemsÚpatternr7   r?   r@   Údate_strs          r:   Ú
parse_listrK   7   s—   € à€Eä�j‰jØnÜ
�	‰	ó€Gð ×Ñ˜dÓ#ò EˆÜ”d˜AŸG™G A›JÓ'ˆØ—‘˜“
× Ñ Ó"ˆØ—7‘7˜1“:×#Ñ#Ó%ˆØ�‰˜S¨5¸(ÑCÕDð	Eð
 €Ló    c                 óh   — 	 | j                  |t        d¬«      }d|_        |j                  S #  Y y xY w)Nr   )r   r    zutf-8)r*   r,   ÚencodingÚtext)r4   r?   r6   s      r:   Ú
fetch_pagerP   G   s5   € ðØ�K‰K˜¤W°bˆKÓ9ˆØˆŒ
Ø�v‰vˆøðÙús   ‚*- ­1c                 óæ  — d}t        j                  d| t         j                  «      }|r4t        j                  dd|j	                  d«      «      j                  «       }d}t        j                  d| «      }|st        j                  d| «      }|r|j	                  d«      }d}t        j                  d| t         j                  «      }|r |j	                  d«      j                  «       }nFt        j                  d| t         j                  «      }|r|j	                  d«      j                  «       }t        j                  d	d|«      }t        j                  d
d|«      }|j                  «       }|||fS )z.Extract title, date, content from detail page.Ú z<h1[^>]*>(.+?)</h1>z<[^>]+>r%   z=<meta[^>]*name="PubDate"[^>]*content="(\d{4}-\d{1,2}-\d{1,2})u,   å�‘å¸ƒæ—¶é—´[ï¼š:]\s*(\d{4}-\d{1,2}-\d{1,2})uF   <div\s+class="ty-content"[^>]*>(.*?)</div>\s*<!--\s*æ­£æ–‡ç»“æ�Ÿ\s*-->z<body[^>]*>(.*?)</body>z\s*style="[^"]*"z
<!--.*?-->)r.   r/   rC   Úsubr1   rF   )r$   r@   r7   rJ   ÚdmÚcontentÚcmÚbms           r:   Úparse_detailrX   P   s.  € à€EÜ
�	‰	Ð(¨$´·	±	Ó:€AÙÜ—‘�z 2 q§w¡w¨q£zÓ2×8Ñ8Ó:ˆà€HÜ	�‰ØHÈ$ó
€Bñ Ü�Y‰YÐFÈÓMˆÙ	Ø—8‘8˜A“;ˆð €GÜ	�‰Ð\Ð^bÔdf×dmÑdmÓ	n€BÙ	Ø—(‘(˜1“+×#Ñ#Ó%‰ô �Y‰YÐ1°4¼¿¹ÓCˆÙØ—h‘h˜q“k×'Ñ'Ó)ˆGô �f‰fÐ(¨"¨gÓ6€Gä�f‰f�] B¨Ó0€GØ�m‰m‹o€Gà�(˜GÐ#Ð#rL   c                 óè   — d}d}|D ]Z  }	 | j                  d|d   dd |d   |d   |d   |d	   |d
   |d   |d   |d   |d   f
«       | j                  dkD  r|dz  }n|dz  }Œ\ ||fS #  |dz  }Y ŒjxY w)z!Bulk insert records into gov_raw.r   z»INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags)
                VALUES (?,?,?,?,?,?,?,?,?,?)Ú	site_nameNr!   Ú
source_urlÚpage_urlr@   Úpublish_dateÚsummaryrU   ÚstatusÚcategoryÚtagsr%   )ÚexecuteÚtotal_changes)ÚdbÚrecordsÚinsertedÚskippedÚitems        r:   Ú
push_to_dbri   t   sÊ   € à€HØ€GØò ˆð	Ø�J‰Jð0ð ˜Ñ% d sÐ+Ø˜Ñ&Ø˜Ñ$Ø˜‘MØ˜Ñ(Ø˜‘OØ˜‘OØ˜‘NØ˜Ñ$Ø˜‘Lðô	ð" ×Ñ !Ò#Ø˜A‘‘à˜1‘�øð-ð2 �WÐÐøð	Ø�q‰LŠGús   ‹AA(Á(A1c                 ó¶  ‡— | €t         } t        j                  t        «      }|j	                  d«       t        d„ |j	                  dt        f«      j                  «       D «       «      Št        dt        ‰«      › �«       t        j                  «       }|j                  j                  t        «       t        dd¬«       t        |d«      \  }}|st        d	«       y t!        |«      }t        d
t        |«      › d|› d�«       t#        | |r|dz   dz  n| «      }t        d|› �«       |r9‰r7|r|d   d   nd}|‰v rt        d«       |j%                  «        y t        d«       t'        |«      }	t)        d|dz   «      D ]Ê  }
t        d|
› d�d¬«       t        ||
«      \  }}|st        d«        n›t!        |«      }|st        d«        n�t        d
t        |«      › d�«       |rTt+        ˆfd„|D «       «      }|rt        d|
› d�«        n@|D �cg c]  }|d   ‰vsŒ|‘Œ }}|s n%|	j-                  |«       Œº|	j-                  |«       ŒÌ |	D �cg c]  }|d   t.        k\  s|d   rŒ|‘Œ }	}|	st        d«       |j%                  «        y t        d t        |	«      › �«       g }t1        |	«      D ]º  \  }}t3        ||d   «      }|s$t        d!|dz   › d"t        |	«      › d#|d   › �«       Œ;t5        |«      \  }}}|j7                  t        |d   d d$ |d   |xs |d%   d d$ |xs |d   d d& |xs |d%   d d$ |d'ddd(œ
«       |dz   d&z  dk(  sŒ�t        d!|dz   › d"t        |	«      › d)�«       Œ¼ t9        ||«      \  }}|j;                  «        |j	                  d*t        f«       |j;                  «        |j%                  «        t        d+|› d,|› �«       y c c}w c c}w )-NzPRAGMA journal_mode=WALc              3   ó&   K  — | ]	  }|d    –— Œ y­w)r   N© )Ú.0r6   s     r:   ú	<genexpr>zrun.<locals>.<genexpr>œ   s   è ø€ ò àð 	
ˆ!�ñùs   ‚z.SELECT page_url FROM gov_raw WHERE site_name=?z  Known in DB: z  Fetching page 1...ú )Úendr%   zFAIL - API returned emptyzOK - z items (total: ú)é   r   z  Total pages: r   r?   rR   z(  No new data (newest URL already in DB)z,  Newest item not in DB, will scan all pagesr=   z  Fetching page z...ÚFAILzempty, donez itemsc              3   ó,   •K  — | ]  }|d    ‰v –— Œ y­w)r?   Nrl   )rm   rh   Úknowns     €r:   rn   zrun.<locals>.<genexpr>Í   s   øè ø€ ÒH°T˜D ™K¨5Ô0ÑHùs   ƒz    All items on page z already in DB, stoppingrA   z  No items to processz   Total items to fetch details: z    [ú/z] SKIP (fetch failed): iô  r@   é
   Úactive)
rZ   r[   r\   r@   r]   r^   rU   r_   r`   ra   ú]zÔINSERT INTO gov_search(rowid,title,site_name,summary)
        SELECT r.id, r.title, r.site_name, r.summary
        FROM gov_raw r
        WHERE r.id NOT IN (SELECT rowid FROM gov_search)
        AND r.site_name=?z  Done: new=z, skip=)Ú	MAX_PAGESÚsqlite3Úconnectr   rb   ÚsetÚ	SITE_NAMEÚfetchallr3   ÚlenÚrequestsÚSessionr   Úupdater,   r;   rK   ÚminÚcloseÚlistÚrangeÚallÚextendÚCUTOFFÚ	enumeraterP   rX   rG   ri   Úcommit)Ú	max_pagesÚincrementalrd   r4   r$   r8   rH   Útotal_pagesÚ
newest_urlÚ	all_itemsr5   Ú_Ú
page_itemsÚ	all_knownÚiÚ	new_itemsÚresultsrh   r@   rJ   rU   rf   rg   ru   s                          @r:   Úrunr˜   ”   s  ø€ ØÐÜˆ	ä	�‰œÓ	#€BØ‡J�JÐ(Ô)ô ñ à—‘Ø<¼y¸ló
ç
‰(‹*ô	ó €Eô 
ˆOœC ›J˜<Ð
(Ô)ä×ÑÓ €GØ‡O�O×Ñœ7Ô#ô 
Ð  sÕ+Ü˜W aÓ(�K€Dˆ%ÙÜÐ)Ô*ØÜ�tÓ€EÜ	ˆE”#�e“*�˜_¨U¨G°1Ð
5Ô6ô �i±u %¨"¡*°Ò!3À)ÓL€KÜ	ˆO˜K˜=Ð
)Ô*ñ ‘uÙ(-�U˜1‘X˜e’_°2ˆ
Ø˜ÑÜÐ<Ô>Ø�H‰HŒJØÜÐ<Ô>ô �U“€IÜ�a˜ q™Ó)ò )ˆÜÐ    cÐ*°Õ4Ü˜W dÓ+‰ˆˆaÙÜ�&ŒMÙÜ Ó%ˆ
ÙÜ�-Ô ÙÜ�”c˜*“oÐ& fÐ-Ô.ñ ÜÓH¸ZÔHÓHˆIÙÜÐ.¨t¨fÐ4LÐMÔNÙà$.ÖH˜q°!°E±(À%Ò2GšÐHˆIÐHÙÙØ×Ñ˜ZÕ(à×Ñ˜ZÕ(ð1)ð6 &ÖN�q¨¨6©´fÒ)<ÀAÀfÃI’ÐN€IÐNáÜÐ%Ô&Ø
�‰Œ
Øä	Ð,¬S°«^Ð,<Ð
=Ô>ð €GÜ˜YÓ'ò 3‰ˆˆ4Ü˜' 4¨¡;Ó/ˆÙÜ�E˜!˜A™#˜˜a¤ I£Ð/Ð/FÀtÈEÁ{ÀmÐTÔUØÜ#/°Ó#5Ñ ˆˆx˜Ø�‰ä&Ø" 5™k¨$¨3Ð/Ø  ™KØÒ0 4¨¡=°$°3Ð7Ø!)Ò!9¨T°&©\¸3¸BÐ ?Ø!Ò2 T¨'¡]°D°SÐ9Ø"Ø"ØØñô	
ð �‰E�R‰<˜1ÓÜ�E˜!˜A™#˜˜a¤ I£Ð/¨qÐ1Õ2ð+3ô0 # 2 wÓ/Ñ€HˆgØ‡I�I„Kð ‡J�Jð	ô
 
ˆôð ‡I�I„KØ‡H�H„Jä	ˆL˜˜
 '¨'¨Ð
3Õ4ùòu Iùò Os   ÈOÈOÉOÉ$OÚ__main__z--incrementalr%   z--test)r�   rŽ   )NF)#Úosr.   Úsysr{   r(   r�   Úurllib.parser   r   r   r~   rE   Ú	LIST_PAGEÚgetenvr   rz   r+   r'   r,   ÚnowÚstrftimerŠ   r;   rK   rP   rX   ri   r˜   Ú__name__ÚargvrŽ   ÚmpÚaÚisdigitr0   rl   rL   r:   ú<module>r¦      s3  ðã 	Ø Kß !× !Ó !Û Ý  ß (à3€	Ø €Ø7€	ØˆB�I‰I�kÐ#4Ó5€	Ø€	àˆFÐIÐ
J€à/Ø0ØØØ:ØØ2ñ€
ð QØ�iÑñ€ð
 ˆ(�,‰,‹.™9¨'Ô2Ñ
2×	<Ñ	<¸ZÓ	H€òò*ò ò!$òHó@x5ðv ˆzÒØ! S§X¡XÐ-€Kà	€BØ�X‰X�a�bˆ\ò ˆØ�9‰9Œ;Ù�Q“ˆBÙØ�‹=ØˆBÙðñ �" +Ö.ð rL   