Ë
    ¸¬ujr  ã                   óæ   — d Z ddlZddlZddlZddlZddlZddlmZmZ ddlm	Z	 dZ
dZddd	œZ ej                  «        ed
¬«      z
  j                  d«      ZdZdZd„ Zd„ Zd„ Zedk(  r e«        yy)u‘   å®œæ˜¥ç»�æµŽæŠ€æœ¯å¼€å�‘åŒº-å…¬å‘Šå…¬ç¤º çˆ¬è™« (æ•°èž�UCAP CMS API)
URL: http://jkq.yichun.gov.cn/ycsjkq/gggs/pc/list.html
API: POST /queryList
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupz/root/search.dbz"http://jkq.yichun.gov.cn/queryListz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36zapplication/json)z
User-AgentzContent-TypeiG  )Údaysz%Y-%m-%du(   å®œæ˜¥ç»�æµŽæŠ€æœ¯å¼€å�‘åŒº-å…¬å‘Šå…¬ç¤ºé2   c                  ó€   — t        j                  t        d¬«      } | j                  d«       | j                  d«       | S )Né   )ÚtimeoutzPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000)Úsqlite3ÚconnectÚDB_PATHÚexecute)Úconns    ú /root/gov_crawler/crawl_ycjkq.pyÚget_connr      s1   € Ü�?‰?œ7¨BÔ/€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø€Kó    c                 ó:  — | t         dgdgdœ}	 t        j                  t        |t        d¬«      }|j
                  dk7  r/t        d| › d|j
                  › �t        j                  ¬	«       g d
fS |j                  «       }|j                  di «      j                  dg «      }|j                  di «      j                  dd
«      }g }|D �]  }|j                  di «      }|j                  dd«      }	|j                  dd«      }
t        |
«      dk\  r|
d d n|
}t        |j                  d«      t        «      r"|j                  di «      j                  dd«      nd}|r#t        |xs dd«      j                  d¬«      d d nd}|j                  dd«      }	 t        j                   |«      }d|j                  dd«      z   }|j#                  ||	|||f«       �Œ ||fS #  d}Y Œ#xY w# t$        $ r/}t        d| › d|› �t        j                  ¬	«       g d
fcY d }~S d }~ww xY w)NÚycsjkqÚ1996823590434611200)ÚcurrentÚpageSizeÚwebSiteCodeÚ	channelIdé   )ÚjsonÚheadersr
   éÈ   z[WARN] API page z status=)Úfiler   ÚdataÚresultsÚtotalÚsourceÚtitleÚ ÚpubDateé
   Úcontentzhtml.parserT)Ústripiô  Úurlsz{}zhttp://jkq.yichun.gov.cnÚpcz[ERROR] API page z: )Ú	PAGE_SIZEÚrequestsÚpostÚAPI_URLÚHEADERSÚstatus_codeÚprintÚsysÚstderrr   ÚgetÚlenÚ
isinstanceÚdictr   Úget_textÚloadsÚappendÚ	Exception)ÚpageÚpayloadÚrespr   r    r!   ÚitemsÚitemÚsrcr#   Úpub_date_fullÚpub_dateÚcontent_htmlÚsummaryÚurls_strr)   Úpage_urlÚes                     r   Ú
fetch_pagerI      s  € àÜØ �zØ+Ð,ñ	€GðÜ�}‰}œW¨7¼GÈRÔPˆØ×Ñ˜sÒ"ÜÐ$ T F¨(°4×3CÑ3CÐ2DÐEÌCÏJÉJÕWØ�q�5ˆLØ�y‰y‹{ˆØ—(‘(˜6 2Ó&×*Ñ*¨9°bÓ9ˆØ—‘˜ Ó$×(Ñ(¨°!Ó4ˆØˆØó 	MˆDØ—(‘(˜8 RÓ(ˆCØ—G‘G˜G RÓ(ˆEØŸG™G I¨rÓ2ˆMÜ-0°Ó-?À2Ò-E�} S bÑ)È=ˆHÜHRÐSV×SZÑSZÐ[dÓSeÔgkÔHl˜3Ÿ7™7 9¨bÓ1×5Ñ5°iÀÔDÐrtˆLáeq”m LÒ$6°B¸ÓF×OÑOÐVZÐOÓ[Ð\`Ð]`ÑaÐwyˆGà—w‘w˜v tÓ,ˆHðÜ—z‘z (Ó+�Ø5¸¿¹ÀÀrÓ8JÑJ�ð �L‰L˜( E¨8°\À7ÐKÖLð	Mð  �eˆ|ÐøðØ’ûô ò ÜÐ! $  r¨!¨Ð-´C·J±JÕ?Ø�1ˆu�ûðús=   �AG" Á.D#G" Æ*GÆ<G" ÇGÇG" Ç"	HÇ+$HÈHÈHc            
      óÄ  — dt         j                  v } t        «       }|j                  «       }d}d}d}t	        d«      \  }}|st        d«       y |t        z   dz
  t        z  }t        d|› d|› �«       |}	| sNt        d|dz   «      D ]<  }
t	        |
«      \  }}|	j                  |«       t        d|
› d	t        |«      › d
�«       Œ> |	D ]”  \  }}}}}|t        k  r|dz  }Œ|r|j                  d|f«       n|j                  d||t        f«       |j                  «       r|dz  }Œ]|j                  dt        |||||f«       |dz  }| sŒ�t        d|› d|d d › �«       Œ– |j                  «        t        d|› d�«       |j                  dt        f«       |j                  «        |j                  «        t        d«       t        d|› d|› d|› �«       | rt        d«       y y )Nz--dailyr   é   u   [ERROR] æ— æ³•èŽ·å�–æ•°æ�®u   æ€»æ�¡æ•°: u   , æ€»é¡µæ•°: é   u	   [API] ç¬¬u   é¡µ -> u    æ�¡z)SELECT id FROM gov_raw WHERE page_url = ?zMSELECT id FROM gov_raw WHERE title = ? AND publish_date = ? AND site_name = ?ztINSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)z  [ADD] ú é(   u   
å�Œæ­¥ FTS (u    æ�¡æ–°å¢ž)...z·INSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?u   
=== å®Œæˆ� ===u   æ–°å¢ž: u   , è·³è¿‡é‡�å¤�: u   , è¶…è¿‡3å¹´: u#   æ¨¡å¼�: æ—¥è·‘ï¼ˆä»…ç¬¬1é¡µ50æ�¡ï¼‰)r2   Úargvr   ÚcursorrI   r1   r+   ÚrangeÚextendr5   ÚTHREE_YEARS_AGOr   Ú	SITE_NAMEÚfetchoneÚcommitÚclose)Ú
daily_moder   ÚcurÚtotal_addedÚtotal_skippedÚtotal_skip_dater?   r!   Útotal_pagesÚ	all_itemsr<   Ú
more_itemsÚ_rG   r#   rC   rD   rE   s                     r   Úmainra   C   s  € ØœcŸh™hÐ&€Jä‹:€DØ
�+‰+‹-€CØ€KØ€MØ€Oô ˜a“=�L€Eˆ5ÙÜÐ*Ô+ØØœ9Ñ$ qÑ(¬YÑ6€KÜ	ˆK˜�w˜m¨K¨=Ð
9Ô:ð €Iáä˜!˜[¨1™_Ó-ò 	BˆDÜ& tÓ,‰MˆJ˜Ø×Ñ˜ZÔ(Ü�I˜d˜V 7¬3¨z«?Ð*;¸4Ð@ÕAð	Bð
 =Fò 6Ñ8ˆ�%˜ <°Ø”oÒ%Ø˜qÑ ˆOØáØ�K‰KÐCÀhÀ[ÕQð �K‰KÐgØ˜x¬Ð3ô5à�<‰<Œ>Ø˜QÑˆMØà�‰ð CÜ˜ %¨°xÀÐIô	
ð 	�qÑˆÚÜ�H˜X˜J a¨¨c¨r¨
 |Ð4Õ5ð+6ð. 	‡K�K„Mä	ˆN˜;˜- ~Ð
6Ô7Ø‡K�Kð	\ô 
ˆô	ð 	‡K�K„MØ‡J�J„Lä	ÐÔÜ	ˆH�[�MÐ!1°-°ÀÈÐN_Ð
`ÔaÙÜÐ3Õ4ð r   Ú__main__)Ú__doc__r,   r   r   r2   Úrer   r   Úbs4r   r   r.   r/   ÚnowÚstrftimerS   rT   r+   r   rI   ra   Ú__name__© r   r   ú<module>rj      sŒ   ðñó Û Û Û 
Û 	ß (Ý à
€Ø
.€àPØ&ñ€ð  �8—<‘<“>¡I°5Ô$9Ñ9×CÑCÀJÓO€Ø6€	Ø€	òò#òJA5ðF ˆzÒÙ…Fð r   