Ë
    ¸¬ujõ  ã                   óè   — d Z ddlZddlZddlZddlZddlmZmZ ddlmZ dZ	dZ
dZdd	iZ ej                  «        ed
¬«      z
  j                  d«      ZdZd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uÅ   å®�ä¸°åŽ¿-é€šçŸ¥å…¬å‘Š çˆ¬è™« (SiteServer CMS)
URL: https://www.baofeng.gov.cn/channels/16434.html
åˆ†é¡µ: /channels/16434.html (ç¬¬1é¡µ), /channels/16434_{N}.html (ç¬¬Né¡µ)
æ¯�é¡µ15æ�¡ï¼Œå…±10é¡µ
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupz/root/search.dbzhttps://www.baofeng.gov.cnz.https://www.baofeng.gov.cn/channels/16434.htmlz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36iG  )Údaysz%Y-%m-%du   å®�ä¸°åŽ¿-é€šçŸ¥å…¬å‘Šc                  ó€   — t        j                  t        d¬«      } | j                  d«       | j                  d«       | S )Né   )ÚtimeoutzPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000)Úsqlite3ÚconnectÚDB_PATHÚexecute)Úconns    ú"/root/gov_crawler/crawl_baofeng.pyÚget_connr      s1   € Ü�?‰?œ7¨BÔ/€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø€Kó    c                 óZ  — 	 t        j                  | t        d¬«      }d|_        |j                  dk7  ryt        |j                  d«      }|j                  d«      }|r|j                  d¬	«      nd
}|j                  d«      }d
}|r7t        j                  d|j                  «       «      }|r|j                  d«      }|j                  d«      xs |j                  d«      }|rt        |«      nd
}	||	|fS # t        $ r+}
t        d| › d|
› �t        j                   ¬«       Y d }
~
yd }
~
ww xY w)Né   ©Úheadersr	   úutf-8éÈ   )NNNúhtml.parserzh1.titleT©ÚstripÚ z.articleInfoz(\d{4}-\d{2}-\d{2})é   z.contentTxt#textz.contentTxtu   [ERROR] è¯¦æƒ…é¡µå¤±è´¥: ú - ©Úfile)ÚrequestsÚgetÚHEADERSÚencodingÚstatus_coder   ÚtextÚ
select_oneÚget_textÚreÚsearchÚgroupÚstrÚ	ExceptionÚprintÚsysÚstderr)ÚurlÚrespÚsoupÚtitle_elÚtitleÚdate_elÚpub_dateÚmÚ
content_elÚcontentÚes              r   Ú
get_detailr;      s  € ð Ü�|‰|˜C¬¸"Ô=ˆØˆŒØ×Ñ˜sÒ"Ø#Ü˜TŸY™Y¨Ó6ˆØ—?‘? :Ó.ˆÙ19�×!Ñ!¨Ð!Ô-¸rˆØ—/‘/ .Ó1ˆØˆÙÜ—	‘	Ð0°'×2BÑ2BÓ2DÓEˆAÙØŸ7™7 1›:�Ø—_‘_Ð%7Ó8ÒZ¸D¿O¹OÈMÓ<Zˆ
Ù%/”#�j”/°RˆØ�g˜xÐ'Ð'øÜò  ÜÐ)¨#¨¨c°!°Ð5¼C¿J¹JÕGÜûð ús   ‚2C6 µC C6 Ã6	D*Ã?!D%Ä%D*c                 ó¦  — g }	 t        j                  | t        d¬«      }d|_        |j                  dk7  r-t        d| › d|j                  › �t        j                  ¬«       |S t        |j                  d«      }|j                  d	«      }|D ]‰  }|j                  d
d«      }|j                  d«      s	t        |z   }|j                  dd«      j                  «       }|j                  d«      }|r|j                  d¬«      nd}	|j!                  |||	f«       Œ‹ 	 |S # t"        $ r,}
t        d| › d|
› �t        j                  ¬«       Y d }
~
|S d }
~
ww xY w)Nr   r   r   r   u   [WARN] åˆ—è¡¨é¡µå¤±è´¥: z status=r   r   z(.wrap-fr-tet ul li a[href*="/contents/"]Úhrefr   Úhttpr4   ÚspanTr   u   [ERROR] åˆ—è¡¨é¡µæŠ“å�–å¤±è´¥: r   )r    r!   r"   r#   r$   r-   r.   r/   r   r%   ÚselectÚ
startswithÚBASE_URLr   r&   r'   Úappendr,   )Úpage_urlÚarticlesr1   r2   ÚitemsÚar=   r4   Ú	date_spanÚdater:   s              r   Ú
crawl_pagerJ   4   sB  € Ø€HðSÜ�|‰|˜H¬g¸rÔBˆØˆŒØ×Ñ˜sÒ"ÜÐ,¨X¨J°h¸t×?OÑ?OÐ>PÐQÔX[×XbÑXbÕcØˆOÜ˜TŸY™Y¨Ó6ˆØ—‘ÐFÓGˆØò 	1ˆAØ—5‘5˜ Ó$ˆDØ—?‘? 6Ô*Ü $‘�Ø—E‘E˜' 2Ó&×,Ñ,Ó.ˆEØŸ™ VÓ,ˆIÙ5>�9×%Ñ%¨DÐ%Ô1ÀBˆDØ�O‰O˜T 5¨$Ð/Õ0ñ	1ð €Oøô ò SÜÐ/°¨z¸¸Q¸CÐ@ÄsÇzÁz×RÑRØ€OûðSús   „AD Á#B5D Ä	EÄ$!EÅEc                  óÊ   — 	 t        j                  t        t        d¬«      } d| _        t        j                  d| j                  «      }|rt        d„ |D «       «      S y#  Y yxY w)Nr   r   r   z"href="/channels/16434_(\d+)\.html"c              3   ó2   K  — | ]  }t        |«      –— Œ y ­w)N)Úint)Ú.0r7   s     r   ú	<genexpr>z"get_total_pages.<locals>.<genexpr>P   s   è ø€ Ò/ !”s˜1—vÑ/ùs   ‚r   )	r    r!   ÚLIST_URLr"   r#   r(   Úfindallr%   Úmax)r1   Úmatchess     r   Úget_total_pagesrT   J   sV   € ðÜ�|‰|œH¬g¸rÔBˆØˆŒÜ—*‘*ÐBÀDÇIÁIÓNˆÙÜÑ/ wÔ/Ó/Ð/ØøðÙús   ‚AA ÁA"c                  ó  — dt         j                  v } t        «       }t        d|› �«       t	        «       }|j                  «       }d}d}| rt        g}n)t        gt        d|dz   «      D �cg c]  }d|› d�‘Œ
 c}z   }|D �]  }t        |«      }	t        d|› d	t        |	«      › d
�«       |	D ]Ö  \  }
}}|t        k  rt        d|› d|› �«       Œ"|j                  d|
f«       |j                  «       r|dz  }ŒKt        |
«      \  }}}|s|}|}t        d|› �«       |r#t        |xs dd«      j                  d¬«      d d nd}|j                  dt         |
||||f«       |dz  }| sd|› d�nd}t        d|› d|› d|d d › �«       ŒØ �Œ |j#                  «        t        d|› d�«       |j                  dt         f«       |j#                  «        |j%                  «        t        d«       t        d|› d |› �«       | rt        d!«       y y c c}w )"Nz--dailyu   æ€»é¡µæ•°: r   é   r   z*https://www.baofeng.gov.cn/channels/16434_z.htmlu   [åˆ—è¡¨é¡µ] z -> u    æ�¡z	  [SKIP] u    è¶…è¿‡3å¹´: z)SELECT id FROM gov_raw WHERE page_url = ?u      [WARN] ä½¿ç”¨åˆ—è¡¨é¡µä¿¡æ�¯: r   r   Tr   iô  ztINSERT OR IGNORE INTO gov_raw (site_name, page_url, title, content, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)ú[ú]z  [ADD]z  ú é(   u   
å�Œæ­¥ FTS (u    æ�¡æ–°å¢ž)...z·INSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?u   
=== å®Œæˆ� ===u   æ–°å¢ž: u   , è·³è¿‡é‡�å¤�: u   æ¨¡å¼�: æ—¥è·‘ï¼ˆä»…ç¬¬1é¡µï¼‰)r.   ÚargvrT   r-   r   ÚcursorrP   ÚrangerJ   ÚlenÚTHREE_YEARS_AGOr   Úfetchoner;   r   r'   Ú	SITE_NAMEÚcommitÚclose)Ú
daily_modeÚtotal_pagesr   ÚcurÚtotal_addedÚtotal_skippedÚpages_to_crawlÚirD   rE   r=   r4   Ú	list_dateÚdetail_titler9   Údetail_dateÚsummaryÚlabels                     r   Úmainrp   U   s^  € ØœcŸh™hÐ&€JÜ!Ó#€KÜ	ˆK˜�}Ð
%Ô&Ü‹:€DØ
�+‰+‹-€CØ€KØ€MÙÜ"˜‰ä"˜ÔfkÐlmÐozÐ}~Ño~Ófö  'AÐabÐ)SÐTUÐSVÐV[Ò'\ò  'Añ  AˆØ"ó BˆÜ˜hÓ'ˆÜ�˜X˜J d¬3¨x«=¨/¸Ð>Ô?Ø&.ò 	BÑ"ˆD�%˜Øœ?Ò*Ü˜	 ) ¨M¸%¸ÐAÔBØØ�K‰KÐCÀdÀWÔMØ�|‰|Œ~Ø Ñ"�ØÜ1;¸DÓ1AÑ.ˆL˜' ;ÙØ$�Ø'�ÜÐ8¸¸Ð@ÔAÙ`g”m G¢M¨r°=ÓA×JÑJÐQUÐJÓVÐW[ÐX[Ñ\ÐmoˆGØ�K‰Kð GÜ˜D ,°¸ÀgÐNôð ˜1ÑˆKÙ.8�a˜�} AÑ&¸iˆEÜ�B�u�g˜Q˜{˜m¨1¨\¸#¸2Ð->Ð,?Ð@ÕAò)	BðBð0 	‡K�K„Mä	ˆN˜;˜- ~Ð
6Ô7Ø‡K�Kð	\ô 
ˆô	ð 	‡K�K„MØ‡J�J„LÜ	ÐÔÜ	ˆH�[�MÐ!1°-°Ð
AÔBÙÜÐ.Õ/ð ùòM 'As   Á*HÚ__main__)Ú__doc__r    r(   r
   r.   r   r   Úbs4r   r   rB   rP   r"   ÚnowÚstrftimer_   ra   r   r;   rJ   rT   rp   Ú__name__© r   r   ú<module>rx      s”   ðñó
 Û 	Û Û 
ß (Ý à
€Ø'€Ø;€àð  Dð€ð  �8—<‘<“>¡I°5Ô$9Ñ9×CÑCÀJÓO€Ø$€	òò ò,ò,	ò20ðh ˆzÒÙ…Fð r   