Ë
    ¸¬ujs!  ã                   ó&  — d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
mZ dZdZej                  Zd„ Zee_        dZd	Zd
Z eddd«      Zej*                  j-                  dd«      ZdZdZdddœZdZd„ Zd„ Zd„ Zedk(  r e«        yy)u{  å®�æ™‹åŽ¿äººæ°‘æ”¿åºœ - æ”¿åŠ¡å…¬å¼€/æ �ç›®24 çˆ¬è™« (é�™æ€�åˆ†é¡µ /channel/list/24_N.html)

Same CMS structure as channel 52:
  - List:    /channel/list/24.html (p1), /channel/list/24_N.html (p2+)
  - Detail:  /single/24/{ID}.html
  - Title:   <h1 class="c-h1">
  - Content: <div class="c-detail"> (depth-counting)
  - Date:    <span class="time">YYYY-MM-DD</span> in list page
é    N)ÚdatetimeÚdate)ÚThreadPoolExecutorÚas_completedz183.196.191.98zwww.ningjin.gov.cnc                 óˆ   — | t         k(  r*t        j                  t        j                  ddt        |ffgS t        | |g|¢­i |¤ŽS )Né   Ú )Ú_FORCED_HOSTÚsocketÚAF_INETÚSOCK_STREAMÚ
_FORCED_IPÚ_original_getaddrinfo)ÚhostÚportÚargsÚkwargss       ú$/root/gov_crawler/crawl_ningjin24.pyÚ_patched_getaddrinfor      sD   € ØŒ|ÒÜ—‘¤×!3Ñ!3°Q¸¼ZÈÐ<NÐOÐPÐPÜ   tÐ=¨dÒ=°fÑ=Ð=ó    zhttps://www.ningjin.gov.cnz/https://www.ningjin.gov.cn/channel/list/24.htmlz2https://www.ningjin.gov.cn/channel/list/24_{}.htmliç  r   é   ÚDB_PATHz/root/search.dbu   å®�æ™‹åŽ¿-æ”¿åŠ¡å…¬å¼€é   z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36ztext/html,application/xhtml+xml)z
User-AgentÚAccepté   c                  ó�  — g } t        dt        dz   «      D �]Û  }|dk(  rt        nt        j	                  |«      }t        d«      D �]{  }	 t        j                  |dt        ¬«      }d|_        |j                  dk7  rt        j                  d«       ŒMt        j                  d|j                  t        j                  «      }|dkD  rd	nd
}d}|D ]¤  \  }}	}
}t        j                   dd|
«      j#                  «       }|r|dk(  rŒ5|j#                  «       dd }t$        |z   }	 t'        j(                  |«      }|t*        k\  r5d
}| j-                  ||f«       |dz  }|dk  rt/        d|› d|dd › �d	¬«       Œ¦ |dk(  r|dkD  rt/        d|› d�d	¬«       | c c S |r)|dkD  r$t/        d|› d|r|d   d   nd› d�d	¬«       | c c S  n |dz  dk(  s�Œ¸t/        d|› dt        › dt3        | «      › d�d	¬«       �ŒÞ | S #  | j-                  ||f«       |dz  }Y �ŒFxY w# t0        $ r%}|dk  rt        j                  d«       Y d}~�Œûd}~ww xY w) u(   æ”¶é›†æ‰€æœ‰æ–‡ç« URLï¼Œè¶…3å¹´è‡ªåŠ¨å�œé   é   é   ©ÚtimeoutÚheadersúutf-8éÈ   é   zehref="(/single/24/(\d+)\.html)"[^>]*>(.*?)</a>\s*<span[^>]*class="[^"]*time[^"]*"[^>]*>([^<]+)</span>TFr   ú<[^>]+>r	   u   ç™»å½•Né
   z    [ú] é2   ©Úflushu     ç¬¬u!   é¡µæ— æœ‰æ•ˆæ•°æ�®ï¼Œå�œæ­¢æ‰«æ��u   é¡µå…¨éƒ¨è¶…3å¹´(éÿÿÿÿzN/Au   ), å�œæ­¢æ‰«æ��é   u     æ‰«æ��é¡µ ú/z... (u   æ�¡))ÚrangeÚ	MAX_PAGESÚLIST_URLÚPAGE_URLÚformatÚrequestsÚgetÚHEADERSÚencodingÚstatus_codeÚtimeÚsleepÚreÚfindallÚtextÚSÚsubÚstripÚBASEr   ÚfromisoformatÚCUTOFF_DATEÚappendÚprintÚ	ExceptionÚlen)Úall_urlsÚpnÚurlÚattÚrÚitemsÚall_before_cutoffÚ	new_countÚhrefÚid_strÚ
title_htmlÚdate_strÚtitleÚ
date_cleanÚfull_urlÚdÚes                    r   Úcollect_urlsrY   -   s[  € à€HÜ�A”y 1‘}Ó%ó -WˆØ š'�h¤x§¡°rÓ':ˆÜ˜“8ó )	"ˆCð("Ü—L‘L ¨b¼'ÔB�Ø$�”
Ø—=‘= CÒ'Ü—J‘J˜q”MØäŸ
™
Ø|Ø—F‘FÜ—D‘Dó�ð
 -/°ªF¡D¸Ð!Ø�	Ø:?ò 'Ñ6�D˜& *¨hÜŸF™F :¨r°:Ó>×DÑDÓF�EÙ  E¨XÒ$5Ø Ø!)§¡Ó!1°#°2Ð!6�JÜ# d™{�Hð
'Ü ×.Ñ.¨zÓ:˜Ø¤Ò+Ø05Ð-Ø$ŸO™O¨X°zÐ,BÔCØ%¨™N˜IØ(¨Aš~Ü %¨¨j¨\¸¸EÀ#À2¸J¸<Ð&HÐPTÕ Uøð'ð"  ’> b¨1¢fÜ˜E " Ð%FÐGÈtÕTØ#”OÙ$¨¨aªÜ˜E " Ð%6Áu°u¸R±yÀ²|ÐRWÐ6XÐXgÐhÐptÕuØ#”OÙðM)	"ðT �‰6�QŒ;Ü�L   A¤i [°´c¸(³m°_ÀDÐIÐQU×Vð[-Wð\ €Oøð'Ø Ÿ™¨°:Ð(>Ô?Ø! Q™›	ûô ò "Ø˜’7Ü—J‘J˜q”Mÿùð"ús?   Á
AHÂBHÄ AG7Å3HÆ&HÇ7H	ÈHÈ	IÈ I É Ic                 óÔ  — t        d«      D �]³  }	 t        j                  | dt        ¬«      }d|_        |j
                  dk7  rt        j                  d«       ŒM|j                  }d}t        j                  d|t        j                  «      }|r4t        j                  d	d|j                  d
«      «      j                  «       }|sHt        j                  d|«      }|r0|j                  d
«      j                  dd«      j                  d«      }d}t        j                  d|«      }|r‘|j!                  «       }d
}|}	|	t#        |«      dz
  k  rV|dkD  rQ||	|	dz    dk(  r||	dz      dv r||	|	dz    dk7  r|d
z  }||	|	dz    dk(  r|d
z  }|	d
z  }	|	t#        |«      dz
  k  r|dkD  rŒQ|||	dz
   j                  «       }|s|}| ||fc S  | ddfS #  |dk  rt        j                  d«       Y �ŒÛxY w)u7   æŠ“å�–è¯¦æƒ…é¡µ: æ ‡é¢˜(h1.c-h1) + æ­£æ–‡(div.c-detail)r   r   r    r#   r$   r%   r	   z.<h1[^>]*class="[^"]*c-h1[^"]*"[^>]*>(.*?)</h1>r&   r   z<title>(.*?)</title>u   å®�æ™‹åŽ¿äººæ°‘æ”¿åºœz -z)<div[^>]*class="[^"]*c-detail[^"]*"[^>]*>r   r   é   z<div)ú ú>ú
ú	úr.   r-   z<div/z</div>N)r/   r4   r5   r6   r7   r8   r9   r:   r=   r;   Úsearchr>   r?   Úgroupr@   ÚreplaceÚendrG   )
rJ   rK   rL   ÚhtmlrT   ÚmÚcontentÚstartÚdepthÚis
             r   Úfetch_detailrk   a   sô  € ä�Q‹xó *ˆð)	Ü—‘˜S¨"´gÔ>ˆAØ ˆAŒJØ�}‰} Ò#Ü—
‘
˜1”ØØ—6‘6ˆDð ˆEÜ—	‘	ÐKÈTÔSU×SWÑSWÓXˆAÙÜŸ™˜z¨2¨q¯w©w°q«zÓ:×@Ñ@ÓB�ÙÜ—I‘IÐ5°tÓ<�ÙØŸG™G A›J×.Ñ.Ð/FÈÓK×QÑQÐRVÓW�Eð ˆGÜ—	‘	ÐFÈÓMˆAÙØŸ™›�Ø�Ø�Øœ#˜d›) a™-Ò'¨E°AªIà˜A  A¡�¨&Ò0°T¸!¸a¹%±[ÐDeÑ5eØ  A¨¡E˜?¨gÒ5Ø! Q™J˜Eà˜A  A¡�¨(Ò2Ø ™
˜Ø˜‘F�Að œ#˜d›) a™-Ò'¨E°A«Ið ˜u q¨1¡uÐ-×3Ñ3Ó5�ÙØ�à˜ Ð(Ò(ðO*ðV ��tÐÐøð	Ø�QŠwÜ—
‘
˜1”ýs   ‘AGÁEGÆ!GÇG'c                  ó  — t         j                  j                  dd«      } t        dt        › d| › d�d¬«       t        «       }t        dt        |«      › d	�d¬«       |st        d
d¬«       y | dk(  rt        dd¬«       |d d }d }n)t        j                  t        «      }|j                  «       }d}d}t        t        ¬«      5 }i }|D ]H  \  }}	|	r$	 t        j                  |	d d «      }
|
t        k  rŒ+	 |j!                  t"        |«      }||	f||<   ŒJ t%        |«      D �]  }|j'                  «       \  }}}||   d   }	|dz  }|dz  dk(  rt        d|› dt        |«      › d�d¬«       |€t        d|› �d¬«       Œ]|s|}|	r|	d d nd}| dk(  rMt        d|› d|d d › d�«       t        d|› �«       t        dt        |«      › d�«       t        d|d d › �«       Œ¼	 j)                  d t        |||||df«       |j*                  dkD  r|dz  }|dz  dk(  sŒ÷|j/                  «        �Œ	 	 d d d «       | dk(  rt        d"t        «      › d#�«       y |j/                  «        j)                  d$t        f«       |j1                  «       \  }}}|j3                  «        t        d%d¬«       t        d&|› d	�d¬«       t        d'|› d(|› d)|› d*�d¬«       y #  Y �ŒáxY w# t,        $ r}t        d!|› �d¬«       Y d }~Œêd }~ww xY w# 1 sw Y   ŒÖxY w)+NÚMODEÚfullz=== z (mode=z) ===Tr*   u
     æ€»URL: u   æ�¡u     æ— æ•°æ�®Útestu   TESTæ¨¡å¼�: å�ªæŠ“å‰�2æ�¡éªŒè¯�r%   r   )Úmax_workersr'   r   r)   u	     è¯¦æƒ… r.   z...z	  [FAIL] r	   u   
  === æµ‹è¯•: [r(   z ===z  URL: u     æ­£æ–‡é•¿åº¦: u    å­—ç¬¦u     æ­£æ–‡é¢„è§ˆ: r$   z‚INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, date_rank) VALUES (?, ?, ?, ?, ?, ?, ?)z  [DB_ERROR] u   
=== æµ‹è¯•å®Œæˆ� (u   æ�¡) ===zTSELECT COUNT(*), MIN(publish_date), MAX(publish_date) FROM gov_raw WHERE site_name=?u   
=== å®Œæˆ� ===u
     æ–°å¢ž: u
     ç´¯è®¡: u   æ�¡ (z ~ ú))ÚosÚenvironr5   rE   Ú	SITE_NAMErY   rG   Úsqlite3Úconnectr   Úcursorr   ÚMAX_WORKERSr   rB   rC   Úsubmitrk   r   ÚresultÚexecuteÚrowcountrF   ÚcommitÚfetchoneÚclose)ÚmoderM   Úconnrw   Ú	total_newÚdoneÚexecutorÚfut_maprJ   rS   rW   ÚfutrT   rg   Úpub_daterX   ÚcntÚmin_dÚmax_ds                      r   Úmainr‹   ‘   sB  € Ü�:‰:�>‰>˜& &Ó)€DÜ	ˆD”�˜7 4 &¨Ð
.°dÕ;ô ‹N€EÜ	ˆJ”s˜5“z�l #Ð
&¨dÕ3ÙÜˆm 4Õ(Øàˆv‚~ÜÐ/°tÕ<Ø�b�q�	ˆØ‰ä�‰œwÓ'ˆØ—‘“ˆØ€IØ€Dä	¬Ô	4ð ,¸ØˆØ"ò 		+‰MˆC�ÙðÜ×*Ñ*¨8°C°R¨=Ó9�AØœ;’Ø ð 'ð —/‘/¤,°Ó4ˆCØ ˜?ˆG�CŠLð		+ô   Ó(ó 	ˆCØ"%§*¡*£,ÑˆC�˜Ø˜s‘| A‘ˆHØ�A‰IˆDØ�b‰y˜AŠ~Ü˜	 $  q¬¨W«¨°cÐ:À$ÕGØˆ}Ü˜	 # Ð'¨tÕ4ØÙØ�á(0�x  ‘}°bˆHà�vŠ~ÜÐ)¨(¨°2°e¸C¸R°j°\ÀÐFÔGÜ˜ ˜u�oÔ&ÜÐ(¬¨W«¨°gÐ>Ô?ÜÐ(¨°°#¨¨Ð8Ô9Øð7Ø—‘ð ]Ü  s¨G°X¸wÈÐJôð —?‘? QÒ&Ø ‘N�Ið �b‰y˜A‹~Ø—‘–ñ?	÷,ð\ ˆv‚~ÜÐ$¤S¨£\ N°(Ð;Ô<Øà‡K�K„MØ
‡N�NØ^Ü	ˆôð Ÿ™Ó)Ñ€Cˆ�Ø‡J�J„LÜ	Ð TÕ*Ü	ˆJ�y�k Ð
%¨TÕ2Ü	ˆJ�s�e˜5   s¨5¨'°Ð
3¸4Ö@øðgÚûô@ ò 7Ü˜ a SÐ)°×6Ò6ûð7ú÷S,ð ,úsU   ÃK?Ã!KÃ3C(K?Ç1KÈK?ÈK?ËKËK?Ë	K<Ë"K7Ë2K?Ë7K<Ë<K?Ë?LÚ__main__) Ú__doc__rr   ru   r;   r9   r   r4   r   r   Úconcurrent.futuresr   r   r   r
   Úgetaddrinfor   r   rA   r1   r2   rC   rs   r5   r   rt   rx   r6   r0   rY   rk   r‹   Ú__name__© r   r   ú<module>r’      sº   ðñ÷ /× .× .ß #ß ?ð
 €
Ø#€Ø×*Ñ*Ð ò>ð *€Ô à#€Ø<€Ø?€Ù�4˜˜BÓ€Ø
�*‰*�.‰.˜Ð$5Ó
6€Ø$€	Ø€àPØ/ñ€ð €	ò1òh-ò`PAðf ˆzÒÙ…Fð r   