Ë
    ¸¬ujÌ  ã                   óò   — d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
m
Z
 ddlmZ ddlmZmZ e	j                   e	_        dZdZdZd	Zd
ZdZdedz   dœZd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uP   Crawl aax.gov.cn - æ˜‚æ˜‚æºªåŒºå…¬ç¤ºå…¬å‘Š (API + concurrent detail extraction)é    N)Údatetime)Úurljoin)ÚThreadPoolExecutorÚas_completedzhttps://www.aax.gov.cnz>https://www.aax.gov.cn/search/1cc659753cd94d769c4692ff6fa7ca61Úaax_tzggz/root/search.dbé   é   z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36z/aax/c102254/list.shtml)z
User-AgentÚRefererc                 ó>  — t         › dt        › d| › �}t        j                  j	                  |t
        ¬«      }t        j                  j                  |d¬«      }t        j                  |j                  «       j                  dd¬«      «      }|j                  d	i «      S )
Nz%?_isAgg=false&_isJson=true&_pageSize=z3&_template=index&_rangeTimeGte=&_channelName=&page=©Úheadersé   ©Útimeoutúutf-8Úreplace©ÚerrorsÚdata)ÚAPI_URLÚ	PAGE_SIZEÚurllibÚrequestÚRequestr   ÚurlopenÚjsonÚloadsÚreadÚdecodeÚget)ÚpageÚurlÚreqÚrespr   s        ú/root/gov_crawler/crawl_aax.pyÚ	fetch_apir&      sƒ   € ÜˆIÐ:¼9¸+ÐExÐy}Ðx~Ð
€CÜ
�.‰.×
 Ñ
  ¬gÐ
 Ó
6€CÜ�>‰>×!Ñ! #¨rÐ!Ó2€DÜ�:‰:�d—i‘i“k×(Ñ(¨¸Ð(ÓCÓD€DØ�8‰8�F˜BÓÐó    c                 ó  — g }	 t         j                  j                  | t        ¬«      }t         j                  j	                  |d¬«      }|j                  «       j                  dd¬«      }d}d|z   d	z   }t        j                  ||t        j                  «      D ]Ö  }t        | |«      }t        j                  d
t        j                  |«      z   dz   |t        j                  t        j                  z  «      }	d}
|	r4t        j                  dd|	j                  d«      «      j!                  «       }
|
s'|j#                  d«      d   j%                  dd«      d   }
|j'                  d|
› d|› d�«       ŒØ 	 |S # t(        $ r}Y d}~|S d}~ww xY w)z/Fetch detail page and extract attachment links.r   r   r   r   r   r   z!(?:pdf|doc|docx|xls|xlsx|zip|rar)zhref=["\']([^"\']+\.z)["\']z<a[^>]*href=["\'][^"\']*z["\'][^>]*>(.*?)</a>Ú z<[^>]+>é   ú/éÿÿÿÿú.r   ú[z](ú)N)r   r   r   r   r   r   r   ÚreÚfindallÚIr   ÚsearchÚescapeÚDOTALLÚsubÚgroupÚstripÚsplitÚrsplitÚappendÚ	Exception)r"   Úattachmentsr#   r$   ÚhtmlÚext_patternÚhref_patternÚhrefÚfull_urlÚa_matchÚtextÚes               r%   Úfetch_detailrF      si  € à€KðÜ�n‰n×$Ñ$ S´'Ð$Ó:ˆÜ�~‰~×%Ñ% c°2Ð%Ó6ˆØ�y‰y‹{×!Ñ! '°)Ð!Ó<ˆà:ˆØ.°Ñ<¸yÑHˆÜ—J‘J˜|¨T´2·4±4Ó8ò 	8ˆDÜ˜s DÓ)ˆHÜ—i‘iÐ ;¼b¿i¹iÈ»oÑ MÐPgÑ gÐimÔoq×osÑosÔvx×vÑvÑoó  AˆGØˆDÙÜ—v‘v˜j¨"¨g¯m©m¸AÓ.>Ó?×EÑEÓG�ÙØ—z‘z #“ rÑ*×1Ñ1°#°qÓ9¸!Ñ<�Ø×Ñ  4 &¨¨8¨*°AÐ6Õ7ñ	8ð Ðøô ò ÛØÐûðús   „E/E6 Å6	F
ÆF
c                 óª  — | j                  dd«      j                  «       }| j                  dd«      j                  «       }| j                  dd«      }|r%|j                  d«      sd|j                  d«      z   }| j                  dd«      }|r|dd	 nd}|sy|r4t	        |«      }|r'|j                  «       }|d
dj                  |«      z   z  }||||dœS )zDProcess a single entry: extract metadata + fetch detail attachments.Útitler)   Úcontentr"   Úhttpzhttp://ÚpublishedTimeStrNé
   u   

**é™„ä»¶ï¼š**
ú
)rH   r"   ÚdaterI   )r    r8   Ú
startswithÚlstriprF   ÚrstripÚjoin)ÚentryrH   rI   r"   Úpub_timerN   r=   s          r%   Úprocess_entryrU   7   sÑ   € à�I‰I�g˜rÓ"×(Ñ(Ó*€EØ�i‰i˜	 2Ó&×,Ñ,Ó.€GØ
�)‰)�E˜2Ó
€CÙ
�3—>‘> &Ô)Ø˜#Ÿ*™* YÓ/Ñ/ˆØ�y‰yÐ+¨RÓ0€HÙ$ˆ8�C�R‰=¨"€DáØñ Ü" 3Ó'ˆÙØ—n‘nÓ&ˆGØÐ,¨t¯y©y¸Ó/EÑEÑEˆGà 3°ÀÑIÐIr'   c                 ó  — | syt        j                  t        «      }|j                  d«       |j	                  «       }d}| D ]«  }|j                  d|d   f«      j                  «       rŒ(|j                  dt        |d   |d   |d   |d   |d   r0t        t        j                  |d   d«      j                  «       «      nd|d	   r|d	   d d
 nd|d	   f«       |j                  «        |dz  }Œ­ |j                  «        |S )Nr   zPRAGMA busy_timeout=30000z)SELECT id FROM gov_raw WHERE source_url=?r"   zŒINSERT INTO gov_raw (site_name, source_url, page_url, title, publish_date, date_rank, summary, content)
            VALUES (?,?,?,?,?,?,?,?)rH   rN   z%Y-%m-%drI   éÈ   r)   r*   )Úsqlite3ÚconnectÚDB_PATHÚexecuteÚcursorÚfetchoneÚ	SITE_NAMEÚintr   ÚstrptimeÚ	timestampÚcommitÚclose)ÚrecordsÚconnÚcÚcountÚrs        r%   Ú
save_to_dbri   N   s  € ÙØÜ�?‰?œ7Ó#€DØ‡L�LÐ,Ô-Ø�‰‹€AØ€EØò 	ˆØ�9‰9Ð@À1ÀUÁ8À+ÓN×WÑWÔYØØ	�	‰	ð (ä˜a ™h¨¨%©°!°G±*¸aÀ¹iØPQÐRXÒPY”3”x×(Ñ(¨¨6©°JÓ?×IÑIÓKÔLÐ_`Ø)*¨9ª�1�Y‘<  Ñ%¸2¸qÀ¹|ðMô	Nð
 	�‰ŒØ�‰
‰ð	ð 	‡J�J„LØ€Lr'   c                  óR  ‡— t        dt        › d�«       t        d«      } | j                  dd«      }| j                  d| j                  dg «      «      }|st        d«       y t        d	|› d
�«       t	        |«      }t        dt        dz   «      D ]U  }	 t        |«      } | j                  d| j                  dg «      «      }|s n#|j                  |«       t        |«      |k\  r nŒW t        dt        |«      › �«       t        «       }|D �cg c];  }|j                  dd«      |v rŒ|j                  |j                  dd«      «      rŒ:|‘Œ= c}Št        dt        ‰«      › �«       g }t        d¬«      5 }t        ‰«      D �	�
ci c]  \  }	}
|j                  t        |
«      |	“Œ }}	}
t!        |«      D ]o  }||   }|j#                  «       }|r8|j%                  |«       t        d|dz   › dt        ‰«      › d|d   d d › �«       ŒRt        d|dz   › dt        ‰«      › d�«       Œq 	 d d d «       |j'                  ˆfd„¬«       t)        |«      }t        d|› dt        |«      › d�«       y # t        $ r}t        d|› d|› �«       Y d }~ �Œ£d }~ww xY wc c}w c c}
}	w # 1 sw Y   ŒyxY w)Nz=== Crawling z ===r*   Útotalr   ÚresultsÚlistzERROR: No results from APIzTotal: z	 articlesé   zPage z
: error - zTotal entries: r"   r)   zUnique: rL   )Úmax_workersr.   r+   z] rH   é2   z] SKIPc                 óx   •‡ — t        ˆ fd„‰D «       «      r#‰j                  t        ˆ fd„‰D «       «      «      S dS )Nc              3   óL   •K  — | ]  }|j                  d «      ‰d    k(  –— Œ y­w©r"   N©r    ©Ú.0rE   rh   s     €r%   ú	<genexpr>z)main.<locals>.<lambda>.<locals>.<genexpr>’   sC   øè ø€ ò  jTð  HIÐjk×joÑjoÐpuÓjvÐz{ð  }Bñ  {Cõ  kCñ  jTùs   ƒ!$c              3   óR   •K  — | ]  }|j                  d «      ‰d    k(  sŒ|–— Œ  y­wrs   rt   ru   s     €r%   rw   z)main.<locals>.<lambda>.<locals>.<genexpr>’   s'   øè ø€ Ò2a¸ÈÏÉÈeËÐXYÐZ_ÑX`ÓH`´1Ñ2aùs   ƒ' 'r   )ÚanyÚindexÚnext)rh   Úuniques   `€r%   ú<lambda>zmain.<locals>.<lambda>’   sC   ù€ Ôfió  jTð  MSô  jTô  gT §¡¬dÓ2a¸fÔ2aÓ.aÓ!b€ ð  Z[€ r'   )Úkeyz
=== Complete: z new records ===)Úprintr^   r&   r    rm   ÚrangeÚ	MAX_PAGESÚextendÚlenr<   ÚsetÚaddr   Ú	enumerateÚsubmitrU   r   Úresultr;   Úsortri   )r   rk   rl   Úall_entriesr!   rE   ÚseenÚ	processedÚexecutorÚirS   ÚfuturesÚfutureÚidxrˆ   Úsavedr|   s                   @r%   Úmainr“   c   sŸ  ø€ Ü	ˆMœ)˜ DÐ
)Ô*ô �Q‹<€DØ�H‰H�W˜aÓ €EØ�h‰h�y $§(¡(¨6°2Ó"6Ó7€GÙÜÐ*Ô+ØÜ	ˆG�E�7˜)Ð
$Ô%ä�w“-€KÜ�aœ Q™Ó'ò ˆð
	Ü˜T“?ˆDØ—h‘h˜y¨$¯(©(°6¸2Ó*>Ó?ˆGÙÙØ×Ñ˜wÔ'Ü�;Ó 5Ò(Ùð )ðô 
ˆOœC Ó,Ð-Ð
.Ô/ô ‹5€DØ$Öe�A¨Q¯U©U°5¸"Ó-=ÀÒ-EÈÏÉÐRS×RWÑRWÐX]Ð_aÓRbÕIcŠaÒe€FÜ	ˆH”S˜“[�MÐ
"Ô#ð €IÜ	¨Ô	+ð 	7¨xÜLUÐV\ÓL]×^ÁÀÀ5�8—?‘?¤=°%Ó8¸!Ñ;Ð^ˆÑ^Ü" 7Ó+ò 	7ˆFØ˜&‘/ˆCØ—]‘]“_ˆFÙØ× Ñ  Ô(Ü˜˜#˜a™%˜ ¤# f£+ ¨b°¸±ÀÀ"Ð1EÐ0FÐGÕHä˜˜#˜a™%˜ ¤# f£+ ¨fÐ5Õ6ñ	7÷	7ð ‡N�Nó  [€Nô  \ô �yÓ!€EÜ	Ð˜U˜G 1¤S¨£^Ð$4Ð4DÐ
EÕFøô; ò 	Ü�E˜$˜˜z¨!¨Ð-Ô.Þûð	üò fùó _÷	7ð 	7úsO   Â/I)ÃI)ÄJÄ&!JÅJÅ4JÆ!JÆ$B JÉ)	JÉ2J
Ê
JÊJÊJ&Ú__main__)Ú__doc__Úsysr0   r   rX   Úurllib.requestr   Úurllib.errorÚ	tracebackÚsslr   Úurllib.parser   Úconcurrent.futuresr   r   Ú_create_unverified_contextÚ_create_default_https_contextÚBASEr   r^   rZ   r   r�   r   r&   rF   rU   ri   r“   Ú__name__© r'   r%   ú<module>r¢      s�   ðá Vß K× K× K× KÝ Ý  ß ?à$'×$BÑ$B€Ô !à€Ø
J€Ø€	Ø
€Ø€	Ø€	ð QØÐ/Ñ/ñ€ò òò0Jò.ò*3Gðl ˆzÒÙ…Fð r'   