Ë
    è‡zjÛ#  ã                   óL  — d Z ddlZddlmZ ddlZddlZddlZddlZddlmZ ej                  j                  j                  «        dZdZdZ ej                  ed¬	«        ej                   «       Zej$                  j'                  d
di«       d„ Zd„ Zd„ Zedk(  r e«        yy)u.  Crawler for tl.gov.cn - é“œé™µå¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€ä¿¡æ�¯å…¬å¼€-æœ€æ–°å…¬å¼€
CMS: æ”¿åºœä¿¡æ�¯å…¬å¼€é€šç”¨å¹³å�° (tl.gov.cn/openness)
Lists: showList/770/1948/page_{N}.html, 15æ�¡/é¡µ, <a class="biaoti-title"> (ç”¨titleå±žæ€§èŽ·å�–å®Œæ•´æ ‡é¢˜)
Detail: show/{id}.html, <div id="zoom"> æ­£æ–‡å�«è¡¨æ ¼HTML
é    N)ÚBeautifulSoup)Údatetimezhttps://www.tl.gov.cnz+/openness/OpennessContent/showList/770/1948z/root/gov_crawler/outputT)Úexist_okz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c                 ó†  — t         › t        › d| › d�}t        j                  |dd¬«      }d|_        t        |j                  d«      }g }|j                  d«      D ]ã  }|j                  d	«      }|sŒ|j                  d
d«      j                  «       }|j                  dd«      j                  «       }|s|j                  d¬«      }|j                  d«      }	|	r|	j                  d¬«      nd}
|sŒ•|sŒ˜|j                  d«      s&|j                  d«      r	t         |z   nt         dz   |z   }|j                  |||
dœ«       Œå |S )z1Extract article links and dates from a list page.z/page_z.htmlé   F©ÚtimeoutÚverifyúutf-8úhtml.parserzli[style*="height"]za.biaoti-titleÚhrefÚ ÚtitleT©ÚstripÚspanÚhttpú/)r   r   Údate)ÚBASEÚ	LIST_PATHÚsessionÚgetÚencodingr   ÚcontentÚselectÚ
select_oner   Úget_textÚ
startswithÚappend)Úpage_numÚurlÚrÚsoupÚarticlesÚliÚa_tagr   r   Ú	date_spanÚpub_dates              ú/root/gov_crawler/crawl_tl.pyÚget_articles_from_pager+      s,  € äˆF”9�+˜V H :¨UÐ
3€Cä�‰�C ¨EˆÓ2€AØ€A„JÜ˜Ÿ™ MÓ2€Dà€HØ�k‰kÐ/Ó0ò NˆØ—‘Ð.Ó/ˆÙØØ�y‰y˜ Ó$×*Ñ*Ó,ˆà—	‘	˜' 2Ó&×,Ñ,Ó.ˆÙØ—N‘N¨�NÓ.ˆEØ—M‘M &Ó)ˆ	Ù5>�9×%Ñ%¨DÐ%Ô1ÀBˆâ’EØ—?‘? 6Ô*Ø&*§o¡o°cÔ&:”t˜d’{ÄÀsÁ
ÈTÑ@Q�Ø�O‰O T°EÀ8ÑLÕMðNð" €Oó    c           
      óŽ  ‡‡‡— t         j                  | dd¬«      }d|_        t        |j                  d«      }d}|j                  ddd	i¬
«      }|r|j                  dd«      }|s%|j                  d«      }|r|j                  d¬«      }d}|j                  dddi¬
«      }|rM|j                  dd«      j                  «       }|r+	 t        j                  |dd d«      }|j                  d«      }d}	|j                  dddi¬
«      }|r|j                  dd«      }	d}
|j                  dddi¬
«      }|r|j                  dd«      }
d}|j                  d«      }|s|j                  d«      }|s|j                  d«      }|�r„t        |«      }g ŠdgŠˆˆfd„}t        j                  d||t        j                  t        j                   z  ¬«      }t        j                  dd|t        j                   ¬«      }t        j                  dd|t        j                   ¬«      }t        j                  dd|t        j                   ¬«      }t        |d«      }|j                  dd¬ «      }g }|j#                  d«      D ]{  }|j                  «       }|sŒt        j$                  d!|«      }|r=t'        |j)                  d"«      «      }|t+        ‰«      k  r|j-                  ‰|   «       Œk|j-                  |«       Œ} d#j/                  |«      }g }|j                  d$«      }|r¾|j1                  d%«      D ]ª  }|j                  d&d«      Š|j                  d¬«      xs d'}t        j2                  d(‰t        j4                  «      sŒQ‰j7                  d)«      s5‰j7                  d*«      r	t8        ‰z   n| j;                  d*d"«      d   d*z   ‰z   Š|j-                  |‰d+œ«       Œ¬ |j                  d,«      }|rÓ|j1                  d%«      D ]¿  }|j                  d&d«      Š|j                  d¬«      xs d'}t        j2                  d(‰t        j4                  «      sŒQ‰j7                  d)«      s5‰j7                  d*«      r	t8        ‰z   n| j;                  d*d"«      d   d*z   ‰z   Št=        ˆfd-„|D «       «      rŒ¬|j-                  |‰d+œ«       ŒÁ |rÓ|j1                  d%«      D ]¿  }|j                  d&d«      Š|j                  d¬«      xs d'}t        j2                  d(‰t        j4                  «      sŒQ‰j7                  d)«      s5‰j7                  d*«      r	t8        ‰z   n| j;                  d*d"«      d   d*z   ‰z   Št=        ˆfd.„|D «       «      rŒ¬|j-                  |‰d+œ«       ŒÁ |||	|
| ||rt?        j@                  |d¬/«      ndd0œ}|S #  |dd }Y �ŒÓxY w)1z9Extract metadata and content from an article detail page.r   Fr   r   r   r   ÚmetaÚnameÚArticleTitle)Úattrsr   z#titleTr   ÚPubDateNé
   z%Y-%m-%dÚContentSourceÚAuthorzdiv#zoomzdiv.m-scrollbarzdiv.g-detailboxr   c                 óx   •— | j                  d«      }‰j                  |«       ‰d   }‰dxx   dz  cc<   d|› d�S )Nr   é   z	
__TABLE_z__
)Úgroupr    )ÚmatchÚtbl_htmlÚidxÚtable_counterÚtabless      €€r*   Ú
save_tablez#extract_article.<locals>.save_tablep   sE   ø€ Ø—{‘{ 1“~ˆHØ�M‰M˜(Ô#Ø Ñ"ˆCØ˜!Ó Ñ!ÓØ ˜u DÐ)Ð)r,   z<table[^>]*>.*?</table>)Úflagsz	<br\s*/?>ú
z</p>z</div>)Ú	separatorr   z__TABLE_(\d+)__r7   z

zdiv.readFileListza[href]r   u   é™„ä»¶z"\.(doc|docx|pdf|xls|xlsx|zip|rar)$r   r   )r/   r"   zdiv.m-detail-downlistc              3   ó.   •K  — | ]  }|d    ‰k(  –— Œ y­w©r"   N© ©Ú.0ÚaÚa_hrefs     €r*   ú	<genexpr>z"extract_article.<locals>.<genexpr>«   ó   øè ø€ ÒC°!˜1˜U™8 vÕ-ÑCùó   ƒc              3   ó.   •K  — | ]  }|d    ‰k(  –— Œ y­wrC   rD   rE   s     €r*   rI   z"extract_article.<locals>.<genexpr>¶   rJ   rK   ©Úensure_ascii©r   r)   ÚsourceÚauthorr"   r   Úattachments)!r   r   r   r   r   Úfindr   r   r   r   ÚstrptimeÚstrftimeÚstrÚreÚsubÚDOTALLÚ
IGNORECASEÚsplitr9   Úintr8   Úlenr    Újoinr   ÚsearchÚIr   r   ÚrsplitÚanyÚjsonÚdumps)Ú
detail_urlr#   r$   r   r.   Ú	title_divr)   Údate_strÚdtrP   rQ   r   Úzoom_divÚhtml_contentr>   Úhtml_with_placeholdersÚ	text_soupÚcontent_textÚlinesÚlineÚplaceholder_matchr;   rR   Úread_file_divr'   Úa_textÚdownlistÚresultrH   r<   r=   s                               @@@r*   Úextract_articleru   8   s‹  ú€ ä�‰�J¨°5ˆÓ9€AØ€A„JÜ˜Ÿ™ MÓ2€Dð €EØ�9‰9�V F¨NÐ#;ˆ9Ó<€DÙØ—‘˜ BÓ'ˆÙØ—O‘O HÓ-ˆ	ÙØ×&Ñ&¨TÐ&Ó2ˆEð €HØ�9‰9�V F¨IÐ#6ˆ9Ó7€DÙØ—8‘8˜I rÓ*×0Ñ0Ó2ˆÙð)Ü×&Ñ& x°° }°jÓA�ØŸ;™; zÓ2�ð
 €FØ�9‰9�V F¨OÐ#<ˆ9Ó=€DÙØ—‘˜) RÓ(ˆð €FØ�9‰9�V F¨HÐ#5ˆ9Ó6€DÙØ—‘˜) RÓ(ˆð €GØ�‰˜zÓ*€HÙØ—?‘?Ð#4Ó5ˆÙØ—?‘?Ð#4Ó5ˆâä˜8“}ˆð ˆØ˜ˆõ	*ô "$§¡Ð(BÀJÐP\Ôdf×dmÑdmÔpr×p}Ñp}Ñd}Ô!~Ðô "$§¡¨°dÐ<RÔZ\×ZgÑZgÔ!hÐä!#§¡¨°Ð7MÔUW×UbÑUbÔ!cÐä!#§¡¨	°4Ð9OÔWY×WdÑWdÔ!eÐô "Ð"8¸-ÓHˆ	Ø ×)Ñ)°B¸eÐ)ÓDˆð ˆØ ×&Ñ& tÓ,ò 	ˆDØ—:‘:“<ˆDÙØä "§¡Ð);¸TÓ BÐÙ ÜÐ+×1Ñ1°!Ó4Ó5�Øœ˜V›Ò$Ø—L‘L ¨¡Ô-ØØ�L‰L˜Õð	ð —+‘+˜eÓ$ˆð €Kà—O‘OÐ$6Ó7€MÙØ"×)Ñ)¨)Ó4ò 	DˆEØ—Y‘Y˜v rÓ*ˆFØ—^‘^¨$�^Ó/Ò;°8ˆFÜ�y‰yÐ>ÀÌÏÉÕMØ×(Ñ(¨Ô0Ø.4×.?Ñ.?ÀÔ.DœT Fš]È*×J[ÑJ[Ð\_ÐabÓJcÐdeÑJfÐilÑJlÐouÑJu�FØ×"Ñ"¨F¸6Ñ#BÕCð	Dð �‰Ð6Ó7€HÙØ—_‘_ YÓ/ò 	HˆEØ—Y‘Y˜v rÓ*ˆFØ—^‘^¨$�^Ó/Ò;°8ˆFÜ�y‰yÐ>ÀÌÏÉÕMØ×(Ñ(¨Ô0Ø.4×.?Ñ.?ÀÔ.DœT Fš]È*×J[ÑJ[Ð\_ÐabÓJcÐdeÑJfÐilÑJlÐouÑJu�FÜÓC°{ÔCÕCØ×&Ñ&°¸vÑ'FÕGð	Hñ Ø—_‘_ YÓ/ò 	HˆEØ—Y‘Y˜v rÓ*ˆFØ—^‘^¨$�^Ó/Ò;°8ˆFÜ�y‰yÐ>ÀÌÏÉÕMØ×(Ñ(¨Ô0Ø.4×.?Ñ.?ÀÔ.DœT Fš]È*×J[ÑJ[Ð\_ÐabÓJcÐdeÑJfÐilÑJlÐouÑJu�FÜÓC°{ÔCÕCØ×&Ñ&°¸vÑ'FÕGð	Hð ØØØØØÙFQ”t—z‘z +¸EÕBÐWYñ€Fð €Møðc)Ø# C R˜=“ús   Ã*V: Ö:Wc                  ó:  — t        t        j                  «      dkD  rt        t        j                  d   «      nd} g }t	        d| dz   «      D ]u  }t        d|› d�t        j                  ¬«       t        |«      }t        dt        |«      › d�t        j                  ¬«       |j                  |«       t        |«      dk  sŒu n t        d	t        |«      › �t        j                  ¬«       g }t        |«      D ]v  \  }}t        d
|dz   › dt        |«      › d|d   d d › d�t        j                  ¬«       	 t        |d   «      }|d   s|d   |d<   |d   s|d   |d<   |j                  |«       Œx t        j                  «       j                  d«      }	t         j"                  j%                  t&        d|	› d�«      }
t)        |
dd¬«      5 }|D ]+  }|j+                  t-        j.                  |d¬«      dz   «       Œ- 	 d d d «       t        d|
› �t        j                  ¬«       t        dt        |«      › �«       y # t        $ rK}t        d|› �t        j                  ¬«       |j                  |d   |d   dd|d   dddœ«       Y d }~�Œ‘d }~ww xY w# 1 sw Y   Œ–xY w) Nr7   é   zFetching page z...)Úfilez  Found z	 articlesé   zTotal articles to process: z  [r   z] r   é(   r   r)   r   z	  ERROR: r   rO   z%Y%m%d_%H%M%SÚtl_tzgg_z.jsonlÚwr   )r   FrM   r@   zOutput: zTotal items: )r]   ÚsysÚargvr\   ÚrangeÚprintÚstderrr+   ÚextendÚ	enumerateru   r    Ú	Exceptionr   ÚnowrU   ÚosÚpathr^   Ú
OUTPUT_DIRÚopenÚwriterc   rd   )Ú	max_pagesÚall_articlesÚpgr%   ÚresultsÚiÚartÚdataÚeÚ	timestampÚoutput_fileÚfr#   s                r*   Úmainr–   Ä   sh  € Ü$'¬¯©£M°AÒ$5””C—H‘H˜Q‘KÔ ¸1€Ið €LÜ�A�y 1‘}Ó%ò ˆÜ�˜r˜d #Ð&¬S¯Z©ZÕ8Ü)¨"Ó-ˆÜ�œ˜X›˜ yÐ1¼¿
¹
ÕCØ×Ñ˜HÔ%Üˆx‹=˜2ÓÙðô 
Ð'¬¨LÓ(9Ð':Ð
;Ä#Ç*Á*ÕMð €GÜ˜LÓ)ò ‰ˆˆ3Ü��A�a‘C�5˜œ#˜lÓ+Ð,¨B¨s°7©|¸C¸RÐ/@Ð.AÀÐEÌCÏJÉJÕWð	Ü" 3 v¡;Ó/ˆDà˜’=Ø # G¡��W‘Ø˜
Ò#Ø#& v¡;��ZÑ Ø�N‰N˜4Õ ðô0 —‘“×'Ñ'¨Ó8€IÜ—'‘'—,‘,œz¨X°i°[ÀÐ+GÓH€KÜ	ˆk˜3¨Ô	1ð >°QØò 	>ˆAØ�G‰G”D—J‘J˜q¨uÔ5¸Ñ<Õ=ñ	>÷>ô 
ˆH�[�MÐ
"¬¯©Õ4Ü	ˆMœ#˜g›,˜Ð
(Õ)øô+ ò 	Ü�I˜a˜S�/¬¯
©
Õ3à�N‰NØ˜W™Ø ™KØØØ˜6‘{ØØ!ñ÷ ò ûð	ú÷ >ð >ús%   Ä69H:Ç
1JÈ:	JÉA J	Ê	JÊJÚ__main__)Ú__doc__ÚrequestsÚbs4r   rc   rW   r}   r†   r   ÚpackagesÚurllib3Údisable_warningsr   r   rˆ   ÚmakedirsÚSessionr   ÚheadersÚupdater+   ru   r–   Ú__name__rD   r,   r*   ú<module>r£      s®   ðñó Ý Û Û 	Û 
Û 	Ý à × Ñ × Ñ × *Ñ *Ô ,à€Ø9€	Ø'€
Ø €‡�ˆJ Õ &à
ˆ(×
Ñ
Ó
€Ø ‡�× Ñ Øð  Dðô òò8JòX0*ðd ˆzÒÙ…Fð r,   