Ë
    ¸¬uj$  ã                   óF  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
m
Z
mZ ddlmZmZ  ej                  dd«      Zd	Zd
Ze› d�ZddddddœZdde› d�ddœdde› d�ddœdde› d�ddœgZdd d!œZd"Zd#Zd$„ Zd%„ Zd&„ Zd'„ Zd(„ Zd)„ Zd*„ Zd+„ Z e!d,k(  r e «        yy)-uæ  
crawl_jinan_epb_new.py â€” æµŽå�—å¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€Â·æ–°æ �ç›®
=============================================
CMS: JCMS (å¤§æ±‰ç‰ˆé€š)ï¼ŒAPI é©±åŠ¨

æ �ç›®æ¸…å�•ï¼š
  col32954 â†’ åŽ†åŸŽåˆ†å±€Â·æ‹Ÿå®¡æŸ¥é¡¹ç›®å…¬ç¤º
  col32967 â†’ å•†æ²³åˆ†å±€Â·é¡¹ç›®å�—ç�†å…¬ç¤º
  col10489 â†’ å¸‚å±€ï¼ˆæœ¬çº§ï¼‰Â·é¡¹ç›®å�—ç�†å…¬ç¤º

ç”¨æ³•:
    python3 crawl_jinan_epb_new.py               # å…¨é‡�ï¼ˆè·‘å…¨éƒ¨3ä¸ªæ �ç›®ï¼‰
    python3 crawl_jinan_epb_new.py 1             # å¢žé‡�ï¼ˆä»…æœ€æ–°é¡µï¼‰
é    N)ÚBeautifulSoup)Úurljoin)ÚdatetimeÚ	timedelta)ÚThreadPoolExecutorÚas_completedÚ	SEARCH_DBz/root/search.dbzhttps://jnepb.jinan.gov.cnu   æµŽå�—å¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€z7/api-gateway/jpaas-publish-server/front/page/build/unitÚbulidstaticÚ21ÚpMB9pkaz19HYYbLAOTZtYÚcolumnu   ä¿¡æ�¯åˆ—è¡¨)Ú	parseTypeÚwebIdÚtplSetIdÚpageTypeÚtagIdÚ32954u#   æ‹Ÿå®¡æŸ¥é¡¹ç›®å…¬ç¤ºÂ·åŽ†åŸŽåˆ†å±€z/col/col32954/index.htmlz[jn_lc_nsc])ÚpageIdÚcolumn_nameÚrefererÚlabelÚ32967u    é¡¹ç›®å�—ç�†å…¬ç¤ºÂ·å•†æ²³åˆ†å±€z/col/col32967/index.htmlz[jn_sh_slgs]Ú10489u   é¡¹ç›®å�—ç�†å…¬ç¤ºÂ·å¸‚å±€z/col/col10489/index.htmlz[jn_sj_slgs]zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zapplication/json)z
User-AgentÚAccepté   é2   c                 ó$   — t        | › d|› �«       y )Nú )Úprint)r   Úmsgs     ú(/root/gov_crawler/crawl_jinan_epb_new.pyÚlogr"   I   s   € Ü	ˆUˆG�1�S�EÐ
Õó    c                 ó¸  — t        j                  | t        dœd¬«      }t        t        «      }|d   |d<   ||d<   t        t
        «      }|d   |d<   	 t        j                  t        ||d¬	«      }|j                  «       }|j                  d
i «      j                  dd«      }g }	d}
t        j                  d|«      }|rt        |j                  d«      «      }
t        j                  d|«      D ]¸  }|j                  d«      }|j                  d«      j!                  «       }||j#                  «       |j%                  «       dz    }t        j                  d|«      }|r|j                  d«      nd}|j'                  d«      r|nt)        t*        |«      }|	j-                  |||f«       Œº |	|
fS # t        $ r#}t        |d   d| › d|› �«       g dfcY d}~S d}~ww xY w)z@Fetch one page of article list from API. Returns (items, total).)ÚpageNoÚpageSizeF)Úensure_asciir   Ú	paramJsonr   ÚRefereré   )ÚheadersÚparamsÚtimeoutÚdataÚhtmlÚ r   zERROR API page ú: r   Nzcount="(\d+)"é   z8<a\s+href="([^"]+)"\s+title="([^"]+)"\s+target="_blank">é   éÈ   z\[(\d{4}-\d{2}-\d{2})\]Úhttp)ÚjsonÚdumpsÚ	PAGE_SIZEÚdictÚAPI_BASEÚHEADERS_TEMPLATEÚrequestsÚgetÚAPI_URLÚ	Exceptionr"   ÚreÚsearchÚintÚgroupÚfinditerÚstripÚstartÚendÚ
startswithr   ÚBASE_URLÚappend)Úpage_noÚcolÚ
param_jsonr,   r+   Úrespr.   r/   ÚeÚitemsÚtotalÚmÚhrefÚtitleÚentryÚdate_mÚdate_strÚfull_urls                     r!   Úfetch_articles_pagerY   M   s³  € ä—‘ w¼IÑFÐUZÔ[€JÜ”(‹^€FØ˜8‘}€Fˆ8ÑØ$€Fˆ;ÑäÔ#Ó$€GØ˜Y™€GˆIÑðÜ�|‰|œG¨W¸VÈRÔPˆØ�y‰y‹{ˆØ�x‰x˜ Ó#×'Ñ'¨°Ó3ˆð
 €EØ€Eä
�	‰	Ð" DÓ)€AÙÜ�A—G‘G˜A“J“ˆä�[‰[ØCØóò 2ˆð �w‰w�q‹zˆØ—‘˜“
× Ñ Ó"ˆà�Q—W‘W“Y˜qŸu™u›w¨™}Ð-ˆÜ—‘Ð5°uÓ=ˆÙ&,�6—<‘< ”?°"ˆàŸ?™?¨6Ô2‘4¼ÄÈ$Ó8OˆØ�‰�h  xÐ0Õ1ð2ð �%ˆ<Ðøô3 ò ÜˆC�‰L˜O¨G¨9°B°q°cÐ:Ô;Ø�1ˆu�ûðús   ÁAF- Æ-	GÆ6GÇGÇGc                 óÞ  — 	 t        j                  | |d¬«      }d|_        |j                  }t        |d	«      }|j                  d
ddi¬«      }|r$|j                  d«      r|d   j                  «       nd}d}|j                  d
ddi¬«      }	|	rK|	j                  d«      r:t        j                  d|	d   j                  «       «      }
|
r|
j                  d«      }|j                  dd„ ¬«      }|s-|j                  dd¬«      }|r|j                  d¬«      }|r|n|}|rt        |«      nd}|st        dd| › �«       d||fS |||fS # t        $ r}t        dd| › d|› �«       Y d}~yd}~ww xY w)z=Return (content_html, date_str, title) or (None, None, None).r*   )r+   r-   zutf-8z[detail]zERROR r1   N©NNNzhtml.parserÚmetaÚnameÚArticleTitle)ÚattrsÚcontentr0   ÚPubDatez(\d{4}-\d{2}-\d{2})r2   Údivc                 óT   — | xr% dt        | t        «      r| ndj                  | «      v S )NÚarticler   )Ú
isinstanceÚstrÚjoin)Úcs    r!   ú<lambda>zfetch_detail.<locals>.<lambda>’   s$   € ˜ÒV˜y´*¸QÄÔ2D©QÈ#Ï(É(ÐSTË+ÐV€ r#   )Úclass_zbt-article-02Úzoom)ÚidzWARNING: no content for )r<   r=   ÚencodingÚtextr?   r"   r   ÚfindrE   r@   ÚmatchrC   rf   )Úurlr+   rN   r/   rO   ÚsoupÚ
meta_titlerT   rW   Ú	meta_daterR   Úcontent_divÚ
bt_articlerk   Úcontent_htmls                  r!   Úfetch_detailrx   w   s†  € ð Ü�|‰|˜C¨¸"Ô=ˆØˆŒØ�y‰yˆô
 ˜˜}Ó-€Dð —‘˜6¨&°.Ð)A�ÓB€JÙ-7¸J¿N¹NÈ9Ô<UˆJ�yÑ!×'Ñ'Ô)Ð[]€Eð €HØ—	‘	˜&¨°Ð(;�	Ó<€IÙ�Y—]‘] 9Ô-Ü�H‰HÐ+¨Y°yÑ-A×-GÑ-GÓ-IÓJˆÙØ—w‘w˜q“zˆHð —)‘)ØÙVð ó €Kñ Ø—Y‘Y˜u¨_�YÓ=ˆ
ÙØ—?‘? f�?Ó-ˆDÙ"&™$¨JˆKÙ'2”3�{Ô#¸€LáÜˆJÐ2°3°%Ð8Ô9Ø�X˜uÐ$Ð$à˜ 5Ð(Ð(øôE ò  ÜˆJ˜&   R¨ sÐ+Ô,Üûð ús   ‚+E Å	E,ÅE'Å'E,c           	      ó&  — i }t        t        ¬«      5 }| D �ci c]  }|j                  t        ||«      |“Œ }}t	        |«      D ]  }||   }	 |j                  «       ||<   Œ 	 d d d «       |S c c}w # t        $ r d||<   Y Œ>w xY w# 1 sw Y   |S xY w)N)Úmax_workersr[   )r   ÚMAX_WORKERSÚsubmitrx   r   Úresultr?   )Úurlsr+   ÚresultsÚexrq   Úfut_mapÚfuts          r!   Úfetch_detail_batchrƒ   ¢   s¥   € Ø€GÜ	¬Ô	4ð 2¸ØIMÖNÀ#�2—9‘9œ\¨3°Ó8¸#Ñ=ÐNˆÐNÜ Ó(ò 	2ˆCØ˜#‘,ˆCð2Ø"Ÿz™z›|�˜’ñ	2÷2ð €Nùò Oøô
 ò 2Ø1�˜“ð2ú÷2ð €Nús?   “B˜A-·BÁA2Á BÁ-BÁ2BÂ BÂBÂBÂBc           
      óN  — | syddl }|j                  t        «      }|j                  «       }d}| D ])  \  }}}}		 |j	                  dt
        ||||	|f«       |dz  }Œ+ |j                  «        |j                  «        |S # t        $ r}
t        dd|› d|
› �«       Y d}
~
Œnd}
~
ww xY w)z3items: list of (url, title, date_str, content_html)r   NzœINSERT OR REPLACE INTO gov_raw
                   (site_name, page_url, title, publish_date, content, category)
                   VALUES (?, ?, ?, ?, ?, ?)r2   z[db]zDB error for r1   )
Úsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚ	SITE_NAMEr?   r"   ÚcommitÚclose)rP   r   r…   Úconnrh   Úinsertedrq   rT   rW   rw   rO   s              r!   Ú
save_to_dbr�   ¯   s¹   € áØÛØ�?‰?œ7Ó#€DØ�‰‹€AØ€HØ.3ò 
4Ñ*ˆˆU�H˜lð		4Ø�I‰Ið0ô ˜C ¨°,ÀÐLô	ð ˜‰M‰Hð
4ð 	‡K�K„MØ‡J�J„LØ€Oøô	 ò 	4Ü�˜-¨ u¨B¨q¨cÐ2×3Ñ3ûð	4ús   º!A?Á?	B$ÂBÂB$c                 óè   — 	 dd l }|j                  t        «      }|j                  d| f«       |j	                  «        |j                  «        y# t        $ r}t        dd|› �«       Y d }~yd }~ww xY w)Nr   a  INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
               SELECT rowid, title, site_name,
                      CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
               FROM gov_raw WHERE category = ?Tz[fts]zFTS rebuild note: F)r…   r†   r‡   r‰   r‹   rŒ   r?   r"   )r   r…   r�   rO   s       r!   Úrebuild_ftsr‘   Ç   sl   € ðÛØ�‰œwÓ'ˆØ�‰ð2ð ˆNô	
ð 	�‰ŒØ�
‰
ŒØøÜò ÜˆGÐ)¨!¨Ð-Ô.Üûðús   ‚AA Á	A1ÁA,Á,A1c                 óZ  — | d   }| d   }t        j                  «       t        d¬«      z
  j                  d«      }t	        t
        «      }| d   |d<   t        |d|› d	| d
   › d�«       t        |d|› �«       g }d}d}	 t        || «      \  }	}
|dk(  r|
}t        |d|› �«       |	s|dk(  rt        |d«       yn�|	D ���cg c]  \  }}}|sŒ
||k\  sŒ|||f‘Œ }}}}|j                  |«       |rt        |dt        |«      › d�«       n=|r(t        |«      t        |	«      k  rt        |d|› d�«       n|t        z  |k\  rn|dz  }ŒÇt        |dt        |«      › �«       |syg }t        dz  }t        dt        |«      |«      D ]Ÿ  }||||z    }|D ��cg c]  \  }}}|‘Œ
 }}}t        |d|dz   › d|t        |«      z   › dt        |«      › d�«       t        ||«      }|D ]A  \  }}}|j                  |d«      \  }}}|xs |}|xs |}|j                  ||||xs df«       ŒC Œ¡ t!        ||«      }t        |d|› dt        |«      › �«       |rBt#        |D �cg c]  }|d   sŒ	|d   ‘Œ c}«      } t        |d| r| d   nd › d!| r| d"   nd › �«       t%        |«       |S c c}}}w c c}}w c c}w )#Nr   r   iG  )Údaysz%Y-%m-%dr   r)   z=== z	 (pageId=r   z) ===z3-year cutoff: r   r2   zExpected total: zNo articles foundzIncremental: page 1, z	 articleszPage z$: some items before cutoff, stoppingzArticles to process: é   z  Details [ú-ú/z]...r[   r0   zSaved: r3   zDate range: zN/Az ~ éÿÿÿÿ)r   Únowr   Ústrftimer9   r;   r"   rY   ÚextendÚlenr8   r{   Úrangerƒ   r=   rJ   r�   Úsortedr‘   )!rL   Úincrementalr   Úcol_nameÚthree_years_agor+   Úall_articlesÚtotal_expectedÚpagerP   rQ   ÚuÚtÚdÚfilteredr   Ú
batch_sizeÚbatch_startÚbatchÚ_Ú
batch_urlsÚ
detail_maprq   rT   rW   rw   Údetail_dateÚdetail_titleÚfinal_titleÚ
final_dateÚsavedÚrÚdatess!                                    r!   Úcrawl_columnrµ   Ú   s$  € Ø�‰L€EØ�=Ñ!€HÜ—|‘|“~¬	°wÔ(?Ñ?×IÑIÈ*ÓU€OäÔ#Ó$€GØ˜Y™€GˆIÑäˆ��h�Z˜y¨¨X©¨°uÐ=Ô>Üˆ� Ð 1Ð2Ô3ð €LØ€NØ€Dà
Ü*¨4°Ó5‰ˆˆuØ�1Š9Ø"ˆNÜ�Ð)¨.Ð)9Ð:Ô;áØ�qŠyÜ�EÐ.Ô/ØØà-2×QÐQ¡' ! Q¨²a¸AÀÓ<P�Q˜˜1’IÐQˆÒQØ×Ñ˜HÔ%áÜ�Ð.¬s°8«}¨o¸YÐGÔHØáœ˜H›¬¨E«
Ò2Ü�˜˜t˜fÐ$HÐIÔJØà”)Ñ˜~Ò-Øà�‰	ˆð5 ô8 ˆÐ&¤s¨<Ó'8Ð&9Ð:Ô;áØð €GÜ˜q‘€JÜ˜Q¤ LÓ 1°:Ó>ò 	OˆØ˜[¨°zÑ)AÐBˆØ',×-™G˜A˜q !’aÐ-ˆ
Ñ-ÜˆE�[ ¨Q¡ ¨q°¼SÀ»ZÑ1GÐ0HÈÌ#ÈlÓJ[ÐI\Ð\`ÐaÔbÜ'¨
°GÓ<ˆ
Ø$)ò 	OÑ ˆC�˜Ø6@·n±nÀSÐJ\Ó6]Ñ3ˆL˜+ |Ø&Ò/¨%ˆKØ$Ò0¨ˆJØ�N‰N˜C ¨j¸,Ò:LÈ"ÐMÕNñ		Oð	Oô �w Ó)€EÜˆ�˜˜˜q¤ W£ Ð/Ô0áÜ gÖ6 °°1³˜˜!›Ò6Ó7ˆÜˆE�\©e %¨¢(¸Ð!?¸sÑPUÀ5ÈÂ9Ð[`ÐBaÐbÔcä�ÔØ€LùôY Rùó4 .ùò 7s$   Â>JÃ
JÃJÆJ"É
J(É J(c                  óà   — t        t        j                  «      dkD  xr t        j                  d   dk(  } d}t        D ]  }t	        || «      }||z  }Œ t        dd› �«       t        d|› �«       y )Nr2   Ú1r   ú
z2==================================================z'[jinan_epb_new] All done! Total saved: )r›   ÚsysÚargvÚCOLUMNSrµ   r   )rž   Útotal_savedrL   r²   s       r!   Úmainr½   %  sp   € Ü”c—h‘h“- !Ñ#Ò:¬¯©°©°sÑ(:€KØ€Käò ˆÜ˜S +Ó.ˆØ�uÑ‰ðô 
ˆBˆxˆjˆ/ÔÜ	Ð3°K°=Ð
AÕBr#   Ú__main__)"Ú__doc__Úosr@   r¹   r6   r<   Úbs4r   Úurllib.parser   r   r   Úconcurrent.futuresr   r   Úgetenvr‡   rI   rŠ   r>   r:   r»   r;   r{   r8   r"   rY   rx   rƒ   r�   r‘   rµ   r½   Ú__name__© r#   r!   ú<module>rÇ      s  ðñó 
Û 	Û 
Û Û Ý Ý  ß (ß ?à
ˆ"�)‰)�KÐ!2Ó
3€à'€Ø&€	àˆJÐMÐ
N€àØØ'ØØñ€ð Ø<Ø�ZÐ7Ð8Øñ	ð Ø9Ø�ZÐ7Ð8Øñ	ð Ø3Ø�ZÐ7Ð8Øñ	ð€ð,3ð !ñ	Ð ð €Ø€	òò'òT()òV
òò0ò&HòV	Cð ˆzÒÙ…Fð r#   