Ë
    ¸¬uj   ã                   ó   — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
m
Z
mZ ddlmZmZ ej                  j!                  dd«      Zd	Zd
ZdZe› d�ZdddddddœZdde› d�dœZdZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Ze d k(  r e«        yy)!u&  
crawl_jinan_epb_yppz.py â€” æµŽå�—å¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€ç« ä¸˜åˆ†å±€Â·å·²æ‰¹å‡†é¡¹ç›®å…¬å‘Š
========================================================
CMS: JCMS (å¤§æ±‰ç‰ˆé€š)ï¼ŒAPI é©±åŠ¨ï¼Œ15æ�¡/é¡µï¼Œ777æ�¡æ€»é‡�

åˆ—è¡¨ï¼šAPI /api-gateway/jpaas-publish-server/front/page/build/unit
      å¸¦ paramJson åˆ†é¡µå�‚æ•°
è¯¦æƒ…ï¼š<div class="article"> æ­£æ–‡ | <meta name="ArticleTitle"> / <meta name="PubDate">

ç”¨æ³•:
    python3 crawl_jinan_epb_yppz.py               # å…¨é‡�
    python3 crawl_jinan_epb_yppz.py 1             # å¢žé‡�ï¼ˆä»…æœ€æ–°é¡µï¼‰
é    N)ÚBeautifulSoup)Úurljoin)ÚdatetimeÚ	timedelta)ÚThreadPoolExecutorÚas_completedÚGOV_DB_PATHz/root/search.dbzhttps://jnepb.jinan.gov.cnu   æµŽå�—å¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€u   å·²æ‰¹å‡†é¡¹ç›®å…¬å‘Šz7/api-gateway/jpaas-publish-server/front/page/build/unitÚbulidstaticÚ21ÚpMB9pkaz19HYYbLAOTZtYÚcolumnu   ä¿¡æ�¯åˆ—è¡¨Ú115248)Ú	parseTypeÚwebIdÚtplSetIdÚpageTypeÚtagIdÚpageIdzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zapplication/jsonz/col/col115248/index.html)z
User-AgentÚAcceptÚRefereré   é2   c                 ó    — t        d| › �«       y )Nz[jinan_yppz] )Úprint)Úmsgs    ú)/root/gov_crawler/crawl_jinan_epb_yppz.pyÚlogr   6   s   € Ü	ˆM˜#˜Ð
Õ ó    c                 óz  — t        j                  | t        dœd¬«      }t        t        «      }||d<   	 t        j                  t        t        |d¬«      }|j                  «       }|j                  di «      j                  dd	«      }g }d}t        j                  d|«      }	|	rt        |	j                  d«      «      }t        j                  d|«      D ]¸  }	|	j                  d«      }
|	j                  d«      j!                  «       }||	j#                  «       |	j%                  «       dz    }t        j                  d|«      }|r|j                  d«      nd	}|
j'                  d«      r|
nt)        t*        |
«      }|j-                  |||f«       Œº ||fS # t        $ r}t        d
| › d|› �«       g dfcY d}~S d}~ww xY w)z@Fetch one page of article list from API. Returns (items, total).)ÚpageNoÚpageSizeF)Úensure_asciiÚ	paramJsoné   )ÚheadersÚparamsÚtimeoutÚdataÚhtmlÚ z  ERROR API page ú: r   Nzcount="(\d+)"é   z8<a\s+href="([^"]+)"\s+title="([^"]+)"\s+target="_blank">é   éÈ   z\[(\d{4}-\d{2}-\d{2})\]Úhttp)ÚjsonÚdumpsÚ	PAGE_SIZEÚdictÚAPI_BASE_PARAMSÚrequestsÚgetÚAPI_URLÚHEADERSÚ	Exceptionr   ÚreÚsearchÚintÚgroupÚfinditerÚstripÚstartÚendÚ
startswithr   ÚBASE_URLÚappend)Úpage_noÚ
param_jsonr&   Úrespr(   r)   ÚeÚitemsÚtotalÚmÚhrefÚtitleÚentryÚdate_mÚdate_strÚfull_urls                   r   Úfetch_articles_pagerR   :   sŠ  € ä—‘ w¼IÑFÐUZÔ[€JÜ”/Ó"€FØ$€Fˆ;ÑðÜ�|‰|œG¬W¸VÈRÔPˆØ�y‰y‹{ˆØ�x‰x˜ Ó#×'Ñ'¨°Ó3ˆð
 €EØ€Eä
�	‰	Ð" DÓ)€AÙÜ�A—G‘G˜A“J“ˆä�[‰[ØCØóò 2ˆð �w‰w�q‹zˆØ—‘˜“
× Ñ Ó"ˆð �Q—W‘W“Y˜qŸu™u›w¨™}Ð-ˆÜ—‘Ð5°uÓ=ˆÙ&,�6—<‘< ”?°"ˆàŸ?™?¨6Ô2‘4¼ÄÈ$Ó8OˆØ�‰�h  xÐ0Õ1ð2ð �%ˆ<Ðøô5 ò ÜÐ ˜y¨¨1¨#Ð.Ô/Ø�1ˆu�ûðús   ´AF Æ	F:ÆF5Æ/F:Æ5F:c                 óâ  — 	 t        j                  | t        d¬«      }d|_        |j                  }t        |d«      }|j                  d	d
di¬«      }|r$|j                  d«      r|d   j                  «       nd}d}|j                  d	d
di¬«      }|rK|j                  d«      r:t        j                  d|d   j                  «       «      }	|	r|	j                  d«      }|j                  dd„ ¬«      }
|
s-|j                  dd¬«      }|r|j                  d¬«      }|r|n|}
|
rt        |
«      nd}|st        d| › �«       d||fS |||fS # t
        $ r}t        d| › d|› �«       Y d}~yd}~ww xY w)z=Return (content_html, date_str, title) or (None, None, None).r$   )r%   r'   zutf-8z  ERROR r+   N©NNNzhtml.parserÚmetaÚnameÚArticleTitle)ÚattrsÚcontentr*   ÚPubDatez(\d{4}-\d{2}-\d{2})r,   Údivc                 óT   — | xr% dt        | t        «      r| ndj                  | «      v S )NÚarticleú )Ú
isinstanceÚstrÚjoin)Úcs    r   ú<lambda>zfetch_detail.<locals>.<lambda>|   s$   € ˜ÒV˜y´*¸QÄÔ2D©QÈ#Ï(É(ÐSTË+ÐV€ r   )Úclass_zbt-article-02Úzoom)Úidz  WARNING: no content for )r5   r6   r8   ÚencodingÚtextr9   r   r   Úfindr?   r:   Úmatchr=   r`   )ÚurlrG   r)   rH   ÚsoupÚ
meta_titlerM   rP   Ú	meta_daterK   Úcontent_divÚ
bt_articlere   Úcontent_htmls                 r   Úfetch_detailrr   a   s‚  € ð Ü�|‰|˜C¬¸"Ô=ˆØˆŒØ�y‰yˆô
 ˜˜}Ó-€Dð —‘˜6¨&°.Ð)A�ÓB€JÙ-7¸J¿N¹NÈ9Ô<UˆJ�yÑ!×'Ñ'Ô)Ð[]€Eð €HØ—	‘	˜&¨°Ð(;�	Ó<€IÙ�Y—]‘] 9Ô-Ü�H‰HÐ+¨Y°yÑ-A×-GÑ-GÓ-IÓJˆÙØ—w‘w˜q“zˆHð —)‘)ØÙVð ó €Kñ à—Y‘Y˜u¨_�YÓ=ˆ
ÙØ—?‘? f�?Ó-ˆDÙ"&™$¨JˆKÙ'2”3�{Ô#¸€LáÜÐ(¨¨Ð.Ô/Ø�X˜uÐ$Ð$à˜ 5Ð(Ð(øôG ò  Üˆh�s�e˜2˜a˜SÐ!Ô"Üûð ús   ‚/E
 Å
	E.ÅE)Å)E.c                 ó$  — i }t        t        ¬«      5 }| D �ci c]  }|j                  t        |«      |“Œ }}t	        |«      D ]  }||   }	 |j                  «       ||<   Œ 	 d d d «       |S c c}w # t        $ r d||<   Y Œ>w xY w# 1 sw Y   |S xY w)N)Úmax_workersrT   )r   ÚMAX_WORKERSÚsubmitrr   r   Úresultr9   )ÚurlsÚresultsÚexrk   Úfut_mapÚfuts         r   Úfetch_detail_batchr}   �   s£   € Ø€GÜ	¬Ô	4ð 2¸Ø@DÖE¸�2—9‘9œ\¨3Ó/°Ñ4ÐEˆÐEÜ Ó(ò 	2ˆCØ˜#‘,ˆCð2Ø"Ÿz™z›|�˜’ñ	2÷2ð €Nùò Føô
 ò 2Ø1�˜“ð2ú÷2ð €Nús?   “B˜A,¶BÁA1ÁBÁ,BÁ1BÁ?BÂBÂBÂBc           
      óT  — ddl }| sy|j                  t        «      }|j                  «       }d}| D ]-  \  }}}}	 |j	                  dt
        ||||t        f«       |dz  }Œ/ |j                  «        |j                  «        |S # t        $ r}	t        d|› d|	› �«       Y d}	~	Œqd}	~	ww xY w)z3items: list of (url, title, date_str, content_html)r   NzœINSERT OR REPLACE INTO gov_raw
                   (site_name, page_url, title, publish_date, content, category)
                   VALUES (?, ?, ?, ?, ?, ?)r,   z  DB error for r+   )Úsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚ	SITE_NAMEÚCOLUMN_NAMEr9   r   ÚcommitÚclose)
rI   r   Úconnrb   Úinsertedrk   rM   rP   rq   rH   s
             r   Ú
save_to_dbrŠ   š   s·   € ãÙØØ�?‰?œ7Ó#€DØ�‰‹€AØ€HØ.3ò 
.Ñ*ˆˆU�H˜lð		.Ø�I‰Ið0ô ˜C ¨°,ÄÐLô	ð ˜‰M‰Hð
.ð 	‡K�K„MØ‡J�J„LØ€Oøô	 ò 	.Ü�/ #  b¨¨Ð,×-Ñ-ûð	.ús   º%BÂ	B'ÂB"Â"B'c                  ó,  — t        t        j                  «      dkD  xr t        j                  d   dk(  } t        j                  «       t        d¬«      z
  j                  d«      }t        d|› �«       g }d}d}	 t        |«      \  }}|dk(  r|}t        d|› �«       |s|dk(  rt        d	«       y n‹|D ���	cg c]  \  }}}	|	sŒ
|	|k\  sŒ|||	f‘Œ }
}}}	|j                  |
«       | rt        d
t        |
«      › d�«       n<|
r't        |
«      t        |«      k  rt        d|› d�«       n|t        z  |k\  rn|dz  }ŒÂt        dt        |«      › �«       g }t        dz  }t        dt        |«      |«      D ]�  }||||z    }|D ��cg c]  \  }}}|‘Œ
 }}}t        d|dz   › d|t        |«      z   › dt        |«      › d�«       t        |«      }|D ]A  \  }}}|j                  |d«      \  }}}|xs |}|xs |}|j                  ||||xs df«       ŒC ŒŸ t!        |«      }t        dd› �«       t        dt        |«      › d|› d�«       |r9t#        |D �cg c]  }|d   sŒ	|d   ‘Œ c}«      }t        d|d   › d|d   › �«       	 dd l}|j'                  t(        «      }|j+                  dt,        f«       |j/                  «        |j1                  «        t        d «       y c c}	}}w c c}}w c c}w # t2        $ r}t        d!|› �«       Y d }~y d }~ww xY w)"Nr,   Ú1iG  )Údaysz%Y-%m-%dz3-year cutoff: r   zTotal expected: zNo articles foundzIncremental: page 1, z	 articleszPage z$: some items before cutoff, stoppingzTotal articles to process: é   z  Details [ú-ú/z]...rT   r*   ú
z2==================================================zDone. Total: z articles (saved: ú)r-   zDate range: z ~ éÿÿÿÿzüINSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
            SELECT rowid, title, site_name,
                   CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
            FROM gov_raw WHERE category = ?zFTS index rebuiltzFTS rebuild note: )ÚlenÚsysÚargvr   Únowr   Ústrftimer   rR   Úextendr2   ru   Úranger}   r6   rD   rŠ   Úsortedr   r€   r�   rƒ   r…   r†   r‡   r9   ) ÚincrementalÚthree_years_agoÚall_articlesÚtotal_expectedÚpagerI   rJ   ÚuÚtÚdÚfilteredry   Ú
batch_sizeÚbatch_startÚbatchÚ_Ú
batch_urlsÚ
detail_maprk   rM   rP   rq   Údetail_dateÚdetail_titleÚfinal_titleÚ
final_dateÚsavedÚrÚdatesr   rˆ   rH   s                                    r   Úmainr²   ²   sL  € Ü”c—h‘h“- !Ñ#Ò:¬¯©°©°sÑ(:€KÜ—|‘|“~¬	°wÔ(?Ñ?×IÑIÈ*ÓU€OÜˆ/˜/Ð*Ð+Ô,ð €LØ€Nà€DØ
Ü*¨4Ó0‰ˆˆuØ�1Š9Ø"ˆNÜÐ" >Ð"2Ð3Ô4áØ�qŠyÜÐ'Ô(ØØð .3×QÐQ¡' ! Q¨²a¸AÀÓ<P�Q˜˜1’IÐQˆÒQØ×Ñ˜HÔ%áÜÐ'¬¨H« °iÐ@ÔAØáœ˜H›¬¨E«
Ò2Ü�%˜�vÐAÐBÔCØà”)Ñ˜~Ò-Øà�‰	ˆð7 ô: Ð
%¤c¨,Ó&7Ð%8Ð9Ô:ð €GÜ˜q‘€JÜ˜Q¤ LÓ 1°:Ó>ò 	OˆØ˜[¨°zÑ)AÐBˆØ',×-™G˜A˜q !’aÐ-ˆ
Ñ-Üˆk˜+ a™-˜¨¨+´c¸%³jÑ*@Ð)AÀÄ3À|ÓCTÐBUÐUYÐZÔ[Ü'¨
Ó3ˆ
Ø$)ò 	OÑ ˆC�˜Ø6@·n±nÀSÐJ\Ó6]Ñ3ˆL˜+ |Ø&Ò/¨%ˆKØ$Ò0¨ˆJØ�N‰N˜C ¨j¸,Ò:LÈ"ÐMÕNñ		Oð	Oô �wÓ€Eäˆ"ˆXˆJˆÔÜˆ-œ˜G›�~Ð%7¸°w¸aÐ@ÔAÙÜ gÖ6 °°1³˜˜!›Ò6Ó7ˆÜˆl˜5 ™8˜* C¨¨b©	 {Ð3Ô4ð&ÛØ�‰œwÓ'ˆØ�‰ð /ô 2=°ô	@ð 	�‰ŒØ�
‰
ŒÜÐÕ ùôg Rùó. .ùò 7øô ò &ÜÐ   Ð$×%Ñ%ûð&ús=   Â7K ÃK Ã	K ÆK'É
K-ÉK-ÊAK2 Ë2	LË;LÌLÚ__main__)!Ú__doc__Úosr:   r•   r0   r5   Úbs4r   Úurllib.parser   r   r   Úconcurrent.futuresr   r   Úenvironr6   r�   rC   r„   r…   r7   r4   r8   ru   r2   r   rR   rr   r}   rŠ   r²   Ú__name__© r   r   ú<module>r¼      sÏ   ðñó 
Û 	Û 
Û Û Ý Ý  ß (ß ?à
�*‰*�.‰.˜Ð(9Ó
:€à'€Ø&€	Ø%€àˆJÐMÐ
N€àØØ'ØØØñ€ð3ð !Ø�Ð4Ð5ñ€ð €Ø€	ò!ò$òN))òX
òò0L&ð^ ˆzÒÙ…Fð r   