Ë
    ��vjý  ã                   óÊ   — d Z ddlZddlZddlZddlZddlZddlZddlZdZ	 ej                  dd«      ZdZd„ Zd„ Zd	d
ddddddœZde	dz   dœZd„ Zd„ Zd„ Zedk(  r e«        yy)uÁ   
æ³°å·žå¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€ - é¡¹ç›®çŽ¯è¯„å…¬ç¤º çˆ¬è™«
URL: https://hbj.taizhou.gov.cn/ztzl/jsxm/xmhpgs/index.html
CMS: Hanweb JPAAS
API: /api-gateway/jpaas-publish-server/front/page/build/unit
é    Nzhttps://hbj.taizhou.gov.cnÚ	SEARCH_DBz/root/search.dbu+   æ³°å·žå¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€-é¡¹ç›®çŽ¯è¯„å…¬ç¤ºc                  óœ   — dd l } | j                  t        «      }|j                  d«       |j                  d«       |j	                  «        |S )Nr   a<  
        CREATE TABLE IF NOT EXISTS gov_raw (
            id INTEGER PRIMARY KEY,
            title TEXT,
            content TEXT,
            publish_date TEXT,
            page_url TEXT UNIQUE,
            source_url TEXT,
            site_name TEXT DEFAULT '',
            category TEXT DEFAULT ''
        )
    z>CREATE INDEX IF NOT EXISTS idx_site_name ON gov_raw(site_name))Úsqlite3Úconnectr   ÚexecuteÚcommit)r   Úconns     ú!/root/gov_crawler/crawl_tzsthj.pyÚ	ensure_dbr      sA   € ÛØ�?‰?œ9Ó%€DØ‡L�Lð ô 	ð 	‡L�LÐQÔRØ‡K�K„MØ€Kó    c           	      ód   — | j                  d||||t        t        f«       | j                  «        y )NzwINSERT OR IGNORE INTO gov_raw (title, content, publish_date, page_url, source_url, site_name) VALUES (?, ?, ?, ?, ?, ?))r   ÚBASE_URLÚ	SITE_NAMEr   )r	   ÚtitleÚcontentÚpublish_dateÚurls        r
   Ú	save_itemr   #   s/   € Ø‡L�Lð 	BØ	�˜ s¬H´iÐ@ôð 	‡K�K…Mr   ÚbulidstaticÚ e4f0e8a733e34ed28f0620789e66a128Ú 6a37c2a62cce48538e78db1205a34cc6Úcolumnu   å½“å‰�æ �ç›®ä¿¡æ�¯åˆ—è¡¨ÚnullÚ 477be6a07a6d446e938718552dfcec37)Ú	parseTypeÚwebIdÚtplSetIdÚpageTypeÚtagIdÚeditTypeÚpageIdz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36z/ztzl/jsxm/xmhpgs/index.html)z
User-AgentÚRefererc                 óV  — t         j                  «       }t        j                  | ddœd¬«      |d<   t        dz   t
        j                  j                  |«      z   }t
        j                  j                  |t        ¬«      }t
        j                  j                  |d¬	«      }t        j                  |j                  «       «      }|d
   d   }g }t        j                  d«      }|j!                  |«      D ]s  }	|	j#                  d«      }
|	j#                  d«      j%                  «       }|	j#                  d«      }|
j'                  d«      s	t        |
z   n|
}|j)                  |||f«       Œu t        j*                  d|«      }|rt-        |j#                  d«      «      n
t/        |«      }||fS )u   èŽ·å�–åˆ—è¡¨é¡µæ•°æ�®é   )ÚpageNoÚpageSizeF)Úensure_asciiÚ	paramJsonz8/api-gateway/jpaas-publish-server/front/page/build/unit?©Úheadersé   ©ÚtimeoutÚdataÚhtmlzI<a[^>]*href="([^"]+)"[^>]*>([^<]+)</a>\s*<span>(\d{4}-\d{2}-\d{2})</span>é   é   é   Úhttpzcount="(\d+)")Ú
API_PARAMSÚcopyÚjsonÚdumpsr   ÚurllibÚparseÚ	urlencodeÚrequestÚRequestÚHEADERSÚurlopenÚloadsÚreadÚreÚcompileÚfinditerÚgroupÚstripÚ
startswithÚappendÚsearchÚintÚlen)Úpage_noÚparamsr   ÚreqÚrespr.   r/   ÚitemsÚpatternÚmÚurl_pathr   ÚdateÚfull_urlÚcount_mÚtotals                   r
   Ú
fetch_listrW   :   sh  € ä�_‰_Ó€FÜŸ*™*°ÀRÑ%HÐW\Ô]€Fˆ;ÑÜ
ÐOÑ
OÔRX×R^ÑR^×RhÑRhÐioÓRpÑ
p€CÜ
�.‰.×
 Ñ
  ¬gÐ
 Ó
6€CÜ�>‰>×!Ñ! #¨rÐ!Ó2€DÜ�:‰:�d—i‘i“kÓ"€DØ�‰<˜Ñ€Dð €EÜ�j‰jØTó€Gð ×Ñ˜dÓ#ò .ˆØ—7‘7˜1“:ˆØ—‘˜“
× Ñ Ó"ˆØ�w‰w�q‹zˆØ.6×.AÑ.AÀ&Ô.I”8˜hÒ&ÈxˆØ�‰�e˜T 8Ð,Õ-ð.ô �i‰iÐ(¨$Ó/€GÙ%,ŒC�—‘˜aÓ Ô!´#°e³*€Eà�%ˆ<Ðr   c                 óä  — t         j                  j                  | t        ¬«      }t         j                  j	                  |d¬«      }|j                  «       j                  dd¬«      }t        j                  d|t        j                  «      }|r|j                  d«      j                  «       nd	}t        j                  d
|«      }|r|j                  d«      nd	}t        j                  d|t        j                  «      }|r|j                  d«      j                  «       nd	}	|	sHt        j                  d|t        j                  «      }|r|j                  d«      j                  «       nd	}	|||	fS )u   èŽ·å�–è¯¦æƒ…é¡µå†…å®¹r)   r+   r,   zutf-8Úreplace)Úerrorsz#<div class="wzzw-title">(.*?)</div>r0   Ú u(   å�‘å¸ƒæ—¥æœŸ[ï¼š:]\s*(\d{4}-\d{2}-\d{2})u>   <div class="wzzw-article">(.*?)</div>\s*<!--\s*æ‰«ä¸€æ‰«\s*-->z%<div class="wzzw-article">(.*?)</div>)r8   r;   r<   r=   r>   r@   ÚdecoderA   rH   ÚDOTALLrD   rE   )
r   rM   rN   r/   Útitle_mr   Údate_mrS   Ú	content_mr   s
             r
   Úfetch_detailra   V   s"  € ä
�.‰.×
 Ñ
  ¬gÐ
 Ó
6€CÜ�>‰>×!Ñ! #¨rÐ!Ó2€DØ�9‰9‹;×Ñ˜g¨iÐÓ8€Dô �i‰iÐ>ÀÄbÇiÁiÓP€GÙ(/ˆG�M‰M˜!Ó×"Ñ"Ô$°R€Eô �Y‰YÐBÀDÓI€FÙ$ˆ6�<‰<˜Œ?¨"€Dô —	‘	ØIØŒb�i‰ió€Iñ -6ˆi�o‰o˜aÓ ×&Ñ&Ô(¸2€GáÜ—I‘IÐFÈÌbÏiÉiÓXˆ	Ù09�)—/‘/ !Ó$×*Ñ*Ô,¸rˆà�$˜ÐÐr   c                  óX  — t        «       } d}t        d«      \  }}|dz   dz  }t        d|› d|› �«       g }t        d|dz   «      D ]Y  }|dk(  r|}nt        |«      \  }}t        d|› dt	        |«      › d	�«       |j                  |«       t        j                  d
«       Œ[ t        dt	        |«      › �«       t        |d«      D ]¬  \  }	\  }
}}	 t        |«      \  }}}|
xs |}|xs |}|r%t        j                  dd|«      j                  «       nd}t	        |«      dk\  }t        | ||||«       |rdnd}t        d|	› dt	        |«      › d|› d|› d|d d › �
«       t        j                  d«       Œ® | j                  dt         f«      }|j#                  «       d   }| j                  dt         f«      j#                  «       d   }t        dd› �«       t        d|› d|› d�«       | j%                  «        y # t        $ r*}t        d|	› dt	        |«      › d|› d|› �«       Y d }~ŒÈd }~ww xY w)Nr   r0   é   r$   zTotal items: z	, pages: zPage z: z itemsg333333Ó?z
Total items from list: z<[^>]+>r[   é2   ÚOKÚSHORTz  [ú/z] ú z] ERROR g      à?z]SELECT COUNT(*) FROM gov_raw WHERE site_name=? AND content IS NOT NULL AND length(content)>50z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?ú
z2==================================================u   å®Œæˆ�ï¼�æ€»è®¡: u    æ�¡, æœ‰å†…å®¹: u    æ�¡)r   rW   ÚprintÚrangerJ   ÚextendÚtimeÚsleepÚ	enumeratera   rA   ÚsubrE   r   Ú	Exceptionr   r   ÚfetchoneÚclose)r	   Útotal_itemsÚfirst_itemsrV   Útotal_pagesÚall_page_itemsrK   rO   Ú_Úidxr   rS   r   Ú	det_titleÚdet_dater   Ú	use_titleÚuse_dateÚtextÚ
content_okÚstatusÚeÚcurÚok_countÚtotal_counts                            r
   Úmainr…   q   sf  € Ü‹;€DØ€Kô $ A›Ñ€K�Ø˜2‘: "Ñ$€KÜ	ˆM˜%˜ 	¨+¨Ð
7Ô8ð €NÜ˜˜K¨!™OÓ,ò ˆØ�aŠ<Ø‰Eä! 'Ó*‰HˆE�1ä��g�Y˜b¤ U£ ¨FÐ3Ô4Ø×Ñ˜eÔ$Ü�
‰
�3�ðô 
Ð%¤c¨.Ó&9Ð%:Ð
;Ô<ô $-¨^¸QÓ#?ò ÑˆÑˆe�T˜3ð	GÜ+7¸Ó+<Ñ(ˆI�x ØÒ* ˆIØÒ'˜xˆHñ ?F”2—6‘6˜* b¨'Ó2×8Ñ8Ô:È2ˆDÜ˜T› b™ˆJÜ�d˜I w°¸#Ô>Ù'‘T¨WˆFÜ�C˜�u˜Aœc .Ó1Ð2°"°V°H¸A¸h¸ZÀqÈÐSVÐTVÈÐHXÐYÔZô 	�
‰
�3�ðð$ �,‰,Ðvô  zCð  yEó  F€CØ�|‰|‹~˜aÑ €HØ—,‘,ÐOÔR[ÐQ]Ó^×gÑgÓiÐjkÑl€Kä	ˆBˆvˆhˆ-ÔÜ	Ð˜k˜]Ð*;¸H¸:ÀTÐ
JÔKà‡J�J…Løô ò 	GÜ�C˜�u˜Aœc .Ó1Ð2°(¸3¸%¸rÀ!ÀÐE×FÑFûð	Gús   ÃBG6Ç6	H)Ç? H$È$H)Ú__main__)Ú__doc__r6   rA   Úurllib.requestr8   Úurllib.parseÚsysÚosrm   r   Úgetenvr   r   r   r   r4   r=   rW   ra   r…   Ú__name__© r   r
   ú<module>r�      s™   ðñ÷ =× <× <Ó <à'€ð ˆB�I‰I�kÐ#4Ó5€	Ø9€	òò&ð Ø/Ø2ØØ'ØØ0ñ€
ð QØÐ8Ñ8ñ€ò
ò8 ò61ðf ˆzÒÙ…Fð r   