Ë
    ¸¬ujÓ  ã                   óÔ   — d Z ddlZddlZddlZddlZddlZddlZddlZddl	Z	 ej                  dd«      ZdZdZdZdZd	d
dœZdddddddœZdadad„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u)   æ³°å…´å¸‚äººæ°‘æ”¿åºœ-ç”Ÿæ€�çŽ¯å¢ƒ çˆ¬è™«é    NÚ	SEARCH_DBz/root/search.dbu   taixing.gov.cn-ç”Ÿæ€�çŽ¯å¢ƒu   çŽ¯è¯„å…¬ç¤ºz
2023-06-16zQhttps://www.taixing.gov.cn/api-gateway/jpaas-publish-server/front/page/build/unitz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36z9https://www.taixing.gov.cn/zwgk/xxgk/zdly/sthj/index.html)z
User-AgentÚRefererÚcolumnu   ä¿¡æ�¯å…¬å¼€å†…å®¹2ÚbulidstaticÚ 569012ebb4b9462a9f6c633227a8ef5bÚNaLBGqC9X7yXPzYhbzhoIÚ a62207fcac654afba2ff9d55a7839094)ÚpageTypeÚtagIdÚ	parseTypeÚwebIdÚpageIdÚtplSetIdc                 ó<  — t        t        «      }t        j                  | ddœd¬«      |d<   t        dz   t
        j                  j                  |«      z   }t
        j                  j                  |t        ¬«      }	 t
        j                  j                  |d¬	«      }t        j                  |j                  «       «      }|d
   d   }t        j                  dt        j                   «      }|j#                  |«      }|stt        j"                  d|«      }	t        j"                  d|«      }
g }t%        |	«      D ]8  \  }\  }}|dz   t'        |
«      k  r|
|dz      nd}|j)                  ||||f«       Œ: |S # t*        $ r}t-        d| › d|› �«       g cY d }~S d }~ww xY w)Né   ©ÚpageNoÚpageSizeF©Úensure_asciiÚ	paramJsonú?©Úheadersé   ©ÚtimeoutÚdataÚhtmlz�href="(/zwgk/xxgk/zdly/sthj/art/\d+[^"]*?/art_[^"]+\.html)"[^>]*title="([^"]*)"[^>]*>([^<]+)</a>.*?<td[^>]*width="14%"[^>]*>(\d{4}-\d{2}-\d{2})<zHhref="(/zwgk/xxgk/zdly/sthj/art/[^/]+/art_[^"]+\.html)"[^>]*>([^<]+)</a>z>(\d{4}-\d{2}-\d{2})<é   Ú z  Error fetching page ú: )ÚdictÚBASE_PARAMSÚjsonÚdumpsÚAPI_URLÚurllibÚparseÚ	urlencodeÚrequestÚRequestÚHEADERSÚurlopenÚloadsÚreadÚreÚcompileÚDOTALLÚfindallÚ	enumerateÚlenÚappendÚ	ExceptionÚprint)ÚpageÚparamsÚurlÚreqÚrespr   r   ÚpatternÚitemsÚlinksÚdatesÚiÚhrefÚtitleÚdtÚes                   ú"/root/gov_crawler/crawl_taixing.pyÚ
fetch_listrI      sy  € Ü”+Ó€FÜŸ*™*°À"Ñ%EÐTYÔZ€Fˆ;ÑÜ
�C‰-œ&Ÿ,™,×0Ñ0°Ó8Ñ
8€CÜ
�.‰.×
 Ñ
  ¬gÐ
 Ó
6€CðÜ�~‰~×%Ñ% c°2Ð%Ó6ˆÜ�z‰z˜$Ÿ)™)›+Ó&ˆØ�F‰|˜FÑ#ˆä—*‘*ð@ä�I‰Ió
ˆð
 —‘ Ó%ˆÙä—J‘JØ[ØóˆEô —J‘JÐ7¸Ó>ˆEØˆEÜ$-¨eÓ$4ò 7Ñ �‘=�D˜%Ø#$ Q¡3¬¨U«Ò#3�U˜1˜Q™3’Z¸�Ø—‘˜d E¨5°"Ð5Õ6ð7ð ˆøÜò ÜÐ& t f¨B¨q¨cÐ2Ô3Ø�	ûðús   Á<C8E5 Å5	FÅ>FÆFÆFc                 óî  — d| z   }t         j                  j                  |t        ¬«      }	 t         j                  j	                  |d¬«      }|j                  «       j                  dd¬«      }t        j                  d|«      }|r|j                  d	«      j                  «       nd
}t        j                  d|t        j                  «      }|s%t        j                  d|t        j                  «      }|r|j                  d	«      j                  «       nd
}t        j                  d|«      }	|	r |	j                  d	«      j                  «       }
n+t        j                  d|«      }|r|j                  d	«      nd
}
t        j                  d|«      }|r|j                  d	«      j                  «       nd
}|xs | j                  d«      d   ||
||dœS # t        $ r}t        d| › d|› �«       Y d }~y d }~ww xY w)Nzhttps://www.taixing.gov.cnr   r   r   zutf-8Úreplace)Úerrorsz<p class="con-title">(.*?)</p>r    r!   z(<div class="main-txt">(.*?)</div>\s*<!--z!<div class="main-txt">(.*?)</div>u&   å�‘æ–‡æ—¥æœŸ[^<]*<td[^>]*>([^<]+)</td>u"   å�‘å¸ƒæ—¥æœŸï¼š(\d{4}-\d{2}-\d{2})u   ä¿¡æ�¯æ�¥æº�ï¼š([^<]+)ú/éÿÿÿÿ)rE   ÚcontentÚpub_dateÚsourceÚpage_urlz  Error fetching detail r"   )r(   r+   r,   r-   r.   r0   Údecoder1   ÚsearchÚgroupÚstripr3   Úsplitr8   r9   )r<   Úfull_urlr=   r>   r   Útitle_mrE   Ú	content_mrO   Údate_mrP   Údate_m2Úsource_mrQ   rG   s                  rH   Úfetch_detailr^   =   s¬  € Ø+¨cÑ1€HÜ
�.‰.×
 Ñ
  ´7Ð
 Ó
;€Cð Ü�~‰~×%Ñ% c°2Ð%Ó6ˆØ�y‰y‹{×!Ñ! '°)Ð!Ó<ˆä—)‘)Ð=¸tÓDˆÙ,3�—‘˜aÓ ×&Ñ&Ô(¸ˆä—I‘IÐIÈ4ÔQS×QZÑQZÓ[ˆ	ÙÜŸ	™	Ð"FÈÌbÏiÉiÓXˆIÙ09�)—/‘/ !Ó$×*Ñ*Ô,¸rˆä—‘ÐDÀdÓKˆÙØ—|‘| A“×,Ñ,Ó.‰Hô —i‘iÐ EÀtÓLˆGÙ+2�w—}‘} QÔ'¸ˆHä—9‘9Ð6¸Ó=ˆÙ.6�—‘ Ó"×(Ñ(Ô*¸Bˆð Ò0˜cŸi™i¨›n¨RÑ0ØØ ØØ ñ
ð 	
øô ò ÜÐ(¨¨¨R°¨sÐ3Ô4Üûðús   ¬F#G Ç	G4ÇG/Ç/G4c           
      ó˜  — | d   r| d   t         k  r
t        dz  ay 	 t        j                  t        «      }|j                  «       }|j                  d| d   f«       |j                  «       rt        dz  a|j                  «        y |j                  d| d   | d   | d   t        | d   | d   f«       |j                  dkD  rG|j                  }|j                  d	|| d   | d   | d   r| d   d d
 ndt        | d   f«       t        dz  a|j                  «        |j                  «        y # t        $ r}t        d|› �«       Y d }~y d }~ww xY w)NrP   r    z)SELECT id FROM gov_raw WHERE page_url = ?rR   zrINSERT OR IGNORE INTO gov_raw (title, content, publish_date, site_name, source_url, page_url) VALUES (?,?,?,?,?,?)rE   rO   r   znINSERT OR IGNORE INTO gov_search(rowid, title, content, summary, site_name, publish_date) VALUES (?,?,?,?,?,?)iô  r!   z  DB error: )ÚCUTOFF_DATEÚtotal_skippedÚsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚfetchoneÚcloseÚ	SITE_NAMEÚrowcountÚ	lastrowidÚtotal_insertedÚcommitr8   r9   )ÚitemÚdbÚcÚinserted_idrG   s        rH   Ú
save_to_dbrr   b   sa  € àˆJÒ˜D Ñ,¬{Ò:Ü˜ÑˆØð"Ü�_‰_œWÓ%ˆØ�I‰I‹Kˆà	�	‰	Ð=ÀÀZÑ@PÐ?RÔSØ�:‰:Œ<Ü˜QÑˆMØ�H‰HŒJØØ	�	‰	ð AØ�'‰]˜D ™O¨T°*Ñ-=¼yÈ$ÈzÑJZÐ\`ÐakÑ\lÐmô	
ð �:‰:˜Š>ØŸ+™+ˆKà�I‰Ið  GØ" D¨¡M°4¸	±?Ð]aÐbkÒ]lÀDÈÁOÐTXÐUXÑDYÐrtÔvð  BFð  GQñ  BRð  SôTä˜aÑˆNØ
�	‰	ŒØ
�‰�
øÜò "Ü�˜Q˜CÐ ×!Ñ!ûð"ús   �A(D( ÂB!D( Ä(	E	Ä1EÅE	c            
      óp  — t        dt        › �«       t        dt        › �«       t        t        «      } t        j                  dddœd¬«      | d<   t        d	z   t        j                  j                  | «      z   }t        j                  j                  |t        ¬
«      }t        j                  j                  |d¬«      }t        j                  |j!                  «       «      }|d   d   }t#        j$                  d|«      }|rt'        |j)                  d«      «      nd}t        d|› �«       t+        dd«      D ]â  }t-        |«      }	|	st        d|› d�«        nÄt        d|› dt/        |	«      › d�«       t1        «       }
|	D ]y  \  }}}}||
v rŒ|
j3                  |«       |r|t        k  r
t4        dz  aŒ3t7        |«      }|r%t        d|d   › d|d   d d › �«       t9        |«       t;        j<                  d«       Œ{ |dz  |k\  r nt;        j<                  d«       Œä t        dt>        › dt4        › �«       y )NzStarting crawl: zCutoff date: r    r   r   Fr   r   r   r   r   r   r   r   zcount="(\d+)"iT  zTotal records: iç  z  Page z: no items, stoppingr"   z itemsz    rP   z - rE   é2   g      à?z
Done! Inserted: z, Skipped: ) r9   ri   r`   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   rT   ÚintrU   ÚrangerI   r6   ÚsetÚaddra   r^   rr   ÚtimeÚsleeprl   )r;   r<   r=   r>   r   r   Úcount_mÚtotal_recordsr:   r@   Ú	seen_urlsrD   Ú_titleÚtextÚdateÚdetails                   rH   Úmainr‚      s  € ä	ÐœY˜KÐ
(Ô)Ü	ˆMœ+˜Ð
'Ô(ô ”+Ó€FÜŸ*™*°¸rÑ%BÐQVÔW€Fˆ;ÑÜ
�C‰-œ&Ÿ,™,×0Ñ0°Ó8Ñ
8€CÜ
�.‰.×
 Ñ
  ¬gÐ
 Ó
6€CÜ�>‰>×!Ñ! #¨rÐ!Ó2€DÜ�:‰:�d—i‘i“kÓ"€DØ�‰<˜Ñ€DÜ�i‰iÐ(¨$Ó/€GÙ-4”C˜Ÿ™ aÓ(Ô)¸#€MÜ	ˆO˜M˜?Ð
+Ô,ä�a˜“ò ˆÜ˜4Ó ˆÙÜ�G˜D˜6Ð!5Ð6Ô7ÙÜ�˜�v˜R¤ E£
˜|¨6Ð2Ô3Ü“Eˆ	Ø(-ò 	Ñ$ˆD�&˜$ Ø�yÑ ØØ�M‰M˜$Ôá˜œ{Ò*Ü Ñ"�ØÜ! $Ó'ˆFÙÜ˜˜V JÑ/Ð0°°F¸7±OÀCÀRÐ4HÐ3IÐJÔKÜ˜6Ô"Ü�J‰J�s�Oð	ð �"‰9˜Ò%ÙÜ�
‰
�1�ð/ô2 
Ðœ~Ð.¨k¼-¸Ð
IÕJó    Ú__main__)Ú__doc__r%   r1   ÚsysÚosry   Úurllib.requestr(   Úurllib.parserb   Úgetenvrd   ri   ÚGROUPr`   r'   r-   r$   rl   ra   rI   r^   rr   r‚   Ú__name__© rƒ   rH   ú<module>rŽ      sž   ðá /ß × Ó ß #Û à
ˆ"�)‰)�KÐ!2Ó
3€Ø)€	Ø€Ø€à
]€àPØJñ€ð
 Ø"ØØ/Ø%Ø2ñ€ð €Ø€òò@#òJ"ò:*KðX ˆzÒÙ…Fð rƒ   