Ë
    ¸¬ujÑ  ã                   ó>  — d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
ZddlZej                  j                  dd«      ZdZdZdZd	Z ej(                  «        e	d
¬«      z
  j+                  d«      ZdZddddœZd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uC   crawl_dsq.py â€” å¤§çŸ³æ¡¥å¸‚äººæ°‘æ”¿åºœ-çŽ¯è¯„å…¬ç¤º (EWB system)é    N)ÚdatetimeÚ	timedeltaÚ	SEARCH_DBz/root/search.dbu%   å¤§çŸ³æ¡¥å¸‚äººæ°‘æ”¿åºœ-çŽ¯è¯„å…¬ç¤ºzEhttp://www.dsq.gov.cn/EWB_YK_Mid/rest/lightfrontaction/getgovinfolistzhttp://www.dsq.gov.cnz$f1c580dd-d7d2-4148-b90e-b08b2ed43c3aiG  )Údaysz%Y-%m-%dé2   zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36zapplication/jsonzGhttp://www.dsq.gov.cn/dynamic/zw/openlist.html?categorynum=003002004004)ú
User-AgentzContent-TypeÚRefererc                 óü  — ddd| t         t        dœdœ}t        j                  j	                  t
        t        j                  |«      j                  «       t        ¬«      }t        d«      D ]H  }	 t        j                  j                  |d¬«      }t        j                  |j                  «       «      c S  y# t        $ r<}|d	k  rt        j                   d	«       nt#        d
| › d|› �«       Y d}~ yY d}~ŒŒd}~ww xY w)zFetch one page of API dataÚ Ú003002004004)ÚdeptcodeÚcategorynumÚ	pageIndexÚpageSizeÚsiteGuid)ÚtokenÚparams)ÚdataÚheadersé   é   ©Útimeouté   z  [WARN] API page z	 failed: N)Ú	PAGE_SIZEÚ	SITE_GUIDÚurllibÚrequestÚRequestÚAPI_URLÚjsonÚdumpsÚencodeÚHEADERSÚrangeÚurlopenÚloadsÚreadÚ	ExceptionÚtimeÚsleepÚprint)Ú
page_indexr   ÚreqÚattemptÚrespÚes         ú/root/gov_crawler/crawl_dsq.pyÚ	fetch_apir3      sÚ   € ð àØ)Ø#Ü!Ü!ñ
ñ	€Dô �.‰.×
 Ñ
 ¤¬t¯z©z¸$Ó/?×/FÑ/FÓ/HÔRYÐ
 Ó
Z€CÜ˜“8ò 	ˆð	Ü—>‘>×)Ñ)¨#°rÐ)Ó:ˆDÜ—:‘:˜dŸi™i›kÓ*Ò*ñ	øô ò 	Ø˜Š{Ü—
‘
˜1•äÐ*¨:¨,°iÀ¸sÐCÔDÝô ûð	ús   Á.AB6Â6	C;Â?,C6Ã6C;c                 óô  — | j                  d«      r$t        j                  j                  t        | «      n| }t        j
                  j                  |ddi¬«      }t        d«      D ]H  }	 t        j
                  j                  |d¬«      }|j                  «       j                  dd	¬
«      }|c S  y# t        $ r<}|dk  rt        j                  d«       nt        d| › d|› �«       Y d}~ yY d}~ŒŒd}~ww xY w)zFetch detail page HTMLú/r   z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36)r   r   r   r   zutf-8Úreplace)Úerrorsr   é   z  [WARN] Detail fetch failed: z - N)Ú
startswithr   ÚparseÚurljoinÚBASEr   r   r%   r&   r(   Údecoder)   r*   r+   r,   )ÚurlÚfull_urlr.   r/   r0   Úhtmlr1   s          r2   Úfetch_detailrA   0   sÞ   € à25·.±.ÀÔ2EŒv�|‰|×#Ñ#¤D¨#Ô.È3€HÜ
�.‰.×
 Ñ
  ØÐTð4Ð
 ó €Cô ˜“8ò 
ˆð		Ü—>‘>×)Ñ)¨#°rÐ)Ó:ˆDØ—9‘9“;×%Ñ% g°iÐ%Ó@ˆDØŠKñ	
øô
 ò 	Ø˜Š{Ü—
‘
˜1•äÐ6°s°e¸3¸q¸cÐBÔCÝô ûð	ús   Á*AB2Â2	C7Â;,C2Ã2C7c                 óº  — d}t        j                  d| t         j                  «      }|r|j                  d«      j	                  «       }d}t        j                  d| t         j                  «      }|s%t        j                  d| t         j                  «      }|r²|j                  d«      j	                  «       }t        j
                  dd|t         j                  ¬«      }t        j
                  dd|t         j                  ¬«      }t        j
                  d	d|«      }t        j
                  d
d|«      }t        j
                  dd|«      }||j	                  «       fS )z*Extract title and content from detail HTMLr   z$<h3\s+id="ivs_title"[^>]*>(.*?)</h3>r8   zF<div\s+class="ewb-article-content"\s+id="ivs_content"[^>]*>(.*?)</div>zF<div\s+id="ivs_content"[^>]*>(.*?)</div>\s*<div[^>]*class="ewb-source"z<script[^>]*>.*?</script>)Úflagsz<style[^>]*>.*?</style>z<o:p>\s*</o:p>z<o:p/>z<p[^>]*>\s*(<br\s*/?>\s*)*</p>)ÚreÚsearchÚDOTALLÚgroupÚstripÚsub)r@   ÚtitleÚmÚcontents       r2   Úparse_detailrM   C   s  € ð €EÜ
�	‰	Ð9¸4ÄÇÁÓK€AÙØ—‘˜“
× Ñ Ó"ˆð €GÜ
�	‰	Ð[Ð]aÔce×clÑclÓm€AÙÜ�I‰IÐ_ÐaeÔgi×gpÑgpÓqˆÙØ—'‘'˜!“*×"Ñ"Ó$ˆä—&‘&Ð5°r¸7Ì"Ï)É)ÔTˆÜ—&‘&Ð3°R¸ÌÏ	É	ÔRˆÜ—&‘&Ð*¨B°Ó8ˆÜ—&‘&˜ B¨Ó0ˆÜ—&‘&Ð:¸BÀÓHˆà�'—-‘-“/Ð!Ð!ó    c                  óè  — t        dt        j                  «       j                  d«      › dt        › �«       t        d«      } | st        d«       y | d   d   }|t        z   dz
  t        z  }t        d	|› d
|› dt        › d�«       | d   d   }t        d|«      D ]V  }t        |«      }|rd|d   v r|j                  |d   d   «       t        j                  t        j                  dd«      «       ŒX t        dt        |«      › d�«       |D �cg c]  }|j                  dd«      t        k\  sŒ|‘Œ  }}t        |«      t        |«      z
  }t        dt        |«      › d|› d�«       t!        j"                  t$        d¬«      }	|	j'                  «       }
dx}x}}t)        |d«      D �]q  \  }}|d   j+                  «       }|d   }t,        j.                  j1                  t2        |d   «      }t        d|› dt        |«      › d|d d › d�d¬«       |
j5                  d|t        f«      j7                  «       rt        d «       |dz  }Œ t9        |d   «      }|st        d!«       |dz  }ŒÁt;        |«      \  }}|xs |}|st        d"«       	 |
j5                  d#t        |||||f«       |
j<                  r|dz  }t        d$t        |«      › d%�«       nt        d&«       |dz  }|	jA                  «        t        j                  t        j                  dd«      «       �Œt |	jC                  «        t        d(«       t        d)|› d*|› d+|› �«       y c c}w # t>        $ r}t        d'|› �«       |dz  }Y d }~Œ’d }~ww xY w),Nú[z%H:%M:%Sz] r   z[ERROR] API failedÚcustomÚtotalr8   zTotal: z
 records, z pages (pageSize=ú)r   g333333Ó?g      à?z
Collected z itemsÚinfodater   zWithin 3 years: z (filtered: é   r   rJ   Úinfourlz  [r5   z...ú )Úendz6SELECT 1 FROM gov_raw WHERE page_url=? AND site_name=?Ú	DUPLICATEz	FETCH ERRzWARN (empty content)z’INSERT OR IGNORE INTO gov_raw
                (site_name, title, page_url, publish_date, source_url, content)
                VALUES (?,?,?,?,?,?)zOK (zB)zDUP(insert)zERR: z
=== Done ===zNew: z, Skipped: z
, Errors: )"r,   r   ÚnowÚstrftimeÚ	SITE_NAMEr3   r   r%   Úextendr*   r+   ÚrandomÚuniformÚlenÚgetÚCUTOFFÚsqlite3ÚconnectÚDB_PATHÚcursorÚ	enumeraterH   r   r:   r;   r<   ÚexecuteÚfetchonerA   rM   Úrowcountr)   ÚcommitÚclose)ÚresultrR   Útotal_pagesÚ	all_itemsÚpÚrÚitemÚrecentÚoldÚconnÚcÚ	new_countÚ
skip_countÚerror_countÚirJ   Úpub_dateÚpage_urlr@   Údetail_titlerL   Úfinal_titler1   s                          r2   Úmainr   \   sm  € Ü	ˆAŒh�l‰l‹n×%Ñ% jÓ1Ð2°"´Y°KÐ
@ÔAô �q‹\€FÙÜÐ"Ô#ØØ�8Ñ˜WÑ%€EØœ9Ñ$ qÑ(¬YÑ6€KÜ	ˆG�E�7˜* [ MÐ1BÄ9À+ÈQÐ
OÔPð �xÑ  Ñ(€IÜ�1�kÓ"ò -ˆÜ�a‹LˆÙ�˜1˜X™;Ñ&Ø×Ñ˜Q˜x™[¨Ñ0Ô1Ü�
‰
”6—>‘> # sÓ+Õ,ð	-ô 
ˆJ”s˜9“~Ð& fÐ
-Ô.ð  )ÖO�t¨D¯H©H°ZÀÓ,DÌÓ,NŠdÐO€FÐOÜ
ˆi‹.œ3˜v›;Ñ
&€CÜ	ÐœS ›[˜M¨°c°U¸!Ð
<Ô=ô �?‰?œ7¨BÔ/€DØ�‰‹€AØ+,Ð,€IÐ,�
˜[ä˜V QÓ'ó *-‰ˆˆ4Ø�W‘×#Ñ#Ó%ˆØ˜
Ñ#ˆÜ—<‘<×'Ñ'¬¨d°9©oÓ>ˆä��A�3�aœ˜F›�} B u¨S¨b z l°#Ð6¸CÕ@ð �9‰9ÐMØ¤	Ð*ó,ß,4©H«Jð7ä�+ÔØ˜!‰OˆJØô ˜D ™OÓ,ˆÙÜ�+ÔØ˜1ÑˆKØä ,¨TÓ 2Ñˆ�gØ"Ò+ eˆÙÜÐ(Ô)ð	Ø�I‰Ið (ô ˜K¨°8¸XÀwÐOôQð �zŠzØ˜Q‘�	Ü˜œS ›\˜N¨"Ð-Õ.ä�mÔ$Ø˜a‘�
ð
 	�‰ŒÜ�
‰
”6—>‘> # sÓ+Ö,ðU*-ðX 	‡J�J„LÜ	ˆNÔÜ	ˆE�)�˜K¨
 |°:¸k¸]Ð
KÕLùòo Pøô\ ò 	Ü�E˜!˜�+ÔØ˜1Ñ�Kûð	ús%   ÄMÄ,MÊAMÍ	M1ÍM,Í,M1Ú__main__)Ú__doc__ÚosrD   r!   rc   r*   Úsysr^   r   r   Úurllib.requestr   Úurllib.errorÚenvironra   re   r\   r    r<   r   rZ   r[   rb   r   r$   r3   rA   rM   r   Ú__name__© rN   r2   ú<module>r‰      s«   ðñ Jç /× /× /Ó /ß (ß #à
�*‰*�.‰.˜Ð&7Ó
8€Ø3€	Ø
Q€Ø€Ø2€	Ø
ˆ(�,‰,‹.™9¨%Ô0Ñ
0×	:Ñ	:¸:Ó	F€Ø€	ð DØ&ØXñ€òò2ò&"ò2NMðb ˆzÒÙ…Fð rN   