Ë
    ¸¬uj5  ã                   óL  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
m
Z
mZ dZdZdZd	Zd
Z e
j"                  «        ed¬«      z
  j%                  d«      ZddiZ ej*                  ej,                  j.                  «       d„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u?   Crawler for www.xam.gov.cn - æŽªæ–½å�Šå®žæ–½æƒ…å†µ (Huilan CMS)é    N)ÚBeautifulSoup)Úurljoin)ÚdatetimeÚ	timedeltazhttps://www.xam.gov.cnzKhttps://www.xam.gov.cn/xam/zwgk73/zfxxgk31/fdzdgknr69/zdlyxx/sthj/csjssqk78z/root/search.dbé   u$   www.xam.gov.cn-æŽªæ–½å�Šå®žæ–½æƒ…å†µiG  )Údaysz%Y-%m-%dz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c                 óx   — t         › d�g}t        d| dz   «      D ]  }|j                  t         › d|› d�«       Œ |S )zGenerate list page URLs.z/index.htmlé   é   z
/f48ec2f5-z.html)Ú	LIST_BASEÚrangeÚappend)Ú	num_pagesÚurlsÚis      ú/root/gov_crawler/crawl_xam.pyÚget_list_urlsr      sI   € äˆk˜Ð%Ð&€DÜ�1�i !‘mÓ$ò 6ˆØ�‰”y�k ¨A¨3¨eÐ4Õ5ð6à€Kó    c                 óì  — t        | d«      }|j                  dd¬«      }|sg S g }|j                  dd¬«      D ]¸  }|j                  d«      }|j                  d	«      }|r|sŒ*|j                  d
d«      }|r|j	                  d«      rŒPt        t        |«      }|j                  d«      xs |j                  d¬«      j                  «       }	|j                  d¬«      }
|j                  |	||
f«       Œº |S )z3Parse list page, return list of (title, url, date).úhtml.parserÚulÚxxgkList©Úclass_ÚliF)Ú	recursiveÚaÚspanÚhrefÚ Ú
javascriptÚtitleT)Ústrip)
r   ÚfindÚfind_allÚgetÚ
startswithr   ÚBASE_URLÚget_textr#   r   )ÚhtmlÚsoupr   Úitemsr   r   r   r   Úfull_urlr"   Údates              r   Ú
parse_listr/   #   sè   € ä˜˜}Ó-€DØ	�‰�4 
ˆÓ	+€BÙØˆ	Ø€EØ�k‰k˜$¨%ˆkÓ0ò .ˆØ�G‰G�C‹LˆØ�w‰w�v‹ˆÙ™ØØ�u‰u�V˜RÓ ˆÙ�t—‘ |Ô4ØÜœ8 TÓ*ˆà—‘�w“Ò9 1§:¡:°D :Ó#9×@Ñ@ÓBˆØ�}‰} 4ˆ}Ó(ˆØ�‰�e˜X tÐ,Õ-ð.ð €Lr   c                 óŽ  — 	 t        j                  | t        dd¬«      }d|_        t	        |j
                  d«      }|j                  dd¬«      }|rt        |«      S |j                  dd	¬
«      }|rt        |«      S |j                  dd¬
«      }|rt        |«      S y# t        $ r}t        d| dd › d|› �«       Y d}~yd}~ww xY w)z8Fetch detail page and extract content from div#mainText.Fé   ©ÚheadersÚverifyÚtimeoutúutf-8r   ÚdivÚmainText)ÚidÚ
detail_txtr   Ú
detail_conr    z  [ERR] detail: Né<   ú: )
Úrequestsr&   ÚHEADERSÚencodingr   Útextr$   ÚstrÚ	ExceptionÚprint)ÚurlÚrespr+   Ú	main_textÚtxtÚconÚes          r   Úfetch_detail_contentrK   :   sÅ   € ðÜ�|‰|˜C¬¸ÈÔKˆØˆŒÜ˜TŸY™Y¨Ó6ˆð —I‘I˜e¨
�IÓ3ˆ	ÙÜ�y“>Ð!ð �i‰i˜ lˆiÓ3ˆÙÜ�s“8ˆOð �i‰i˜ lˆiÓ3ˆÙÜ�s“8ˆOàøÜò ÜÐ   S b  
¨"¨Q¨CÐ0Ô1Üûðús$   ‚AB ÁB Á<B Â	CÂ&B?Â?Cc                 ó0  ‡— t        | d«      }g }|j                  dd¬«      D ]p  }|d   Št        ˆfd„dD «       «      sŒ‰j                  d«      r|j	                  ‰«       Œ@‰j                  d	«      sŒR|j	                  t        t        ‰«      «       Œr |S )
z+Extract attachment links from content HTML.r   r   T)r   r   c              3   óB   •K  — | ]  }|‰j                  «       v –— Œ y ­w)N)Úlower)Ú.0Úextr   s     €r   ú	<genexpr>z&extract_attachments.<locals>.<genexpr>\   s   øè ø€ Òi sˆs�d—j‘j“lÔ"Ñiùs   ƒ)z.pdfz.docz.docxz.xlsz.xlsxz.zipz.rarÚhttpú/)r   r%   Úanyr'   r   r   r(   )r*   Úpage_urlr+   Úattachmentsr   r   s        @r   Úextract_attachmentsrW   V   s‹   ø€ ä˜˜}Ó-€DØ€KØ�]‰]˜3 Tˆ]Ó*ò <ˆØ�‰yˆÜÓiÐ.hÔiÕiØ�‰˜vÔ&Ø×"Ñ" 4Õ(Ø—‘ Õ%Ø×"Ñ"¤7¬8°TÓ#:Õ;ð<ð Ðr   c                  óž  — t        t        «      } g }t        dt        › dt	        | «      › d�«       | D ]�  }	 t        j                  |t        dd¬«      }d|_        t        |j                  «      }t        d|j                  d	«      d
   › dt	        |«      › d�«       |j                  |«       t        j                  d«       Œ‘ t!        «       }g }|D ]0  \  }}	}
|	|vsŒ|j#                  |	«       |j%                  ||	|
f«       Œ2 t        dt        › dt	        |«      › �«       |D ��	�
cg c]  \  }}	}
|
t&        k\  sŒ||	|
f‘Œ }}	}}
t        dt        › dt&        › dt	        |«      › �«       t)        j*                  t,        d¬«      }|j/                  «       }|j1                  dt        f«       |j3                  «       d   }t        dt        › d|› �«       dx}}|D ]à  \  }}}|j1                  d|f«       |j3                  «       d   dkD  r|dz  }Œ6t5        |«      }|rt	        |«      dk  r1t        d|d d › �«       t        j                  d«       t5        |«      }|rt	        |«      dk  r|dz  }Œ˜|j1                  dt        |||||d d f«       |j7                  «        |dz  }t        j                  d«       Œâ t        dt        › d |› d!|› d"�«       |j1                  dt        f«       |j3                  «       d   }t        dt        › d#|› �«       |dkD  rD|j1                  d$t        f«       |j9                  «       D ]  \  }}t        d|d d% › d|› d&�«       Œ |j;                  «        t        |||d'œ}t        d(t=        j>                  |d¬)«      › �«       y # t        $ r}t        d|› d|› �«       Y d }~�Œd }~ww xY wc c}
}	}w )*Nú[z] Scanning z	 pages...Fr1   r2   r6   z  rS   éÿÿÿÿz -> z itemsz  [ERR] r=   g      à?z
[z] Total unique: z] In 3 years (>z): é   )r5   z.SELECT COUNT(*) FROM gov_raw WHERE site_name=?r   z] Existing: z-SELECT COUNT(*) FROM gov_raw WHERE page_url=?r   z  [WARN] Short content for r
   ztINSERT OR IGNORE INTO gov_raw (site_name, title, content, page_url, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)éÈ   gš™™™™™é?z] Done: z new, z skippedz] Total in DB: zDSELECT title, length(content) FROM gov_raw WHERE site_name=? LIMIT 3é#   z chars)ÚsiteÚnewÚskippedÚtotalz
JSON_OUTPUT:)Úensure_ascii) r   Ú	MAX_PAGESrD   Ú	SITE_NAMEÚlenr>   r&   r?   r@   r/   rA   ÚsplitÚextendrC   ÚtimeÚsleepÚsetÚaddr   ÚCUTOFF_DATEÚsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚfetchonerK   ÚcommitÚfetchallÚcloseÚjsonÚdumps)Ú	page_urlsÚ	all_itemsÚpurF   r,   rJ   ÚseenÚuniqueÚtÚuÚdÚfilteredÚconnÚcÚexistingÚ	new_countÚ
skip_countr"   rE   r.   Úcontent_htmlra   ÚlÚsummarys                           r   Úmainr‰   d   s­  € ÜœiÓ(€IØ€Iä	ˆAŒiˆ[˜¤C¨	£NÐ#3°9Ð
=Ô>Øò 	ˆð	(Ü—<‘< ¬G¸EÈ2ÔNˆDØ#ˆDŒMÜ˜tŸy™yÓ)ˆEÜ�B�r—x‘x “} RÑ(Ð)¨¬c°%«j¨\¸Ð@ÔAØ×Ñ˜UÔ#ô 	�
‰
�3�ð	ô ‹5€DØ€FØò %‰ˆˆ1ˆaØ�DŠ=Ø�H‰H�QŒKØ�M‰M˜1˜a ˜)Õ$ð%ô
 
ˆC”	ˆ{Ð*¬3¨v«;¨-Ð
8Ô9ð *0×DÐD™g˜a  A°1¼Ó3C��A�q’	ÐD€HÒDÜ	ˆAŒiˆ[˜¬ }°C¼¸H»°Ð
GÔHô �?‰?œ7¨BÔ/€DØ�‰‹€Að ‡I�IÐ>ÄÀÔMØ�z‰z‹|˜A‰€HÜ	ˆAŒiˆ[˜ X JÐ
/Ô0àÐ€I�
à$ò Ñˆˆs�DØ	�	‰	ÐAÀCÀ6ÔJØ�:‰:‹<˜‰?˜QÒØ˜!‰OˆJØä+¨CÓ0ˆáœs <Ó0°2Ò5ÜÐ/°°c°r°
¨|Ð<Ô=Ü�J‰J�qŒMÜ/°Ó4ˆLáœs <Ó0°2Ò5Ø˜!‰OˆJØà	�	‰	ð CÜ˜˜|¨S°$¸¸d¸s¸ÐDô	
ð 	�‰ŒØ�Q‰ˆ	Ü�
‰
�3�ð/ô2 
ˆC”	ˆ{˜( 9 +¨V°J°<¸xÐ
HÔIà‡I�IÐ>ÄÀÔMØ�J‰J‹L˜‰O€EÜ	ˆAŒiˆ[˜¨ wÐ
/Ô0àˆq‚yØ	�	‰	ÐXÔ[dÐZfÔgØ—J‘J“Lò 	,‰DˆAˆqÜ�B�q˜˜"�v�h˜b   6Ð*Õ+ð	,ð 	‡J�J„Lä ¨¸zÐTYÑZ€GÜ	ˆNœ4Ÿ:™: g¸EÔBÐCÐ
DÕEøôI ò 	(Ü�H˜R˜D  1 #Ð&×'Ò'ûð	(üô Es$   ·A7N Ä*OÄ=OÎ 	OÎ)O Ï OÚ__main__)Ú__doc__r>   rm   rv   rh   Úurllib3Úbs4r   Úurllib.parser   r   r   r(   r   ro   rc   rd   ÚnowÚstrftimerl   r?   Údisable_warningsÚ
exceptionsÚInsecureRequestWarningr   r/   rK   rW   r‰   Ú__name__© r   r   ú<module>r–      s»   ðá Eã Û Û Û Û Ý Ý  ß (à#€ØY€	Ø
€Ø€	Ø2€	Øˆx�|‰|‹~¡	¨wÔ 7Ñ7×AÑAÀ*ÓM€ð ð  Dð€ð €× Ñ ˜×+Ñ+×BÑBÔ Còòò.ò8òPFðf ˆzÒÙ…Fð r   