Ë
    ¸¬uj×  ã                   óœ   — d Z ddlZddlZddlZddlmZ ddlZ ej                  d«       dZe› d�Z	dZ
dZd	d
iZd„ Zd„ Zedk(  r e«        yy)z%Crawl jzsfyq.luan.gov.cn - Lonsun CMSé    N)ÚBeautifulSoupÚignorezhttps://jzsfyq.luan.gov.cnz/zwzx/tzgg/index.htmlz/root/search.dbu.   å®‰å¾½å…­å®‰é‡‘å®‰ç»�æµŽå¼€å�‘åŒº-å…¬ç¤ºå…¬å‘Šz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36c                 ó   — | j                  d«      }|s| j                  d«      }|syg }|j                  D ]q  }|j                  dk(  s|j                  dk(  r'|j                  d¬«      }|sŒ6|j	                  |«       ŒH|j                  dk(  sŒX|j	                  t        |«      «       Œs |s|j                  d	d¬
«      S d	j                  |«      S )z;Extract content from article page preserving HTML structureú'div#J_content.j-fontContent.newscontnetzdiv.newscontnetÚ ÚpÚdivT©ÚstripÚtableú

©Ú	separatorr   )Ú
select_oneÚdescendantsÚnameÚget_textÚappendÚstrÚjoin)ÚsoupÚcontÚresult_partsÚelÚtexts        ú!/root/gov_crawler/crawl_jzsfyq.pyÚextract_clean_textr      s¿   € à�?‰?ÐDÓE€DÙØ�‰Ð0Ó1ˆÙØà€LØ×Ñò )ˆØ�7‰7�cŠ>˜RŸW™W¨Ò-Ø—;‘; T�;Ó*ˆDÚØ×#Ñ# DÕ)Ø�W‰W˜ÓØ×Ñ¤ B£Õ(ð)ñ Ø�}‰} v°Tˆ}Ó:Ð:à�;‰;�|Ó$Ð$ó    c            
      óà  — t        j                  t        t        dd¬«      } d| _        t        | j                  d«      }|j                  d«      }t        dt        |«      › d�«       t        j                  t        «      }|j                  d	«       |j                  d
«       d}|D �]  }|j                  dd«      }|j                  dd«      xs |j                  d¬«      }|r|sŒA|j                  d«      s%|j                  d«      r	t         |z   n
t         › d|› �}|j#                  d«      d   }|j                  d|f«      j%                  «       }	|	rt        d|d d › �«       ŒÀ	 t        j                  |t        dd¬«      }
d|
_        t        |
j                  d«      }|j)                  d«      }|r|j                  d¬«      n|}d}|j)                  d«      }|rI|j                  «       }t+        j,                  d|«      }|r!|j/                  d«      j1                  dd«      }t3        |«      }|s&|j)                  d«      }|r|j                  dd¬ «      }|st        d!|› �«       d"}|r|d d# nd}	 |j                  d$t4        |||||f«       |j7                  «        |dz  }t        d%|d d › d&|› �«       �Œ |j=                  «        t        d)|› �«       y # t&        $ r}t        d|› d|› �«       Y d }~�ŒVd }~ww xY w# t        j8                  $ r% t        d'|d d › �«       |j;                  «        Y �Œ•t&        $ r)}|j;                  «        t        d(|› �«       Y d }~�ŒÁd }~ww xY w)*Né   F)ÚheadersÚtimeoutÚverifyzutf-8zhtml.parserzul.doc_list li a.leftzFound z items on pagezPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000r   Úhrefr   ÚtitleTr
   Úhttpú/z/zwzx/tzgg/ú?z(SELECT 1 FROM gov_raw WHERE page_url = ?z  SKIP (exists): z  ERROR fetching z: zh1.newstitlezdiv.newsinfou2   å�‘å¸ƒæ—¶é—´[ï¼š:]\s*(\d{4}[-/]\d{1,2}[-/]\d{1,2})é   ú-r   r   r   z  WARN: empty content for u   (æ— å†…å®¹)iô  zjINSERT INTO gov_raw (site_name, page_url, title, content, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)z  OK: z | z  DUPLICATE: z  DB ERROR: z
Done. New articles: )ÚrequestsÚgetÚLIST_URLr!   Úencodingr   r   ÚselectÚprintÚlenÚsqlite3ÚconnectÚDB_PATHÚexecuter   Ú
startswithÚBASE_URLÚsplitÚfetchoneÚ	Exceptionr   ÚreÚsearchÚgroupÚreplacer   Ú	SITE_NAMEÚcommitÚIntegrityErrorÚrollbackÚclose)Úrespr   ÚitemsÚdbÚ	new_countÚar$   r%   Úpage_urlÚexistsÚdrespÚdsoupÚeÚh1Ú
page_titleÚdateÚinfo_divÚtxtÚmÚcontentÚcontent_divÚsummarys                         r   ÚcrawlrW   )   sA  € Ü�<‰<œ¬'¸2ÀeÔL€DØ€D„MÜ˜Ÿ™ MÓ2€Dà�K‰KÐ/Ó0€EÜ	ˆF”3�u“:�,˜nÐ
-Ô.ä	�‰œÓ	!€BØ‡J�JÐ(Ô)Ø‡J�JÐ*Ô+à€IØó =&ˆØ�u‰u�V˜RÓ ˆØ—‘�g˜rÓ"Ò< a§j¡j°t jÓ&<ˆÙ™5ØØ�‰˜vÔ&Ø&*§o¡o°cÔ&:”8˜d’?Ä8À*ÈKÐX\ÐW]Ð@^ˆDà—:‘:˜c“? 1Ñ%ˆØ—‘ÐFÈÈÓT×]Ñ]Ó_ˆÙÜÐ% e¨C¨R j \Ð2Ô3Øð	Ü—L‘L ´7ÀBÈuÔUˆEØ$ˆEŒNÜ! %§*¡*¨mÓ<ˆEð ×Ñ˜nÓ-ˆÙ02�R—[‘[ t�[Ô,¸ˆ
ð ˆØ×#Ñ# NÓ3ˆÙØ×#Ñ#Ó%ˆCÜ—	‘	ÐOÐQTÓUˆAÙØ—w‘w˜q“z×)Ñ)¨#¨sÓ3�ô % UÓ+ˆÙØ×*Ñ*Ð+TÓUˆKÙØ%×.Ñ.¸ÀtÐ.ÓL�áÜÐ.¨z¨lÐ;Ô<Ø#ˆGá#*�'˜$˜3‘-°ˆð	&Ø�J‰JØ|Ü˜H j°'¸4ÀÐIôð �I‰IŒKØ˜‰NˆIÜ�F˜: c r˜?Ð+¨3¨t¨fÐ5Ö6ðo=&ð~ ‡H�H„JÜ	Ð" 9 +Ð
.Õ/øô] ò 	ÜÐ% h Z¨r°!°Ð5Ô6Ýûð	ûôL ×%Ñ%ò 	Ü�M *¨S¨b /Ð!2Ð3Ô4Ø�K‰K�MÜò 	&Ø�K‰KŒMÜ�L  Ð$×%Ò%ûð	&ús7   Å(:KÉ6ALË	LË&K=Ë=LÌ4M-Ì<M-ÍM(Í(M-Ú__main__)Ú__doc__r+   r2   r;   Úbs4r   ÚwarningsÚfilterwarningsr7   r-   r4   r?   r!   r   rW   Ú__name__© r   r   ú<module>r_      su   ðÙ +Û Û Û 	Ý Û Ø €× Ñ ˜Ô !à'€ØˆZÐ,Ð-€Ø
€Ø<€	ð ð  Dð€ò%ò.M0ð^ ˆzÒÙ	…Gð r   