Ë
    %&˜j“  ã                   óX  — d Z ddlZddlZddlZddlZddlmZ ddlmZ ddlZ	 e
e	j                  «      dkD  r2e	j                  d   j                  «       r ee	j                  d   «      ndZe� ed ee«      z   «       dZdZd	ZdZd
ZddddœZd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)uÙ   


è¥¿å¡žå±±åŒºäººæ°‘æ”¿åºœ - é€šçŸ¥å…¬å‘Šçˆ¬è™«
CMS: TRS (æ‹“å°”æ€�)
åˆ†é¡µ: index_N.html (1~33é¡µ, æ¯�é¡µçº¦15æ�¡)
åˆ—è¡¨: div.news_list1 > ul > li > span(æ—¥æœŸ) + a[title](æ ‡é¢˜)
è¯¦æƒ…: div.view.TRS_UEDITOR > p
é    N)ÚBeautifulSoup)Úurljoiné   z[AutoPg] max_pages=z/mnt/data/search.dbz/http://www.xisaishan.gov.cn/xxgk/fdzdgknr/tzgg/u   è¥¿å¡žå±±åŒº-é€šçŸ¥å…¬å‘Šé#   zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagec                 óÈ  — | syg }| j                   D ]¾  }t        |dd«      }|dk(  r(|j                  dd¬«      }|sŒ+|j                  |«       Œ=|dv r|j                  t	        |«      «       Œ\|€5t        |t        «      r%|j                  «       }|sŒ�|j                  |«       Œ“|dv sŒ˜|j                  dd¬«      }|sŒ®|j                  |«       ŒÀ d	j                  |«      S )
uj   æ��å�–æ­£æ–‡å†…å®¹ï¼Œä¿�ç•™è¡¨æ ¼HTMLï¼Œæ®µè�½ç”¨\n\nåˆ†éš”ï¼ŒåŽ»æŽ‰å†…è�”span/font/bæ ‡ç­¾é˜²æ—¥æœŸæ‹†åˆ†Ú ÚnameNÚpT)Ú	separatorÚstrip)Útable)ÚdivÚsectionz

)ÚchildrenÚgetattrÚget_textÚappendÚstrÚ
isinstancer   Újoin)Úview_divÚpartsÚchildÚtag_nameÚtxts        ú$/root/gov_crawler/crawl_xisaishan.pyÚextract_contentr   %   sÔ   € áØà€EØ×"Ñ"ò "ˆÜ˜5 &¨$Ó/ˆØ�sŠ?à—.‘.¨2°T�.Ó:ˆCÚØ—‘˜SÕ!Ø˜Ñ#à�L‰Lœ˜U›Õ$ØÐ¤*¨U´CÔ"8Ø—+‘+“-ˆCÚØ—‘˜SÕ!ØÐ+Ò+à—.‘.¨2°T�.Ó:ˆCÚØ—‘˜SÕ!ð%"ð( �;‰;�uÓÐó    c                 ój  — 	 | dk(  r*t         }t        j                  |t        d¬«      }d|_        n/t         › d| › d�}t        j                  |t        d¬«      }d|_        |j
                  dk7  rt        d|j
                  › d	|› d
�«       yt        |j                  d«      }| dk(  r|j                  d«      }n|j                  d«      }|st        d| › �«       g S g }|D ]©  }|j                  d«      }|j                  d«      }	|r|j                  d«      sŒ9|d   j                  «       }
|j                  dd«      xs |j                  d¬«      }|	r|	j                  d¬«      nd}t        ||
«      }|j                  |||| dœ«       Œ« |S # t        $ r}t        d› d|› �«       Y d}~yd}~ww xY w)u   çˆ¬å�–æŒ‡å®šé¡µç �çš„åˆ—è¡¨r   é   ©ÚheadersÚtimeoutúutf-8Úindex_z.htmléÈ   ú  HTTP z for z
, stoppingNúhtml.parserzdiv.news_list1 ul lizdiv#op1 > ul > liz  Error fetching ú: z  No items found on page ÚaÚspanÚhrefÚtitler	   T©r   )r.   ÚurlÚdateÚpage_num)ÚBASE_URLÚrequestsÚgetÚHEADERSÚencodingÚstatus_codeÚprintr   ÚtextÚselectÚ	ExceptionÚfindr   r   r   r   )r2   r0   ÚrÚsoupÚitemsÚeÚresultsÚlir+   r,   Úlinkr.   Údate_strÚfull_urls                 r   Ú
crawl_pagerG   B   s­  € ðØ�qŠ=ÜˆCÜ—‘˜S¬'¸2Ô>ˆAØ ˆA�Jä�J˜f X J¨eÐ4ˆCÜ—‘˜S¬'¸2Ô>ˆAØ ˆAŒJà�=‰=˜CÒÜ�G˜AŸM™M˜?¨%°¨u°JÐ?Ô@Øä˜QŸV™V ]Ó3ˆà�qŠ=Ø—K‘KÐ 6Ó7‰Eà—K‘KÐ 3Ó4ˆEñ
 ÜÐ)¨(¨Ð4Ô5Øˆ	à€GØò ˆØ�G‰G�C‹LˆØ�w‰w�v‹ˆÙ˜Ÿ™˜fœØà�‰y�‰Ó ˆØ—‘�g˜rÓ"Ò< a§j¡j°t jÓ&<ˆÙ04�4—=‘= t�=Ô,¸"ˆä˜3 Ó%ˆà�‰ØØØØ ñ	
õ 	ðð& €Nøô9 ò ÜÐ! #  b¨¨Ð,Ô-Üûðús   ‚B	F Â>F Æ	F2ÆF-Æ-F2c                 ó^  — | d   }	 t        j                  |t        d¬«      }d|_        |j                  dk7  rt        d|j                  › d|› �«       y	 t        |j                  d«      }|j                  d«      xs |j                  d«      }|st        d|› �«       yt        |«      }|rt        |«      dk  rt        d|› �«       y|j                  d«      }|r|j                  d¬«      nd}|s| d   }||| d   |t        dœS # t        $ r}t        d	|› d
|› �«       Y d}~yd}~ww xY w)u   çˆ¬å�–è¯¦æƒ…é¡µr0   r!   r"   r%   r'   r(   z for detail Nz  Error fetching detail r*   r)   zdiv.view.TRS_UEDITORzdiv.Viewz  No content div found for é   z  Content too short for Úh1Tr/   r	   r.   r1   )r.   Úcontentr1   r0   Ú	site_name)r4   r5   r6   r7   r8   r9   r<   r   r:   Ú
select_oner   Úlenr=   r   Ú	SITE_NAME)	Úitemr0   r>   rA   r?   ÚviewrK   rJ   Útitle_h1s	            r   Úcrawl_detailrS   w   sA  € à
ˆu‰+€CðÜ�L‰L˜¤g°rÔ:ˆØˆŒ
Ø�=‰=˜CÒÜ�G˜AŸM™M˜?¨,°s°eÐ<Ô=Øð  ô ˜Ÿ™ Ó/€Dð �?‰?Ð1Ó2ÒQ°d·o±oÀjÓ6Q€DÙÜÐ+¨C¨5Ð1Ô2Øä˜dÓ#€GÙ”c˜'“l RÒ'ÜÐ(¨¨Ð.Ô/Øð 
�‰�4‹€BÙ*,ˆr�{‰{ ˆ{Ô&°"€HÙØ˜‘=ˆð ØØ�V‘ØÜñð øô/ ò ÜÐ(¨¨¨R°¨sÐ3Ô4Üûðús   ‡AD Ä	D,ÄD'Ä'D,c           
      ó¤  — ddl }| st        d«       y|j                  t        «      }|j	                  «       }d}| D ]=  }	 |j                  d|d   |d   |d   |d   |d	   f«       |j                  dkD  r|d
z  }Œ? |j                  «        |j                  «        |j                  «        |S # t        $ r}t        d|› �«       Y d}~ŒŽd}~ww xY w)u   æ‰¹é‡�å†™å…¥SQLiter   Nz  No records to savezhINSERT OR IGNORE INTO gov_raw (title, content, page_url, publish_date, site_name) VALUES (?, ?, ?, ?, ?)r.   rK   r0   r1   rL   r   z  DB error: )
Úsqlite3r9   ÚconnectÚDB_PATHÚcursorÚexecuteÚrowcountr<   ÚcommitÚclose)ÚrecordsrU   ÚconnÚcÚinsertedÚrecrA   s          r   Ú
save_to_dbrb       sÙ   € ãáÜÐ$Ô%Øà�?‰?œ7Ó#€DØ�‰‹€Aà€HØò 
&ˆð		&Ø�I‰Ið)à�W‘˜s 9™~¨s°5©z¸3¸v¹;ÈÈKÑHXÐYôð
 �z‰z˜AŠ~Ø˜A‘�øð
&ð 	‡K�K„Mð 	‡K�K„MØ‡J�J„Là€Oøô ò 	&Ü�L  Ð$×%Ñ%ûð	&ús   Á :B.Â.	CÂ7C
Ã
Cc            	      óh  — dd l } dt        «       v rAt        �;dd l}|j                  d   gdt        t        «      gz   |j                  dd  z   |_        | j                  d¬«      }|j                  ddd	¬
«       |j                  dt        t        d¬«       |j                  dt        t        d¬«       |j                  «       }d}d}t        |j                  |j                  |j                  z   «      D �]  }t        d|› d�«       t!        |«      }|€t        d|› d�«        nÙ|st        d|› d�«       ŒC|t#        |«      z  }t        dt#        |«      › d�«       g }|D ]5  }	t%        |	«      }
|
r|j'                  |
«       t)        j*                  d«       Œ7 |r+t-        |«      }||z  }t        d|› dt#        |«      › d�«       ||j                  |j                  z   dz
  k  sŒòt)        j*                  d«       �Œ	 t        dd› �«       t        d|› �«       t        d|› �«       t        d› «       y ) Nr   Ú_MAX_PGz--pagesé   zCrawl xisaishan tzgg)Údescriptionz--debugÚ
store_truez
Debug mode)ÚactionÚhelpzMax pages to crawl)ÚtypeÚdefaultri   z--startz
Start pagez
Page z...z!  No more pages (stopped at page ú)z  Empty page z
, skippingz  Found z itemsg      à?z  Inserted ú/z new recordsr   ú
z2==================================================zTotal found: zTotal inserted: )ÚargparseÚglobalsrd   ÚsysÚargvr   ÚArgumentParserÚadd_argumentÚintÚ	MAX_PAGESÚ
START_PAGEÚ
parse_argsÚrangeÚstartÚpagesr9   rG   rN   rS   r   ÚtimeÚsleeprb   )ro   rq   ÚparserÚargsÚtotal_foundÚtotal_insertedÚpager@   ÚbatchrP   Údetailr`   s               r   Úmainr…   Á   s  € ÛØ”G“IÑ¤'Ð"5ÛØ—H‘H˜Q‘K�= I¬s´7«|Ð#<Ñ<¸s¿x¹xÈÈ¸|ÑKˆŒà×$Ñ$Ð1GÐ$ÓH€FØ
×Ñ˜	¨,¸\ÐÔJØ
×Ñ˜	¬´YÐEYÐÔZØ
×Ñ˜	¬´ZÀlÐÔSØ×ÑÓ€Dà€KØ€Nä�d—j‘j $§*¡*¨t¯z©zÑ"9Ó:ó ˆÜ�˜�v˜SÐ!Ô"Ü˜4Ó ˆàˆ=ÜÐ5°d°V¸1Ð=Ô>ÙáÜ�M $  zÐ2Ô3Øà”s˜5“zÑ!ˆÜ�œ˜U›˜ FÐ+Ô,àˆØò 	ˆDÜ! $Ó'ˆFÙØ—‘˜VÔ$Ü�J‰J�s�Oð		ñ Ü! %Ó(ˆHØ˜hÑ&ˆNÜ�K ˜z¨¬3¨u«:¨,°lÐCÔDà�$—*‘*˜tŸz™zÑ)¨AÑ-Ó-Ü�J‰J�qŽMð9ô< 
ˆBˆvˆhˆ-ÔÜ	ˆM˜+˜Ð
'Ô(Ü	Ð˜^Ð,Ð
-Ô.Ü	ˆVˆHÕr   Ú__main__)Ú__doc__rq   Úosr|   r4   Úbs4r   Úurllib.parser   Ú_SYSrN   rr   Úisdigitru   rd   r9   r   rW   r3   rO   rw   rv   r6   r   rG   rS   rb   r…   Ú__name__© r   r   ú<module>r�      sÍ   ðñó Û 	Û Û Ý Ý  ã Ù" 4§9¡9›~°Ò1°d·i±iÀ±l×6JÑ6JÔ6L‰#ˆd�i‰i˜‰lÔ
ÐRV€Ø
ÐÙ	Ð
¡# g£,Ñ
.Ô/à
€Ø<€Ø'€	Ø€
Ø€	ðGàOØ0ñ	€òò:2òj&òRòB0ðf ˆzÒÙ…Fð r   