Ë
    ¸¬uj“  ã                   óZ  — d Z ddlZddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
  ej                  dd«      Z ej                  «        ed¬«      z
  j                  d	«      Zd
ZddiZdZe› d�Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r" e«       Z e ej:                  ed¬«      «       yy)u1  
å¦‚çš‹å¸‚é•¿æ±Ÿé•‡ - å…¬å‘Šå…¬ç¤º crawling script
Site: https://www.rugao.gov.cn/rgscjz/tzgg/tzgg.html
CMS: TrueCMS (jQuery jpage pagination)
Only page 1 accessible (30 items); pagination API behind broken dataproxy.jsp
For daily crawl, new items appear on page 1 â€” sufficient for incremental updates.
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupÚ	SEARCH_DBz/root/search.dbiG  )Údaysz%Y-%m-%du   å¦‚çš‹å¸‚é•¿æ±Ÿé•‡-å…¬å‘Šå…¬ç¤ºz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zhttps://www.rugao.gov.cnz/rgscjz/tzgg/tzgg.htmlc                  ó¦   — t        j                  t        «      } | j                  d«       | j                  d«       t         j                  | _        | S )NzPRAGMA journal_mode=WALzPRAGMA busy_timeout=10000)Úsqlite3ÚconnectÚDB_PATHÚexecuteÚRowÚrow_factory)Úconns    ú /root/gov_crawler/crawl_rugao.pyÚget_connr       s;   € Ü�?‰?œ7Ó#€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ü—{‘{€DÔØ€Kó    c                 óL   — | j                  d|f«      }|j                  «       d uS )Nz)SELECT id FROM gov_raw WHERE page_url = ?)r   Úfetchone)r   Úpage_urlÚcurs      r   Úis_dupr   '   s&   € Ø
�,‰,ÐBÀXÀKÓ
P€CØ�<‰<‹> Ð%Ð%r   c                 ó2  — |rd|v rt        |j                  dd«      «      nd}d}|r#t        |d«      }	|	j                  d¬«      }
|
d d }	 | j	                  d||||||||f«      }|j
                  dkD  S # t        $ r}t        d	|› �«       Y d }~y
d }~ww xY w)Nú-Ú r   úhtml.parserT©ÚstripéÈ   z�INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)z  [ERR] insert: F)ÚintÚreplacer   Úget_textr   ÚrowcountÚ	ExceptionÚprint)r   Ú	site_nameÚtitler   Úpublish_dateÚcontentÚcategoryÚ	date_rankÚsummaryÚ	text_soupÚplainr   Úes                r   Úinsert_itemr/   +   sº   € Ù6BÀsÈlÑGZ”�L×(Ñ(¨¨bÓ1Ô2Ð`a€IØ€GÙÜ! '¨=Ó9ˆ	Ø×"Ñ"¨Ð"Ó.ˆØ˜˜�+ˆð
Ø�l‰lð.ð ˜˜x¨°wÀ	È8ÐU\Ð]ó	
ˆð �|‰|˜aÑÐøÜò ÜÐ   Ð$Ô%Üûðús   Á(A5 Á5	BÁ>BÂBc                 óD  — g }t        | d«      }|j                  d¬«      }|r|j                  dd¬«      }n|j                  dd¬«      }|D ]Ñ  }|j                  d«      }|sŒ|j                  d«      }|j                  d	«      }|r|sŒ>|j                  d
d«      }	|j	                  d¬«      }
|j	                  d¬«      }|	r|
sŒyt        j                  d|	«      }|sŒ’|	j                  d«      r|	nt        |	z   }|j                  |j                  d«      |
||f«       ŒÓ |S )zKParse initData XML from page HTML, extract (uuid, title, url, date) tuples.r   ÚinitData)ÚidÚulzlist-ul©Úclass_ÚliÚaÚspanÚhrefr   Tr   z/([a-f0-9-]{36})\.htmlÚhttpé   )r   ÚfindÚfind_allÚgetr!   ÚreÚsearchÚ
startswithÚBASEÚappendÚgroup)ÚhtmlÚitemsÚsoupÚinit_divÚulsr3   r6   r7   r8   r9   r&   Údate_strÚmÚurls                 r   Úparse_list_pagerM   ?   s  € à€EÜ˜˜}Ó-€DØ�y‰y˜JˆyÓ'€HÙØ×Ñ ¨YÐÓ7‰à�m‰m˜D¨ˆmÓ3ˆàò 9ˆØ�W‰W�T‹]ˆÙØØ�G‰G�C‹LˆØ�w‰w�v‹ˆÙ™ØØ�u‰u�V˜RÓ ˆØ—
‘
 �
Ó&ˆØ—=‘= t�=Ó,ˆÙ™5Øä�I‰IÐ/°Ó6ˆÙØØ—o‘o fÔ-‰d´4¸$±;ˆØ�‰�a—g‘g˜a“j %¨¨hÐ7Õ8ð%9ð( €Lr   c                 ó,  — t        | d«      }|j                  dd¬«      }|r|j                  d¬«      nd}|j                  ddd	¬
«      }d}|rt        |«      }d}|j                  dddi¬«      }|rM|j	                  d«      r<|d   j                  «       }t        j                  d|«      }|r|j                  d«      }|sP|j                  dddi¬«      }	|	r9|	j	                  d«      r(t        j                  dd|	d   «      j                  «       }|||fS )z8Extract (title, content, publish_date) from detail page.r   Údivzcontainer-main-titler4   Tr   r   Úzoomztest-1)r2   r5   ÚmetaÚnameÚPubDate)Úattrsr(   z(\d{4}-\d{1,2}-\d{1,2})r;   ÚArticleTitlez<[^>]+>)
r   r<   r!   Ústrr>   r   r?   ÚmatchrD   Úsub)
rE   rG   Ú	title_divr&   rP   r(   Úpub_daterQ   Úm2Úmeta_ts
             r   Úparse_detailr]   `   s  € ä˜˜}Ó-€Dð —	‘	˜%Ð(>�	Ó?€IÙ.7ˆI×Ñ TÐÔ*¸R€Eð �9‰9�U˜v¨hˆ9Ó7€DØ€GÙä�d“)ˆð €HØ�9‰9�V F¨IÐ#6ˆ9Ó7€DÙ�—‘˜Ô#Ø˜	‘?×(Ñ(Ó*ˆÜ�X‰XÐ0°(Ó;ˆÙØ—x‘x “{ˆHáØ—‘˜6¨&°.Ð)A�ÓBˆÙ�f—j‘j Ô+Ü—F‘F˜: r¨6°)Ñ+<Ó=×CÑCÓEˆEà�'˜8Ð#Ð#r   c                  óV  — t        «       } t        dt        › dt        › �«       t	        j
                  t        t        dd¬«      }d|_        |j                  }t        |«      }t        dt        › dt        |«      › d�«       |D ����cg c]  \  }}}}|t        k\  sŒ||||f‘Œ }}}}}|D ���	�cg c]  \  }}}	}|t        k  sŒ|||f‘Œ }
}	}}}t        dt        › d	t        › d
t        |«      › d�«       |
r=t        dt        › dt        |
«      › �«       |
D ]  \  }}}t        d|› d|d d › �«       Œ d}d}d}t        |d«      D ]Ã  \  }\  }}}}	 t        | |«      r|dz  }Œt	        j
                  |t        dd¬«      }d|_        |j                  }t        |«      \  }}}|s|}|s|}t        | t        ||||d«      }|r|dz  }n|dz  }| j#                  «        |dz  dk(  sŒ t        d|› dt        |«      › d|› d|› d|› d�«       ŒÅ | j%                  «        t        dt        › d�«       t        d t        |«      › d!|› d"|› d#|› �«       t        t        |«      |||d$œS c c}}}}w c c}}	}}w # t         $ r)}|dz  }t        d|d d › d|d d › d|› �«       Y d }~ŒÔd }~ww xY w)%Nú[z] Fetching list page: é   F)ÚheadersÚtimeoutÚverifyzutf-8z] Found z items on page 1z] Within cutoff (z): z itemsz] Skipped (older): z  SKIP ú é2   r   r;   u   çŽ¯ä¿�å…¬ç¤ºz	  ERROR [é   z] é(   z: é
   z  Progress: ú/z (new=z, dup=z, err=ú)z] Done!z	  Total: z | New: z | Duplicate: z
 | Error: )ÚsiteÚtotalÚnewÚdupÚerr)r   r$   Ú	SITE_NAMEÚLIST_URLÚrequestsr>   ÚHEADERSÚencodingÚtextrM   ÚlenÚCUTOFF_DATEÚ	enumerater   r]   r/   r#   ÚcommitÚclose)r   ÚresprE   rF   ÚuÚtrL   ÚdÚactiveÚ_ÚskippedÚ	new_countÚ	dup_countÚerror_countÚidxÚuuidr&   rJ   Údetail_respÚdetail_htmlÚdetail_titler(   rZ   Úinsertedr.   s                            r   Úmainr‹   €   sí  € Ü‹:€Dô 
ˆAŒiˆ[Ð.¬x¨jÐ
9Ô:Ü�<‰<œ¬'¸2ÀeÔL€DØ€D„MØ�9‰9€Dä˜DÓ!€EÜ	ˆAŒiˆ[˜¤ U£ Ð,<Ð
=Ô>ð 27×KÑK¡  A s¨A¸!¼{Ó:Jˆq�!�S˜!ŠnÐK€FÓKØ+0×DÑD™Z˜Q  1 a°A¼³O��1�aŠyÐD€GÓDÜ	ˆAŒiˆ[Ð)¬+¨°c¼#¸f»+¸ÀfÐ
MÔNÙÜ�”)�Ð/´°G³¨~Ð>Ô?Øò 	*‰GˆAˆq�!Ü�G˜A˜3˜a  # 2 ˜xÐ(Õ)ð	*ð €IØ€IØ€Kä-6°v¸qÓ-Aò lÑ)ˆÑ)ˆd�E˜3 ð	=ä�d˜CÔ Ø˜Q‘�	Øä"Ÿ,™, s´GÀRÐPUÔVˆKØ#*ˆKÔ Ø%×*Ñ*ˆKä.:¸;Ó.GÑ+ˆL˜' 8ÙØ$�ÙØ#�ä" 4¬°LÀ#ÀxÐQXÐZhÓiˆHÙØ˜Q‘‘	à˜Q‘�	ð 	�‰Œà�‰8�q‹=Ü�L   Q¤s¨6£{ m°6¸)¸ÀFÈ9È+ÐU[Ð\gÐ[hÐhiÐjÕkð=lð@ 	‡J�J„Lä	ˆAŒiˆ[˜Ð
 Ô!Ü	ˆI”c˜&“k�] (¨9¨+°^ÀIÀ;ÈjÐYdÐXeÐ
fÔgÜ¬¨F«¸IÈiÐ`kÑlÐlùõc LùÝDøôF ò 	=Ø˜1ÑˆKÜ�I˜d 2 A˜h˜Z r¨%°°¨*¨°R¸°sÐ;×<Ñ<ûð	=ús7   Â	I&
ÂI&
Â1I.
ÃI.
ÅI6Å&A)I6É6	J(É?J#Ê#J(Ú__main__F)Úensure_ascii)Ú__doc__Úosr?   Úsysr	   rr   Újsonr   r   Úbs4r   Úgetenvr   ÚnowÚstrftimerw   rp   rs   rB   rq   r   r   r/   rM   r]   r‹   Ú__name__Úresultr$   Údumps© r   r   ú<module>rš      sÓ   ðñó 
Û 	Û 
Û Û Û ß (Ý ð ˆ"�)‰)�KÐ!2Ó
3€Øˆx�|‰|‹~¡	¨wÔ 7Ñ7×AÑAÀ*ÓM€Ø-€	ð ð  Dð€ð "€ØˆVÐ)Ð*€òò&òò(òB$ò@>mðB ˆzÒÙ‹V€FÙ	ˆ*ˆ$�*‰*�V¨%Ô
0Õ1ð r   