Ë
    ¸¬uj‘  ã                   óf  — d Z ddlZddlZddlZddlZddlmZ ddlmZmZ ddl	Z	dZ
dZ e	j                  dd«      Zd	Z ej                  «       Zej"                  j%                  d
di«        ej&                  «        ed¬«      z
  j)                  d«      Zd„ Zd„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)u‚   
Crawl ä¹�æ±Ÿå¸‚æ”¿åºœ - å…¬å‘Šå…¬ç¤º (jiujiang.gov.cn/zwzx/gggs/)
Static HTML with TRS pagination: index.html, index_2.html, ...
é    N)ÚBeautifulSoup)ÚdatetimeÚ	timedeltazhttps://www.jiujiang.gov.cnz
/zwzx/gggsÚ	SEARCH_DBz/root/search.dbu   ä¹�æ±Ÿå¸‚æ”¿åºœ - å…¬å‘Šå…¬ç¤ºz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36iG  )Údaysz%Y-%m-%dc                 óV  — g }t        | d«      }|j                  dd¬«      }|s|S |j                  d«      D ]ï  }|j                  d«      D ]Ù  }|j                  d«      }|sŒ|j                  dd	«      }|j	                  d
¬«      }|r|sŒ@|j                  d«      rd|z   }nG|j                  d«      r
t        |z   }n,|j                  d«      st        dz   |j                  d«      z   }|j                  d«      }	|	r|	j	                  d
¬«      nd	}
|j                  |||
dœ«       ŒÛ Œñ |S )z%Parse list page, return list of dictsúhtml.parserÚdivÚcontent©Úclass_ÚulÚliÚaÚhrefÚ T)Ústripz//zhttps:ú/ÚhttpÚspan)ÚtitleÚurlÚdate)	r   ÚfindÚfind_allÚgetÚget_textÚ
startswithÚBASE_URLÚlstripÚappend)Ú	html_textÚrecordsÚsoupr   r   r   r   r   r   r   Údate_strs              ú#/root/gov_crawler/crawl_jiujiang.pyÚparse_list_pager'      s-  € à€GÜ˜ MÓ2€DØ�i‰i˜ iˆiÓ0€GÙØˆà×Ñ˜tÓ$ò ˆØ—+‘+˜dÓ#ò 	ˆBØ—‘˜“ˆAÙØà—5‘5˜ Ó$ˆDØ—J‘J T�JÓ*ˆEÙ™uØð �‰˜tÔ$Ø $‘‘Ø—‘ Ô%Ü $‘‘Ø—_‘_ VÔ,Ü #‘~¨¯©°CÓ(8Ñ8�ð —7‘7˜6“?ˆDÙ48�t—}‘}¨4�}Ô0¸bˆHà�N‰NØØØ ñõ ñ-	ðð: €Nó    c                 ó‚  — 	 t         j                  | d¬«      }d|_        t	        |j
                  d«      }|j                  dd¬«      }|s|j                  dd	„ ¬«      }d
}|rt        |«      }d
}|j                  dddi¬«      }|r'|j                  d«      r|d   j                  «       dd }||fS # t        $ r Y yw xY w)z2Fetch detail page, return (content_html, date_str)é   ©Útimeoutúutf-8)r   r   r	   r
   zmain-body-wrapr   c                 óz   — | xr8 dt        | «      j                  «       v xs dt        | «      j                  «       v S )Nr   Úarticle)ÚstrÚlower)Úcs    r&   ú<lambda>zfetch_detail.<locals>.<lambda>M   s6   € ¸Ò8z¸yÌCÐPQËFÏLÉLËNÐ?ZÒ?yÐ^gÔknÐopÓkq×kwÑkwÓkyÐ^y€ r(   r   ÚmetaÚnameÚPubDate)Úattrsr   Né
   )	Úsessionr   ÚencodingÚ	Exceptionr   Útextr   r0   r   )r   Úrespr$   Úcontent_divÚcontent_htmlr%   r4   s          r&   Úfetch_detailr@   @   sÓ   € ðÜ�{‰{˜3¨ˆ{Ó+ˆØˆŒô ˜Ÿ™ MÓ2€Dð —)‘)˜EÐ*:�)Ó;€KÙØ—i‘i Ñ.z�iÓ{ˆà€LÙÜ˜;Ó'ˆð €HØ�9‰9�V F¨IÐ#6ˆ9Ó7€DÙ�—‘˜Ô#Ø˜	‘?×(Ñ(Ó*¨3¨BÐ/ˆà˜Ð!Ð!øô) ò Ùðús   ‚B2 Â2	B>Â=B>c                 óŽ  — |j                  «       }d}d}| D ]u  }|d   r|d   t        k  r|dz  }Œ	 |j                  d|d   t        |d   d|j	                  dd«      |j	                  dd«      t
        f«       |j                  dkD  r|dz  }Œw |j                  «        ||fS # t        $ r}t        d	|› �«       Y d
}~Œ¨d
}~ww xY w)zInsert records into gov_rawr   r   é   zÂ
                INSERT OR IGNORE INTO gov_raw 
                (page_url, source_url, title, summary, content, publish_date, site_name)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            r   r   r   r   z  DB error: N)
ÚcursorÚTHREE_YEARS_AGOÚexecuter   r   Ú	SITE_NAMEÚrowcountr;   ÚprintÚcommit)r#   ÚconnrC   ÚinsertedÚskippedÚrecÚes          r&   Úinsert_to_dbrO   [   sæ   € à�[‰[‹]€FØ€HØ€Gàò &ˆØˆvŠ;˜3˜v™;¬Ò8Ø�q‰LˆGØð	&Ø�N‰Nð ð
 �E‘
ÜØ�G‘ØØ—‘˜	 2Ó&Ø—‘˜ Ó#Üðô	ð �‰ Ò"Ø˜A‘�øð)&ð0 	‡K�K„MØ�WÐÐøô	 ò 	&Ü�L  Ð$×%Ñ%ûð	&ús   ²AB#Â#	CÂ,B?Â?Cc                 ó¤   — t        j                  d| «      }|r8t        |j                  d«      «      }t        |j                  d«      «      }||fS y)z'Extract total pages from createPageHTMLzcreatePageHTML\((\d+),\s*(\d+)rB   é   )rB   r   )ÚreÚsearchÚintÚgroup)r"   ÚmÚtotal_recordsÚtotal_pagess       r&   Úget_pagination_inforY   |   sF   € ä
�	‰	Ð3°YÓ?€AÙÜ˜AŸG™G A›J›ˆÜ˜!Ÿ'™' !›*“oˆØ˜MÐ)Ð)Ør(   c            	      ód  — dt         j                  v xs t         j                  d   dk(  } dd l}|j                  t        «      }| rd}t        d«       nd}t        › t        › d�}t        d	«       t        j                  |d
¬«      }d|_
        t        |j                  «      }t        dt        |«      › d�«       | s8t        |j                  «      \  }}|dkD  r||k  r|}t        d|› d|› d|› �«       t        d|dz   «      D ]É  }	t        › t        › d|	› d�}
	 t        j                  |
d
¬«      }d|_
        |j                   dk7  rt        d|	› d|j                   › d�«        nkt        |j                  «      }|st        d|	› d�«        nCt        d|	› dt        |«      › d�«       |j#                  |«       t%        j&                  d«       ŒË t        dt        |«      › �«       t        d«       t+        |«      D ]N  \  }}|dkD  r$|dz  dk(  rt        d |› d!t        |«      › �d"¬#«       t-        |d$   «      \  }}|r||d%<   |sŒJ||d&<   ŒP t/        ||«      \  }}|j1                  «        t        d'«       t        d(|› d)|› �«       y # t(        $ r}t        d|	› d|› �«       Y d }~ ŒÚd }~ww xY w)*NÚincrementaléÿÿÿÿÚ1r   rB   zMode: incremental (page 1 only)é   z/index.htmlzFetching page 1...r*   r+   r-   z
  Page 1: z recordsz	  Total: z
 records, z pages, crawling rQ   z/index_z.htmléÈ   z  Page z: HTTP z - stoppingz: empty - stoppingz: g333333Ó?z error: z
Total records collected: zFetching detail pages...r8   z  r   T)Úflushr   r   r   z
=== SUMMARY ===z
Inserted: z, Skipped (old): )ÚsysÚargvÚsqlite3ÚconnectÚDB_PATHrH   r   Ú	LIST_PATHr9   r   r:   r'   r<   ÚlenrY   ÚrangeÚstatus_codeÚextendÚtimeÚsleepr;   Ú	enumerater@   rO   Úclose)r[   rc   rJ   Úpages_to_crawlÚurl1r=   Úall_recordsrX   rW   ÚpageÚpage_urlr#   rN   ÚirM   r   Údate_from_detailrK   rL   s                      r&   Úmainrv   …   s½  € Ø¤3§8¡8Ð+ÒB¬s¯x©x¸©|¸sÑ/B€KãØ�?‰?œ7Ó#€DáØˆÜÐ/Õ0àˆô ˆZœ	�{ +Ð.€DÜ	ÐÔ Ü�;‰;�t Rˆ;Ó(€DØ€D„Mä! $§)¡)Ó,€KÜ	ˆJ”s˜;Ó'Ð(¨Ð
1Ô2áÜ%8¸¿¹Ó%CÑ"ˆ�]Ø˜Š?˜{¨^Ò;Ø(ˆNÜ�	˜-˜¨
°;°-Ð?PÐQ_ÐP`ÐaÔbô �a˜¨!Ñ+Ó,ò ˆÜ�Z¤	˜{¨'°$°°uÐ=ˆð	Ü—;‘;˜x°�;Ó4ˆDØ#ˆDŒMØ×Ñ 3Ò&Ü˜ ˜v W¨T×-=Ñ-=Ð,>¸kÐJÔKÙÜ% d§i¡iÓ0ˆGÙÜ˜ ˜vÐ%7Ð8Ô9ÙÜ�G˜D˜6 ¤C¨£L >°Ð:Ô;Ø×Ñ˜wÔ'Ü�J‰J�s�Oðô& 
Ð'¬¨KÓ(8Ð'9Ð
:Ô;ô 
Ð
$Ô%Ü˜KÓ(ò +‰ˆˆ3ØˆqŠ5�Q˜‘V˜q’[Ü�B�q�c˜œ3˜{Ó+Ð,Ð-°TÕ:ä$0°°U±Ó$<Ñ!ˆÐ!ÙØ$ˆC�	‰NÚØ*ˆC�ŠKð+ô % [°$Ó7Ñ€HˆgØ‡J�J„Lä	ÐÔÜ	ˆJ�x�jÐ 1°'°Ð
;Õ<øô/ ò 	Ü�G˜D˜6 ¨!¨Ð-Ô.Ýûð	ús&   ÄA	J
Å%&J
ÆAJ
Ê
	J/ÊJ*Ê*J/Ú__main__)Ú__doc__ra   rR   rk   ÚrequestsÚbs4r   r   r   Úosr   rf   Úgetenvre   rF   ÚSessionr9   ÚheadersÚupdateÚnowÚstrftimerD   r'   r@   rO   rY   rv   Ú__name__© r(   r&   ú<module>r„      sÅ   ðñ÷
 Ó Û Ý ß (Û 	à(€Ø€	Ø
ˆ"�)‰)�KÐ!2Ó
3€Ø,€	à
ˆ(×
Ñ
Ó
€Ø ‡�× Ñ Øð  Dðô ð  �8—<‘<“>¡I°5Ô$9Ñ9×CÑCÀJÓO€ò%òN"ò6òBòB=ðH ˆzÒÙ…Fð r(   