Ë
    €Ûµj~%  ã                   óÜ   — d Z ddlZddlZddlZddlZddlZddlZddlZddlmZ ddl	m
Z
 ddlmZ dZdZdZd	Zd
ZdZdZdZddddœZd„ Zd„ Zd„ Zd„ Zdd„Zd„ Zd„ Zd„ Zedk(  r e«        yy)u,  
Crawler for é»”å�—çƒ­çº¿ - çƒ­ç‚¹å…³æ³¨ (qnz.com.cn)
CMS: SiteServer (STL dynamic pagination)
List: ul.list > li > p > a[href][title] + span (date)
       Pages 2+: AJAX POST https://sites.qnz.com.cn/api/stl/actions/pagecontents
Pagination: hot.html (page1), hot_{N}.html (page N), 24/page, 42 pages ~1000 items
Detail: div.show-content > p (paragraphs), span.time (date), span.source (source)

Usage:
    python3 /root/crawl_qnz_hot.py             # Full crawl (all 42 pages)
    python3 /root/crawl_qnz_hot.py --max-pages 1  # Incremental (page 1 only)
é    N)Údatetime)ÚBeautifulSoup)Úurljoinz/root/search.dbu   é»”å�—çƒ­çº¿-çƒ­ç‚¹å…³æ³¨u   çƒ­ç‚¹å…³æ³¨u   è´µå·žz$https://www.qnz.com.cn/news/hot.htmlz5https://sites.qnz.com.cn/api/stl/actions/pagecontentsé*   é   zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9,en;q=0.8)z
User-AgentÚAcceptzAccept-Languagec                  ó  — 	 t        j                  dt        d¬«      } d| _        t	        j
                  d| j                  «      }|r|j                  d«      S 	 y	# t        $ r(}t        d|› �t        j                  ¬«       Y d	}~y	d	}~ww xY w)
z+Get stlPageContentsElement from page 2 HTMLz&https://www.qnz.com.cn/news/hot_2.htmlé   ©ÚheadersÚtimeoutúutf-8z,stlPageContentsElement:\s*\x27([^\x27]+)\x27é   z  Error getting API token: ©ÚfileN)ÚrequestsÚgetÚHEADERSÚencodingÚreÚsearchÚtextÚgroupÚ	ExceptionÚprintÚsysÚstderr)ÚrÚmÚes      ú"/root/gov_crawler/crawl_qnz_hot.pyÚextract_api_tokenr"   %   sƒ   € ðBÜ�L‰LÐAÌ7Ð\^Ô_ˆØˆŒ
Ü�I‰IÐEÀqÇvÁvÓNˆÙØ—7‘7˜1“:Ðð ð øô ò BÜÐ+¨A¨3Ð/´c·j±j×AÑAØûðBús   ‚AA Á	BÁ#BÂBc                 óX  — g }t        | d«      }|j                  dd¬«      }|s|j                  d«      }|s|S |j                  dd¬«      }|D ]Ù  }|j                  d«      }|j                  d	«      }|r|j                  d
«      sŒ9|d
   j	                  «       }	|j                  dd«      j	                  «       xs |j                  d¬«      }
t        ||	«      }|r|j                  d¬«      nd}|rt        j                  dd|«      dd nd}|
sŒÃ|sŒÆ|j                  |
||f«       ŒÛ |S )z8Parse list HTML (from inline or API response) into itemsúhtml.parserÚulÚlist©Úclass_ÚliF)Ú	recursiveÚaÚspanÚhrefÚtitleÚ T©Ústripú\s+Né
   )
r   ÚfindÚfind_allr   r1   Úget_textr   r   ÚsubÚappend)ÚhtmlÚbase_urlÚitemsÚsoupr%   Úlisr)   r+   r,   r-   r.   Úpage_urlÚdate_strs                r!   Úparse_list_htmlr@   2   s  € à€EÜ˜˜}Ó-€DØ	�‰�4 ˆÓ	'€BÙØ�m‰m˜DÓ!ˆÙØˆLà�k‰k˜$¨%ˆkÓ0ˆàò 6ˆØ�G‰G�C‹LˆØ�w‰w�v‹ˆÙ˜Ÿ™˜fœØØ�‰y�‰Ó ˆØ—‘�g˜rÓ"×(Ñ(Ó*ÒD¨a¯j©j¸t¨jÓ.DˆÜ˜8 TÓ*ˆÙ04�4—=‘= t�=Ô,¸"ˆÙ8@”2—6‘6˜& " hÓ/°°Ñ4ÀbˆÚ’XØ�L‰L˜% ¨8Ð4Õ5ð6ð €Ló    c                 óŒ  — | dk(  rT	 t        j                  t        t        d¬«      }d|_        |j
                  dk(  rt        |j                  t        «      S 	 g S |sg S dd	d
dt        | |dœ}	 t        j                  t        |i t        ¥ddi¥d¬«      }|j
                  dk(  r4|j                  «       }|j                  dd«      }|rt        |t        «      S g S # t        $ r)}t        d|› �t        j                  ¬«       Y d}~g S d}~ww xY w# t        $ r,}t        d| › d|› �t        j                  ¬«       Y d}~g S d}~ww xY w)z Get list items for a page numberr   r
   r   r   éÈ   z  Error fetching page 1: r   Ni.  é   iè  )ÚsiteIdÚpageChannelIdÚ
templateIdÚtotalNumÚ	pageCountÚcurrentPageIndexÚstlPageContentsElementzContent-Typezapplication/json)Újsonr   r   r9   r/   z  Error fetching page z
 via API: )r   r   ÚLIST_URLr   r   Ústatus_coder@   r   r   r   r   r   ÚTOTAL_PAGESÚpostÚAPI_URLrL   )Úpage_numÚ	api_tokenr   r    ÚparamsÚdatar9   s          r!   Úget_list_pagerV   M   sE  € à�1‚}ð	DÜ—‘œX¬wÀÔCˆAØ ˆAŒJØ�}‰} Ò#Ü& q§v¡v¬xÓ8Ð8ð $ð ˆ	áØˆIàØ ØØÜ$Ø (Ø&/ñ
ˆð	UÜ—‘œg¨FÐ<k¼wÐ<kÈÐXjÑ<kÐuwÔxˆAØ�}‰} Ò#Ø—v‘v“x�Ø—x‘x ¨Ó+�ÙÜ*¨4´Ó:Ð:ð ˆ	øô1 ò 	DÜÐ-¨a¨SÐ1¼¿
¹
×CÑCØˆ	ûð	Dûô, ò 	UÜÐ*¨8¨*°J¸q¸cÐBÌÏÉ×TÑTØˆ	ûð	Uús1   ‡AC Á-A)D Ã	DÃ"DÄDÄ	EÄ!D>Ä>Ec                 ó  — 	 t        j                  | t        d¬«      }d|_        t        |j                  d«      }d}|j                  d	«      }|r0|j                  d
¬«      j                  dd«      j                  «       }d}|j                  dd¬«      }|r|j                  d
¬«      d d }|j                  dd¬«      }d}	|r¶g }
|j                  d«      D ]�  }|j                  d
¬«      }|sŒ|j                  d«      }|rT|j                  dd«      }|sŒ@t        | |«      }|j                  dd«      }|
j!                  |r	d|› d|› d�nd|› d�«       Œ|
j!                  |«       Œ‘ dj#                  |
«      }	g }t%        «       }|j                  dd
¬«      D ]©  }|d   }t'        j(                  d|t&        j*                  «      sŒ.t        | |«      }|j                  d
¬«      xs& |j-                  d «      d!   j-                  d"«      d#   }||vsŒy|j/                  |«       |j!                  ||d$œ«       |	d%|› d&|› d'�z  }	Œ« |||	|fS # t        $ r1}t        d| › d|› �t        j                  ¬«       d d d g fcY d }~S d }~ww xY w)(Nr
   r   r   z  fetch_detail error ú: r   r$   r/   r.   Tr0   u    - é»”å�—çƒ­çº¿r,   Útimer'   r3   Údivzshow-contentÚpÚimgÚsrcÚaltz![ú](ú)z![](z

r+   )r-   r-   z&\.(pdf|doc|docx|xls|xlsx|rar|zip|txt)$ú/éÿÿÿÿú?r   )ÚnameÚurlu   

é™„ä»¶ï¼š<p><a href="z">z</a></p>)r   r   r   r   r   r   r   r   r   r   r4   r6   Úreplacer1   r5   r   r8   ÚjoinÚsetr   r   ÚIÚsplitÚadd)re   r   r    r<   r.   Útr?   r,   Úcontent_divÚcontentÚpartsr[   Útxtr\   r]   Úfull_srcr^   ÚattachmentsÚseenr+   r-   Úfull_urlrd   s                          r!   Úfetch_detailru   p   s‡  € ð$Ü�L‰L˜¤g°rÔ:ˆØˆŒ
ô
 ˜Ÿ™ Ó/€DØ€EØ�	‰	�'Ó€AÙØ—
‘
 �
Ó&×.Ñ.Ð/@À"ÓE×KÑKÓMˆà€HØ�9‰9�V Fˆ9Ó+€DÙØ—=‘= t�=Ó,¨S¨bÐ1ˆà—)‘)˜E¨.�)Ó9€KØ€GÙØˆØ×%Ñ% cÓ*ò 	&ˆAØ—*‘* 4�*Ó(ˆCÚØ—f‘f˜U“m�ÙØŸ'™' %¨Ó,�CÚÜ#*¨3°Ó#4˜Ø!Ÿg™g e¨RÓ0˜ØŸ™Á r¨#¨¨b°°
¸!Ñ%<È4ÐPXÈzÐYZÐI[Õ\à—L‘L Õ%ð	&ð —+‘+˜eÓ$ˆà€KÜ‹5€DØ�]‰]˜3 Tˆ]Ó*ò RˆØ�‰yˆÜ�9‰9Ð>ÀÄbÇdÁdÕKÜ˜s DÓ)ˆHØ—:‘: D�:Ó)ÒN¨T¯Z©Z¸«_¸RÑ-@×-FÑ-FÀsÓ-KÈAÑ-NˆDØ˜tÒ#Ø—‘˜Ô"Ø×"Ñ"¨D¸Ñ#BÔCØÐ6°x°jÀÀ4À&ÈÐQÑQ‘ðRð �(˜G [Ð0Ð0øô[ ò $ÜÐ% c U¨"¨Q¨CÐ0´s·z±zÕBØ�T˜4 Ð#Õ#ûð$ús   ‚#I
 É
	JÉ&I?É9JÉ?Jc                 ól  — t        d|dz   «      D ]  }	  | «       c S  y
# t        j                  $ r�}dt        |«      v rC||k  r>t	        d|› d|› d|› d|› d�	t
        j                  ¬«       t        j                  |«       n't	        d|› d	|› �t
        j                  ¬«       Y d }~ y
Y d }~Œžd }~ww xY w)Nr   Úlockedz  z: locked (attempt ra   z), retry in zs...r   z	 failed: F)	ÚrangeÚsqlite3ÚOperationalErrorÚstrr   r   r   rY   Úsleep)ÚfnÚdescÚmax_attemptsÚdelayÚattemptr    s         r!   Ú
with_retryr‚   ¤   s¸   € Ü˜˜L¨1Ñ,Ó-ò 	ˆð	Ù“4ŠKð	ð øô ×'Ñ'ò 	Øœ3˜q›6Ñ! g°Ò&<Ü˜˜4˜&Ð 2°7°)¸1¸\¸NÈ,ÐW\ÐV]Ð]aÐbÔil×isÑisÕtÜ—
‘
˜5Õ!ä˜˜4˜& 	¨!¨Ð-´C·J±JÕ?Ýô "ûð	ús   ”ŸB3²A1B.Â.B3c                  ó€   — t        j                  t        d¬«      } | j                  d«       | j                  d«       | S )Né   )r   zPRAGMA journal_mode=WALzPRAGMA busy_timeout=30000)ry   ÚconnectÚDB_PATHÚexecute©Úconns    r!   Úinit_dbrŠ   ²   s1   € Ü�?‰?œ7¨BÔ/€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ø€KrA   c                 ó�  — |r|d d n|}t        j                  dd|«      j                  «       }|rt        j                  |d¬«      nd}	 | j                  dt        t        ||j                  «       |||t        |t        f
«      }|j                  dkD  S # t        $ r(}	t        d	|	› �t        j                  ¬
«       Y d }	~	yd }	~	ww xY w)NrC   r2   ú F)Úensure_asciiz[]zÍINSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date, summary, content, category, attachments, group_name)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)r   z  DB error: r   )r   r7   r1   rL   Údumpsr‡   Ú	SITE_NAMEÚCATEGORYÚGROUPÚrowcountr   r   r   r   )
r‰   r.   r>   Úpublish_datern   rr   ÚsummaryÚatt_jsonÚcurr    s
             r!   Úsave_articler—   ¹   s´   € Ù&ˆg�d�s‰m¨E€GÜ�f‰f�V˜S 'Ó*×0Ñ0Ó2€GÙ>IŒt�z‰z˜+°EÕ:Èt€HðØ�l‰lð8ô œ	 8¨U¯[©[«]¸LØ�gœx¨´5ð:ó	
ˆð �|‰|˜aÑÐøÜò Ü�˜Q˜CÐ ¤s§z¡zÕ2Üûðús   ÁAB Â	CÂC Ã Cc            
      ó<  ‡— t         } t        t        j                  «      dkD  r	 t	        t        j                  d   «      } t        dt        › d| › �t        j                  ¬«       t        «       }|r)t        dt        |«      › d�t        j                  ¬«       nt        d	t        j                  ¬«       t        «       Šd
}d
}ˆfd„}t        |dt        › �¬«       t        dt        › �t        j                  ¬«       t        d| dz   «      D �]\  }t        ||«      }|s"t        d|› d�t        j                  ¬«        �n,|t        |«      z  }|D ]Ø  \  }}}	|	r*|	dk  r%t        d|	› d|d d › �t        j                  ¬«       Œ3‰j                  d|f«      }
|
j                  «       rŒWt!        |«      \  }}}}|xs |}|xs |	}|xs	 d|› d|› d�}|xs g }t#        ‰|||||«      r.|dz  }|dk  r$t        d|› d|d d › �t        j                  ¬«       t%        j&                  d«       ŒÚ t        d|› dt        |«      › d|› d�t        j                  ¬«       ‰j)                  «        �Œ_ ‰j)                  «        ‰j+                  «        t        d |› d!|› �t        j                  ¬«       y # t
        $ rb t        j                  d   dv rIt        t        j                  «      dkD  r-	 t	        t        j                  d   «      } n# t
        $ r Y nw xY wY �ŒÙw xY w)"Nr   )z--max-pagesrD   z	Scraping z, max pages: r   z  API token obtained (z chars)z*  WARNING: No API token found, only page 1r   c                  ó&   •— ‰ j                  «        y )N)Úcommitrˆ   s   €r!   Údo_cleanzmain.<locals>.do_cleanã   s   ø€ à�‰�rA   zClean )r~   z  Cleaned old data for z  Page z: 0 articles (end)z
2020-01-01z  Skip old: rŒ   é(   z)SELECT id FROM gov_raw WHERE page_url = ?ú[r_   r`   é   z  + é2   g      à?rX   z found, z totalz
Done! Found: z, New: )rO   Úlenr   ÚargvÚintÚ
ValueErrorr   r�   r   r"   rŠ   r‚   rx   rV   r‡   Úfetchoneru   r—   rY   r|   rš   Úclose)Ú	max_pagesrS   Ú	total_newÚtotal_foundr›   ÚpnÚarticlesr.   r>   Údater–   Ú	det_titleÚdet_dateÚdet_contentÚdet_attsÚfinal_titleÚ
final_dateÚfinal_contentÚ
final_attsr‰   s                      @r!   Úmainr´   Ë   sÍ  ø€ Ü€IÜ
Œ3�8‰8ƒ}�qÒð	ÜœCŸH™H Q™KÓ(ˆIô 
ˆI”i�[ ¨i¨[Ð
9ÄÇ
Á
ÕKä!Ó#€IÙÜÐ&¤s¨9£~Ð&6°gÐ>ÄSÇZÁZÖPäÐ:ÄÇÁÕLä‹9€DØ€IØ€Kôô ˆx ¤y kÐ2Õ3Ü	Ð#¤I ;Ð
/´c·j±jÕAä�A�y 1‘}Ó%ó ˆÜ   YÓ/ˆÙÜ�G˜B˜4Ð1Ð2¼¿¹ÕDÚØ”s˜8“}Ñ$ˆØ%-ò 	Ñ!ˆE�8˜TÙ˜˜|Ò+Ü˜ T F¨!¨E°#°2¨J¨<Ð8¼s¿z¹zÕJØØ—,‘,ÐJÈXÈKÓXˆCØ�|‰|Œ~ØÜ9EÀhÓ9OÑ6ˆI�x ¨hØ#Ò, uˆKØ!Ò) TˆJØ'ÒI¨Q¨{¨m¸2¸h¸ZÀqÐ+IˆMØ!š RˆJÜ˜D +¨x¸À]ÐT^Ô_Ø˜Q‘�	Ø ’>Ü˜D  ¨A¨k¸#¸2Ð.>Ð-?Ð@ÄsÇzÁzÕRÜ�J‰J�s�Oð!	ô" 	�˜�t˜2œc (›m˜_¨H°Y°K¸vÐFÌSÏZÉZÕXØ�‰Žð1ð4 	‡K�K„MØ‡J�J„LÜ	ˆO˜K˜=¨°	¨{Ð
;Ä#Ç*Á*ÖMøôk ò 	Ü�x‰x˜‰{Ð.Ñ.´3´s·x±x³=À1Ò3DðÜ #¤C§H¡H¨Q¡KÓ 0‘IøÜ!ò Ùðýð		ús5   ¥J0 Ê0:LË+LÌLÌ	LÌLÌLÌLÌLÚ__main__)zDB opé   rD   )Ú__doc__r   r   rL   ry   rY   Úosr   r   Úbs4r   Úurllib.parser   r†   r�   r�   r‘   rM   rQ   rO   ÚPER_PAGEr   r"   r@   rV   ru   r‚   rŠ   r—   r´   Ú__name__© rA   r!   ú<module>r¾      sœ   ðñ÷ 2× 1× 1Ó 1Ý Ý Ý  à
€Ø'€	Ø€Ø€Ø1€Ø
A€Ø€Ø€ð DØOØ0ñ€ò
òò6 òF11óhòòò$:Nðz ˆzÒÙ…Fð rA   