Ë
    t$}j_  ã                   óô   — d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
mZ  ej                  dd«      ZdZdZd	Zd
ZdZdZdZ e	ddd«      ZdZdde› d�dœZd„ Zd„ Zd„ Zedk(  r e«        yy)u—  
ç»�å…´å¸‚ä¸Šè™žåŒºäººæ°‘æ”¿åºœ - å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·ä¿¡æ�¯å…¬ç¤º çˆ¬è™«
CMS: JCMS (æµ™æ±Ÿæ”¿åŠ¡ jpaas-publish-server)
åˆ—è¡¨: GET api-gateway/jpaas-publish-server/front/page/build/unit
      (pageId=1229857239, åˆ†é¡µç”¨ paramJson={"pageNo":N,"pageSize":"30"})
è¯¦æƒ…: curl ç›´è¿ž, æ­£æ–‡åœ¨ div.article, å®Œæ•´æ ‡é¢˜åœ¨ h2
ç”¨æ³•: python3 crawl_shangyu.py [--pages N]  (é»˜è®¤ 5 é¡µ = 150 æ�¡)
é    N)ÚdatetimeÚdate)ÚBeautifulSoupÚ	SEARCH_DBz/root/search.dbu"   ç»�å…´ä¸Šè™žåŒºæ”¿åºœ-çŽ¯è¯„å…¬ç¤ºzQhttps://www.shangyu.gov.cn/api-gateway/jpaas-publish-server/front/page/build/unitzhttps://www.shangyu.gov.cnÚ2328Ú
1229857239u   å½“å‰�æ �ç›®åˆ—è¡¨1aÚ9nuI0IJgTDxmXUM0arEPbiç  é   é   é   zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36z"https://www.shangyu.gov.cn/col/colz/index.html?key)z
User-AgentÚRefererc                 ó€  — t         t        ddt        t        dœ}| dkD  rt	        j
                  | ddœd¬«      |d	<   t        j                  t        |t        d
¬«      }|j                  «        |j	                  «       }|j                  d«      s"t        d| › d|j                  d«      › �«       g S t        |d   d   d«      }g }|j                  d«      D ]æ  }|j                  d«      }|sŒ|j                  dd«      }|sŒ,|j                  d«      s	t         |z   }|j                  d«      xs |j#                  d¬«      j%                  «       }	d}
|j                  d«      D ]=  }t'        j(                  d|j#                  d¬«      «      }|sŒ,|j+                  d«      }
 n |sŒÏ|	sŒÒ|j-                  ||	|
dœ«       Œè |S )zHFetch one list page from the JCMS build/unit API. Returns list of dicts.ÚbulidstaticÚcolumn)ÚwebIdÚpageIdÚ	parseTypeÚpageTypeÚtagIdÚtplSetIdr   Ú30)ÚpageNoÚpageSizeF)Úensure_asciiÚ	paramJsoné   )ÚparamsÚheadersÚtimeoutÚsuccessz  [API] page z not success: ÚmessageÚdataÚhtmlÚlxmlÚtrÚaÚhrefÚ ÚhttpÚtitleT©ÚstripÚtdz(\d{4}-\d{2}-\d{2}))Úurlr*   Úpub_date)ÚWEB_IDÚPAGE_IDÚTAG_IDÚ
TPL_SET_IDÚjsonÚdumpsÚrequestsÚgetÚAPIÚHEADERSÚraise_for_statusÚprintr   Úfind_allÚfindÚ
startswithÚBASEÚget_textr,   ÚreÚsearchÚgroupÚappend)Úpnr   ÚrÚdÚsoupÚitemsr%   r&   r'   r*   r/   r-   Úms                ú"/root/gov_crawler/crawl_shangyu.pyÚfetch_list_pagerL      s¡  € ô ÜØ"ØÜÜñ€Fð 
ˆA‚vÜ"Ÿj™j°BÀDÑ)IÐX]Ô^ˆˆ{ÑÜ�‰”S ´À"ÔE€AØ×ÑÔØ	�‰‹€AØ�5‰5�ÔÜ�˜b˜T °·±°iÓ0@Ð/AÐBÔCØˆ	Ü˜˜6™ 6Ñ*¨FÓ3€DØ€EØ�m‰m˜DÓ!ò NˆØ�G‰G�C‹LˆÙØØ�u‰u�V˜RÓ ˆÙØØ�‰˜vÔ&Ü˜$‘;ˆDà—‘�w“Ò9 1§:¡:°D :Ó#9×@Ñ@ÓBˆØˆØ—+‘+˜dÓ#ò 	ˆBÜ—	‘	Ð0°"·+±+ÀD°+Ó2IÓJˆAÚØŸ7™7 1›:�Ùð		ò
 ’EØ�L‰L °À8ÑLÕMð%Nð& €Ló    c                 óª  — 	 t        j                  | t        d¬«      }d|_        t	        |j
                  d«      }|j                  d«      }|r|j                  d¬«      nd}|j                  d	d
¬«      }|s|j                  d	d¬«      }|r6|j                  g d¢¬«      D ]  }|j                  «        Œ |t        |«      fS |dfS # t        $ r Y yw xY w)z5Fetch detail page: return (full_title, content_html).r   )r   r   zutf-8r$   Úh2Tr+   r(   ÚdivÚarticle)Úclass_Úcontent)ÚpositionÚwarpÚcrumb)r(   r(   )r6   r7   r9   Úencodingr   Útextr=   r@   r<   Ú	decomposeÚstrÚ	Exception)r.   rF   rH   rO   r*   ÚartÚnavs          rK   Úfetch_detailr^   I   sÉ   € ðÜ�L‰L˜¤g°rÔ:ˆØˆŒ
Ü˜QŸV™V VÓ,ˆØ�Y‰Y�t‹_ˆÙ+-�—‘ $�Ô'°2ˆØ�i‰i˜ iˆiÓ0ˆÙØ—)‘)˜E¨)�)Ó4ˆCÙà—|‘|Ò+H�|ÓIò  �Ø—‘•ð àœ#˜c›(�?Ð"Ø�bˆyÐøÜò Ùðús   ‚B?C ÃC Ã	CÃCc                  ó.  — t         } t        j                  dd  }t        |«      D ]\  \  }}|dk(  r#|dz   t	        |«      k  rt        ||dz      «      } Œ.|j                  d«      sŒ@t        |j                  d«      d   «      } Œ^ t        dt        › d| › dt        › �d¬	«       t        j                  t        d
¬«      }|j                  «       }g t        «       }}t        d| dz   «      D ]¥  }	 t!        |«      }	|	s n•d}|	D ]4  }|d   |v rŒ|j%                  |d   «       |j'                  |«       |dz  }Œ6 t        d|› d|› dt	        |«      › d�d¬	«       |	r!t)        d„ |	D «       «      rt        dd¬	«        nt+        j,                  d«       Œ§ t        dt	        |«      › �d¬	«       dx}x}}t        |«      D ]÷  \  }}|j/                  d|d   f«       |j1                  «       r|dz  }Œ2t3        |d   «      \  }}|s|d   }|s|dz  }t        d|d   dd  › �d¬	«       Œh|j/                  dt        |d   |d   ||d   ||f«       |j4                  r|dz  }t+        j,                  d«       |dz   dz  dk(  sŒÀ|j7                  «        t        d |dz   › d!t	        |«      › d"|› d#|› d$|› �
d¬	«       Œù |j7                  «        |j9                  «        t        d%|› d&|› d'|› d(�d¬	«       y # t"        $ r}
t        d|› d|
› �d¬	«       Y d }
~
 �Œd }
~
ww xY w))Nr   z--pagesz--pages=ú=z[shangyu] SITE=z pages=z db=T)Úflushé<   )r   z  [list] page z error: r   r.   z  Page z: +z links (total ú)c              3   ó�   K  — | ]>  }|d    r7|d    xr. t        j                  |d    d«      j                  «       t        k  –— Œ@ y­w)r/   z%Y-%m-%dN)r   Ústrptimer   ÚCUTOFF_DATE)Ú.0Úits     rK   ú	<genexpr>zmain.<locals>.<genexpr>   sJ   è ø€ ò 
à B z¢Nð ˆz‰NÒaœx×0Ñ0°°J±ÀÓL×QÑQÓSÔVaÑaÓañ
ùs   ‚AAz#  All items older than cutoff, stopg      à?z[shangyu] Total collected: z(SELECT 1 FROM gov_raw WHERE page_url = ?r*   z  [detail] empty content: iÄÿÿÿz¡INSERT OR IGNORE INTO gov_raw
               (site_name, source_url, page_url, title, publish_date, content, summary)
               VALUES (?, ?, ?, ?, ?, ?, ?)r/   g333333Ó?é   z  [ú/z] +z dup=z empty=z[shangyu] Done. +z new, z dup, z empty-skip)ÚMAX_PAGES_DEFAULTÚsysÚargvÚ	enumerateÚlenÚintr>   Úsplitr;   Ú	SITE_NAMEÚDB_PATHÚsqlite3ÚconnectÚcursorÚsetÚrangerL   r[   ÚaddrD   ÚallÚtimeÚsleepÚexecuteÚfetchoner^   ÚrowcountÚcommitÚclose)Ú	max_pagesÚargsÚir&   ÚconnÚcrI   ÚseenrE   Ú
page_itemsÚeÚnewrh   ÚinsertedÚskippedÚemptyr*   rS   s                     rK   Úmainr�   ^   s@  € Ü!€IÜ�8‰8�A�Bˆ<€DÜ˜$“ò -‰ˆˆ1Ø�	Š>˜a !™e¤c¨$£iÒ/Ü˜D  Q¡™KÓ(‰IØ�\‰\˜*Õ%Ü˜AŸG™G C›L¨™OÓ,‰Ið	-ô 
ˆOœI˜; g¨i¨[¸¼W¸IÐ
FÈdÕSä�?‰?œ7¨BÔ/€DØ�‰‹€Að ”c“eˆ4€EÜ�A�y 1‘}Ó%ò ˆð	Ü(¨Ó,ˆJñ ÙØˆØò 	ˆBØ�%‰y˜DÑ ØØ�H‰H�R˜‘YÔØ�L‰L˜ÔØ�1‰H‰Cð	ô 	�˜�t˜3˜s˜e >´#°e³*°¸QÐ?ÀtÕLáœ#ñ 
à ô
ô 
ô Ð7¸tÕDÙÜ�
‰
�3�ð/ô0 
Ð'¬¨E«
 |Ð
4¸DÕAð "#Ð"€HÐ"ˆw˜Ü˜5Ó!ò a‰ˆˆ2Ø	�	‰	Ð<¸rÀ%¹y¸lÔKØ�:‰:Œ<Ø�q‰LˆGØÜ% b¨¡iÓ0‰ˆˆwÙØ�w‘KˆEÙØ�Q‰JˆEÜÐ.¨r°%©y¸¸¨Ð.?Ð@ÈÕMØØ	�	‰	ð/ô ˜˜5™	 2 e¡9¨e°R¸
±^ÀWÈeÐTô		
ð �:Š:Ø˜‰MˆHÜ�
‰
�3ŒØ�‰E�R‰<˜1ÓØ�K‰KŒMÜ�C˜˜!™�u˜Aœc %›j˜\¨¨X¨J°e¸G¸9ÀGÈEÈ7ÐSÐ[_Ö`ð/að0 	‡K�K„MØ‡J�J„LÜ	Ð˜h˜Z v¨g¨Y°f¸U¸GÀ;Ð
OÐW[Ö\øôg ò 	Ü�N 2 $ h¨q¨cÐ2¸$Õ?Þûð	ús   Ã,K,Ë,	LË5LÌLÚ__main__)Ú__doc__Úosrm   r4   rA   r|   ru   r6   r   r   Úbs4r   Úgetenvrt   rs   r8   r?   r0   r1   r2   r3   rf   rl   r9   rL   r^   r�   Ú__name__© rM   rK   ú<module>r—      s¨   ðñ÷ 2× 1× 1Ó 1ß #Ý à
ˆ"�)‰)�KÐ!2Ó
3€Ø0€	ØY€Ø#€Ø	€Ø
€Ø	€Ø$€
Ù�4˜˜AÓ€ØÐ ð DØ3°G°9¸OÐLñ€ò'òTò*F]ðR ˆzÒÙ…Fð rM   