Ë
    €Ûµjs  ã                   óB  — d Z ddlZddlZddlZddlZddlZddlZddlmZmZ dZ	dZ
dZdZdZ ej                  «        ed	¬
«      z
  j                  d«      ZddddœZddiZ ej&                  ej(                  j*                  «       d„ Zd„ Zd„ Zd„ Zedk(  r e«        yy)ud   Crawler for www.fjql.gov.cn - å…¬å‘Šå…¬ç¤º
TRS WCM API for list + detail page for full HTML content
é    N)ÚdatetimeÚ	timedeltaz$https://www.fjql.gov.cn/fjdzapp/dataz/root/search.dbé   é   u   www.fjql.gov.cn-å…¬å‘Šå…¬ç¤ºiG  )Údaysú%Y-%m-%dzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36z!application/json, text/plain, */*zapplication/json)ú
User-AgentÚAcceptzContent-Typer	   c                 ó*  — dddt         | dœ}t        j                  t        |t        dd¬«      }|j                  «       }|j                  d«      r!t        d	|j                  d
«      › �«       g dfS |j                  dg «      |j                  dd«      fS )z,Fetch one page from TRS WCM API (POST JSON).i † zchnlid=6272z-docorderpri,-docreltime)Ú	channelidÚclasssqlÚ	sortfieldÚprepageÚpageFé   )ÚjsonÚheadersÚverifyÚtimeoutÚerrorz  [ERR] API error: Úmsgr   ÚdataÚrecordCount)ÚITEMS_PER_PAGEÚrequestsÚpostÚAPI_URLÚHEADERSr   ÚgetÚprint)r   ÚpayloadÚrespr   s       ú/root/gov_crawler/crawl_fjql.pyÚ
fetch_pager$       s�   € ð Ø!Ø/Ü!Øñ€Gô �=‰=œ w¼ÈÐWYÔZ€DØ�9‰9‹;€DØ‡x�x�ÔÜÐ# D§H¡H¨U£OÐ#4Ð5Ô6Ø�1ˆuˆØ�8‰8�F˜BÓ §¡¨-¸Ó!;Ð;Ð;ó    c                 óV  — 	 t        j                  | t        dd¬«      }d|_        |j                  }ddlm}  ||d«      }|j                  dd	¬
«      }|rt        |«      S |j                  d«      D ]6  }|j                  dg «      }|sŒt        d„ |D «       «      sŒ+t        |«      c S  |j                  dd¬
«      }|r@|j                  dd	¬
«      xs |j                  dd¬«      }	|	rt        |	«      S t        |«      S y# t        $ r}
t        d| dd › d|
› �«       Y d}
~
yd}
~
ww xY w)zNFetch detail page and extract content from div.TRS_Editor using BeautifulSoup.Fr   )r   r   r   zutf-8r   )ÚBeautifulSoupzhtml.parserÚdivÚ
TRS_Editor)Úclass_Úclassc              3   ó$   K  — | ]  }d |v –— Œ
 y­w)r)   N© )Ú.0Úcs     r#   ú	<genexpr>z'fetch_detail_content.<locals>.<genexpr>C   s   è ø€ Ò:°˜<¨1Ô,Ñ:ùs   ‚zsmgb-article)Ú	recursiveÚ z  [ERR] detail fetch: Né<   ú: )r   r   ÚDETAIL_HEADERSÚencodingÚtextÚbs4r'   ÚfindÚstrÚfind_allÚanyÚ	Exceptionr    )Úurlr"   Úhtmlr'   ÚsoupÚeditorr(   ÚclsÚarticleÚinnerÚes              r#   Úfetch_detail_contentrF   1   s#  € ðÜ�|‰|˜C¬ÀÈrÔRˆØˆŒØ�y‰yˆå%Ù˜T =Ó1ˆð —‘˜5¨�Ó6ˆÙÜ�v“;Ðð —=‘= Ó'ò 	 ˆCØ—'‘'˜' 2Ó&ˆCÚ”sÑ:°cÔ:Õ:Ü˜3“x’ð	 ð —)‘)˜E¨.�)Ó9ˆÙà—L‘L ¨|�LÓ<ÒdÀÇÁÈUÐ^cÀÓ@dˆEÙÜ˜5“zÐ!Ü�w“<ÐàøÜò ÜÐ& s¨3¨B x j°°1°#Ð6Ô7Üûðús7   ‚AD Á!(D Â
D ÂD Â*A
D Ã5
D Ä	D(Ä
D#Ä#D(c                 ó¦   — 	 t        j                  | j                  «       d«      }|j                  d«      S #  | d d j	                  dd«      cY S xY w)Nz%Y.%m.%d %H:%M:%Sr   é
   ú.ú-)r   ÚstrptimeÚstripÚstrftimeÚreplace)ÚdatestrÚdts     r#   Úconvert_daterQ   U   sO   € ð.Ü×Ñ˜wŸ}™}›Ð0CÓDˆØ�{‰{˜:Ó&Ð&øð.Ø�s˜ˆ|×#Ñ# C¨Ó-Ò-ús	   ‚47 ·Ac                  ó€  — g } d}t        dt        › dt        › d�«       t        dt        dz   «      D ]X  }	 t	        |«      \  }}|s|r|}t        d|› dt        |«      › d�«       | j                  |«       t        j                  d
«       ŒZ t        «       }g }| D ]T  }|j                  d«      xs |j                  d«      xs d}	|	sŒ.|	|vsŒ3|j                  |	«       |j                  |«       ŒV t        dt        › dt        |«      › �«       g }
|D ]N  }|j                  d«      xs dj                  «       }|sŒ)t        |«      }|t         k\  sŒ>|
j                  |«       ŒP t        dt        › dt        |
«      › �«       t#        j$                  t&        d¬«      }|j)                  d«       |j+                  «       }|j-                  «        t        dt        › d�«       d}d}t/        |
«      D �]C  \  }}|j                  d«      xs |j                  d«      xs d}|sŒ2|j                  d«      xs dj                  «       }t        |j                  dd«      «      }t        d|dz   › dt        |
«      › d|d d › d�«       t1        |«      }|rt        |«      dk  r+t        d«       t        j                  d«       t1        |«      }|rt        |«      dk  rt        d«       |dz  }Œû|j)                  d t        |||||d d! f«       |j-                  «        |dz  }t        j                  d"«       �ŒF t        dt        › d#|› d$|› d%�«       |j)                  d&t        f«       |j3                  «       d   }t        dt        › d'|› �«       |dkD  rD|j)                  d(t        f«       |j5                  «       D ]  \  }}t        d)|d d › d|› d*�«       Œ |j7                  «        t        |||d+œ}t        d,t9        j:                  |d-¬.«      › �«       y # t        $ r}t        d	|› d|› �«       Y d }~�ŒÂd }~ww xY w)/Nr   ú[z] Fetching z pages from API...é   z  Page r4   z itemsz  [ERR] Page g      à?Ú	docpuburlr>   r2   z
[z] Total unique: Ú
docreltimez] In 3 years: é   )r   zPRAGMA busy_timeout=60000z] Cleaned old recordsÚdoctitlez  [ú/z] Fetching detail: é(   z...r   z0    [WARN] Empty/short content, retrying once...é   z!    [SKIP] No content after retryztINSERT OR IGNORE INTO gov_raw (site_name, title, content, page_url, publish_date, summary) VALUES (?, ?, ?, ?, ?, ?)éÈ   gš™™™™™é?z] Done: z new, z errorsz.SELECT COUNT(*) FROM gov_raw WHERE site_name=?z] Total in DB: zDSELECT title, length(content) FROM gov_raw WHERE site_name=? LIMIT 3z  z chars)ÚsiteÚnewÚerrorsÚtotalz
JSON_OUTPUT:F)Úensure_ascii)r    Ú	SITE_NAMEÚ	MAX_PAGESÚranger$   ÚlenÚextendr=   ÚtimeÚsleepÚsetr   ÚaddÚappendrL   rQ   ÚCUTOFF_DATEÚsqlite3ÚconnectÚDB_PATHÚexecuteÚcursorÚcommitÚ	enumeraterF   ÚfetchoneÚfetchallÚcloser   Údumps)Ú	all_itemsÚtotal_recordsr   Úitemsr`   rE   ÚseenÚunique_itemsÚitemr>   ÚfilteredÚreltimeÚdÚconnr/   Ú	new_countÚerror_countÚiÚpage_urlÚtitleÚ	date_partÚcontent_htmlÚtÚlÚsummarys                            r#   ÚmainrŒ   ]   s  € Ø€IØ€Mä	ˆAŒiˆ[˜¤I ;Ð.@Ð
AÔBÜ�aœ Q™Ó'ò 	ˆð	/Ü% dÓ+‰LˆE�5Ù ¡UØ %�Ü�G˜D˜6 ¤C¨£J <¨vÐ6Ô7Ø×Ñ˜UÔ#ô 	�
‰
�3�ð	ô ‹5€DØ€LØò &ˆØ�h‰h�{Ó#Ò< t§x¡x°£Ò<¸"ˆÚ�3˜d’?Ø�H‰H�SŒMØ×Ñ Õ%ð	&ô 
ˆC”	ˆ{Ð*¬3¨|Ó+<Ð*=Ð
>Ô?ð €HØò &ˆØ—8‘8˜LÓ)Ò/¨R×6Ñ6Ó8ˆÚÜ˜WÓ%ˆAØ”KÓØ—‘ Õ%ð&ô 
ˆAŒiˆ[˜¤s¨8£} oÐ
6Ô7ô �?‰?œ7¨BÔ/€DØ‡L�LÐ,Ô-Ø�‰‹€Að
 	‡K�K„MÜ	ˆAŒiˆ[Ð-Ð
.Ô/ð €IØ€Kä˜XÓ&ó ‰ˆˆ4Ø—8‘8˜KÓ(ÒA¨D¯H©H°U«OÒA¸rˆÙØà—‘˜*Ó%Ò+¨×2Ñ2Ó4ˆÜ  §¡¨,¸Ó!;Ó<ˆ	ô 	��A�a‘C�5˜œ#˜h›-˜Ð(;¸EÀ#À2¸J¸<ÀsÐKÔLÜ+¨HÓ5ˆáœs <Ó0°2Ò5ÜÐDÔFÜ�J‰J�qŒMÜ/°Ó9ˆLáœs <Ó0°2Ò5ÜÐ5Ô7Ø˜1ÑˆKØà	�	‰	ð CÜ˜˜|¨X°yÀ%ÈÈÀ+ÐNô	
ð 	�‰ŒØ�Q‰ˆ	Ü�
‰
�3Žð9ô< 
ˆC”	ˆ{˜( 9 +¨V°K°=ÀÐ
HÔIà‡I�IÐ>ÄÀÔMØ�J‰J‹L˜‰O€EÜ	ˆAŒiˆ[˜¨ wÐ
/Ô0ð ˆq‚yØ	�	‰	ÐXÔ[dÐZfÔgØ—J‘J“Lò 	,‰DˆAˆqÜ�B�q˜˜"�v�h˜b   6Ð*Õ+ð	,ð 	‡J�J„Lä ¨¸kÐTYÑZ€GÜ	ˆNœ4Ÿ:™: g¸EÔBÐCÐ
DÕEøôm ò 	/Ü�M $  r¨!¨Ð-×.Ò.ûð	/ús   ¶A PÐ	P=Ð!P8Ð8P=Ú__main__)Ú__doc__r   rm   r   rg   ÚreÚurllib3r   r   r   ro   rc   r   rb   ÚnowrM   rl   r   r5   Údisable_warningsÚ
exceptionsÚInsecureRequestWarningr$   rF   rQ   rŒ   Ú__name__r-   r%   r#   ú<module>r–      sË   ðñó Û Û Û Û 	Û ß (à
0€Ø
€Ø€	Ø€Ø*€	Øˆx�|‰|‹~¡	¨wÔ 7Ñ7×AÑAÀ*ÓM€ð DØ1Ø&ñ€ð ð  Dð€ð €× Ñ ˜×+Ñ+×BÑBÔ Cò<ò"!òH.òbFðJ ˆzÒÙ…Fð r%   