Ë
    ¸¬ujF#  ã                   óH  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
m
Z
mZ ddlmZmZ ej                  j!                  dej"                  j%                  d«      «      Zd	Zd
ZdddddœdddddœgZddiZdZdd„Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Ze dk(  r e«        yy)u³  
crawl_gaoan.py â€” é«˜å®‰å¸‚äººæ°‘æ”¿åºœÂ·çŽ¯å¢ƒä¿�æŠ¤å¤‡æ¡ˆ + è¡Œæ”¿è®¸å�¯
============================================================
CMS: æ”¿åºœä¿¡æ�¯å…¬å¼€å¹³å�°ï¼Œåˆ—è¡¨ä¸ºå�•æ–‡ä»¶ JSONï¼ˆæ— åˆ†é¡µï¼‰

åˆ—è¡¨ï¼š{codeName}-articleList.json  â†’  æ¯�æ¬¡è¿”å›žå…¨éƒ¨æ–‡ç« 
è¯¦æƒ…ï¼š<div class="article-content"> æ­£æ–‡ | <meta name="ArticleTitle"> /
       <meta name="PubDate"> / <meta name="ContentSource">

ä¸¤ä¸ªæ �ç›®ï¼š
  1. hjbha  â€” çŽ¯å¢ƒä¿�æŠ¤å¤‡æ¡ˆï¼ˆ50æ�¡ï¼Œ2024-08 ~ 2026-05ï¼‰
  2. xzxkgh â€” è¡Œæ”¿è®¸å�¯      ï¼ˆ50æ�¡ï¼Œ2026-03 ~ 2026-06ï¼‰

ç”¨æ³•:
    python3 crawl_gaoan.py               # å…¨é‡�
    python3 crawl_gaoan.py 1             # å¢žé‡�ï¼ˆä»…æœ€æ–°5æ�¡ï¼‰
é    N)ÚBeautifulSoup)Úurljoin)ÚdatetimeÚ	timedelta)ÚThreadPoolExecutorÚas_completedÚGOV_DB_PATHz/mnt/data/search.dbzhttp://www.gaoan.gov.cnu   é«˜å®‰å¸‚äººæ°‘æ”¿åºœÚhjbhau   çŽ¯å¢ƒä¿�æŠ¤å¤‡æ¡ˆz%/gasrmzf/hjbha/hjbha-articleList.jsoné2   )ÚcodeÚnameÚlist_urlÚmax_articlesÚxzxkghu   è¡Œæ”¿è®¸å�¯z'/gasrmzf/xzxkgh/xzxkgh-articleList.jsonz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36é   c                 ó&   — t        d|› d| › �«       y )Nú[z] )Úprint)ÚmsgÚprefixs     ú /root/gov_crawler/crawl_gaoan.pyÚlogr   @   s   € Ü	ˆAˆfˆX�R˜�uÐ
Õó    c                 ó”  — t        t        | d   «      }	 t        j                  |t        d¬«      }d|_        |j                  «       }g }|D ]®  }|j                  d	d
«      xs |j                  dd
«      }|j                  dd
«      }|j                  dd«      }	t        |	t        «      r	 t        j                  |	«      }
n|	}
|
j                  dd
«      }|sŒ‹t        t        |«      }|j                  |||f«       Œ° |S # t        $ r$}t        d| d   › d|› �| d   «       g cY d}~S d}~ww xY w# t        j                  $ r i }
Y Œ„w xY w)z/Fetch the full article list JSON for a channel.r   é   ©ÚheadersÚtimeoutúutf-8zERROR fetching list for r   ú: NÚtitleÚ Ú	showTitleÚpubDateÚurlsz{}Úpc)r   ÚBASE_URLÚrequestsÚgetÚHEADERSÚencodingÚjsonÚ	Exceptionr   Ú
isinstanceÚstrÚloadsÚJSONDecodeErrorÚappend)ÚchannelÚurlÚrespÚdataÚeÚitemsÚartr!   Úpub_dateÚurls_rawÚurls_objÚpc_urlÚfull_urls                r   Úfetch_article_listr?   D   sE  € ä
”(˜G JÑ/Ó
0€CðÜ�|‰|˜C¬¸"Ô=ˆØˆŒØ�y‰y‹{ˆð
 €EØò 2ˆØ—‘˜ Ó$Ò@¨¯©°¸RÓ(@ˆØ—7‘7˜9 bÓ)ˆØ—7‘7˜6 4Ó(ˆÜ�h¤Ô$ðÜŸ:™: hÓ/‘ð  ˆHØ—‘˜d BÓ'ˆÙØÜœ8 VÓ,ˆØ�‰�h  xÐ0Õ1ð2ð" €Løô- ò ÜÐ& w¨v¡Ð&7°r¸!¸Ð=¸wÀv¹ÔOØ�	ûðûô ×'Ñ'ò Ø’ðús/   •3C? Â*D/Ã?	D,ÄD'Ä!D,Ä'D,Ä/EÅEc                 óØ  — 	 t        j                  | t        d¬«      }d|_        |j                  }t        |d«      }|j                  d	d
di¬«      }|r$|j                  d«      r|d   j                  «       nd}|s(|j                  dd„ ¬«      }|r|j                  d¬«      }d}|j                  d	d
di¬«      }	|	rK|	j                  d«      r:t        j                  d|	d   j                  «       «      }
|
r|
j                  d«      }|j                  dd„ ¬«      }|rt        |«      nd}|st        d| › �«       d||fS |||fS # t
        $ r}t        d| › d|› �«       Y d}~yd}~ww xY w)z=Return (content_html, date_str, title) or (None, None, None).r   r   r   z  ERROR r    N©NNNzhtml.parserÚmetar   ÚArticleTitle)ÚattrsÚcontentr"   Údivc                 óT   — | xr% dt        | t        «      r| ndj                  | «      v S )Nzarticle-titleú ©r.   r/   Újoin©Úcs    r   ú<lambda>zfetch_detail.<locals>.<lambda>v   s*   € ¨qÒ/r°_ÌjÐYZÔ\_ÔN`ÉÐfi×fnÑfnÐopÓfqÐ5r€ r   )Úclass_T)ÚstripÚPubDatez(\d{4}-\d{2}-\d{2})é   c                 óT   — | xr% dt        | t        «      r| ndj                  | «      v S )Nzarticle-contentrH   rI   rK   s    r   rM   zfetch_detail.<locals>.<lambda>ƒ   s-   € °AÒ4yÐ:KÔU_Ð`aÔcfÔUgÑPQÐmp×muÑmuÐvwÓmxÐ:y€ r   z  WARNING: no content for )r(   r)   r*   r+   Útextr-   r   r   ÚfindrO   Úget_textÚreÚmatchÚgroupr/   )r4   r5   Úhtmlr7   ÚsoupÚ
meta_titler!   ÚatÚdate_strÚ	meta_dateÚmÚcontent_divÚcontent_htmls                r   Úfetch_detailrb   d   st  € ð Ü�|‰|˜C¬¸"Ô=ˆØˆŒØ�y‰yˆô
 ˜˜}Ó-€Dð —‘˜6¨&°.Ð)A�ÓB€JÙ-7¸J¿N¹NÈ9Ô<UˆJ�yÑ!×'Ñ'Ô)Ð[]€Eñ Ø�Y‰Y�uÑ%rˆYÓsˆÙØ—K‘K d�KÓ+ˆEð €HØ—	‘	˜&¨°Ð(;�	Ó<€IÙ�Y—]‘] 9Ô-Ü�H‰HÐ+¨Y°yÑ-A×-GÑ-GÓ-IÓJˆÙØ—w‘w˜q“zˆHð —)‘)˜EÑ*y�)Óz€KÙ'2”3�{Ô#¸€LáÜÐ(¨¨Ð.Ô/Ø�X˜uÐ$Ð$à˜ 5Ð(Ð(øôA ò  Üˆh�s�e˜2˜a˜SÐ!Ô"Üûð ús   ‚/E Å	E)ÅE$Å$E)c                 ó2  — i }t        t        ¬«      5 }| D �ci c]  }|j                  t        |«      |“Œ }}t	        |«      D ]  }||   }	 |j                  «       ||<   Œ 	 d d d «       |S c c}w # t        $ r}d||<   Y d }~ŒAd }~ww xY w# 1 sw Y   |S xY w)N)Úmax_workersrA   )r   ÚMAX_WORKERSÚsubmitrb   r   Úresultr-   )r%   ÚresultsÚexr4   Úfut_mapÚfutr7   s          r   Úfetch_detail_batchrl   �   s¤   € Ø€GÜ	¬Ô	4ð 2¸Ø@DÖE¸�2—9‘9œ\¨3Ó/°Ñ4ÐEˆÐEÜ Ó(ò 	2ˆCØ˜#‘,ˆCð2Ø"Ÿz™z›|�˜’ñ	2÷2ð €Nùò Føô
 ò 2Ø1�˜–ûð2ú÷2ð €NúsE   “B˜A,¶BÁA1ÁBÁ,BÁ1	B	Á:BÁ?BÂB	Â	BÂBc           
      óN  — ddl }| sy|j                  t        «      }|j                  «       }d}| D ]*  \  }}}}}		 |j	                  dt
        |||||	f«       |dz  }Œ, |j                  «        |j                  «        |S # t        $ r}
t        d|› d|
› �«       Y d}
~
Œnd}
~
ww xY w)z=items: list of (url, title, date_str, content_html, category)r   NzœINSERT OR REPLACE INTO gov_raw
                   (site_name, page_url, title, publish_date, content, category)
                   VALUES (?, ?, ?, ?, ?, ?)rQ   z  DB error for r    )
Úsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚ	SITE_NAMEr-   r   ÚcommitÚclose)r8   rn   ÚconnrL   Úinsertedr4   r!   r]   ra   Úcategoryr7   s              r   Ú
save_to_dbry   š   s¹   € ãÙØØ�?‰?œ7Ó#€DØ�‰‹€AØ€HØ8=ò 
.Ñ4ˆˆU�H˜l¨Hð		.Ø�I‰Ið0ô ˜C ¨°,ÀÐIô	ð ˜‰M‰Hð
.ð 	‡K�K„MØ‡J�J„LØ€Oøô	 ò 	.Ü�/ #  b¨¨Ð,×-Ñ-ûð	.ús   »!B Â 	B$Â	BÂB$c                 óÊ  — | d   }t        d| d   › d| d   › d�|«       t        | «      }t        dt        |«      › d�|«       |st        d|«       g S t        j                  «       t        d	¬
«      z
  j                  d«      }t        d|› �|«       g }|D ]B  \  }}}|r|dd nd}	|	r|	|k\  r|j                  |||	f«       Œ,|	rŒ/|j                  |||	f«       ŒD |r|dd }t        dt        |«      › d�|«       nt        dt        |«      › d�|«       g }
t        dz  }t        dt        |«      |«      D ]¤  }||||z    }|D ��cg c]  \  }}}|‘Œ
 }}}t        d|dz   › d|t        |«      z   › dt        |«      › d�|«       t        |«      }|D ]G  \  }}}|j                  |d«      \  }}}|xs |}|xs |}| d   }|
j                  ||||xs d|f«       ŒI Œ¦ |
S c c}}w )z)Process one channel, return results list.r   z--- r   z (z) ---zList contains z articles totalzNo articles foundiG  )Údaysz%Y-%m-%dz3-year cutoff: Né
   r"   é   zIncremental: z	 articleszAfter date filter: é   r   z  Details [rQ   ú-ú/z]...rA   )r   r?   Úlenr   Únowr   Ústrftimer2   re   Úrangerl   r)   )r3   Úincrementalr   ÚarticlesÚthree_years_agoÚfilteredr4   r!   r:   Údrh   Ú
batch_sizeÚbatch_startÚbatchÚuÚ_Ú
batch_urlsÚ
detail_mapr]   ra   Údetail_dateÚdetail_titleÚfinal_titleÚ
final_daterx   s                            r   Úrun_channelr•   ²   s:  € à�V‰_€FÜˆ$ˆw�v‰Ð˜r '¨&¡/Ð!2°%Ð8¸&ÔAä! 'Ó*€HÜˆ.œ˜X›˜ Ð7¸Ô@áÜÐ Ô(Øˆ	ä—|‘|“~¬	°wÔ(?Ñ?×IÑIÈ*ÓU€OÜˆ/˜/Ð*Ð+¨VÔ4ð €HØ (ò -ÑˆˆU�HÙ%ˆH�S�b‰M¨2ˆÙ��oÒ%Ø�O‰O˜S %¨˜OÕ,Úà�O‰O˜S %¨˜OÕ,ð-ñ à˜B˜Q�<ˆÜˆmœC ›M˜?¨)Ð4°fÕ=äÐ!¤# h£- °	Ð:¸FÔCð €GÜ˜q‘€JÜ˜Q¤ H£¨zÓ:ò 
YˆØ˜ [°:Ñ%=Ð>ˆØ',×-™G˜A˜q !’aÐ-ˆ
Ñ-Üˆk˜+ a™-˜¨¨+´c¸%³jÑ*@Ð)AÀÄ3ÀxÃ=À/ÐQUÐVÐX^Ô_Ü'¨
Ó3ˆ
Ø$)ò 	YÑ ˆC�˜Ø6@·n±nÀSÐJ\Ó6]Ñ3ˆL˜+ |Ø&Ò/¨%ˆKØ$Ò0¨ˆJØ˜v‘ˆHØ�N‰N˜C ¨j¸,Ò:LÈ"ÈhÐWÕXñ	Yð
Yð €Nùó .s   ÅGc            	      óD  — t        t        j                  «      dkD  xr t        j                  d   dk(  } g }t        D ]  }t	        || «      }|j                  |«       Œ! t        |«      }t        dd› �«       t        dt        |«      › d|› d�«       t        D ]o  }|d   }|D �cg c]  }|d	   |k(  sŒ|‘Œ }}t        |D �cg c]  }|d
   sŒ	|d
   ‘Œ c}«      }	|	r|	d   › d|	d   › �nd}
t        d|› dt        |«      › d|
› �«       Œq 	 dd l	}|j                  t        «      }t        D ]  }|d   }|j                  d|f«       Œ |j                  «        |j                  «        t        d«       y c c}w c c}w # t        $ r}t        d|› �«       Y d }~y d }~ww xY w)NrQ   Ú1ú
z2==================================================zDone. Total: z articles (saved: ú)r   é   é   r   z ~ éÿÿÿÿzN/Az  r    z articles, a&  
                INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
                SELECT rowid, title, site_name,
                       CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
                FROM gov_raw WHERE category = ?
            zFTS index rebuiltzFTS rebuild note: )r�   ÚsysÚargvÚCHANNELSr•   Úextendry   r   Úsortedrn   ro   rp   rr   rt   ru   r-   )r…   Úall_resultsr3   rh   ÚsavedÚchÚcatÚrÚ
ch_resultsÚdatesÚ
date_rangern   rv   r7   s                 r   Úmainrª   ä   s°  € Ü”c—h‘h“- !Ñ#Ò:¬¯©°©°sÑ(:€Kà€KÜò $ˆÜ˜g {Ó3ˆØ×Ñ˜7Õ#ð$ô
 �{Ó#€Eäˆ"ˆXˆJˆÔÜˆ-œ˜KÓ(Ð)Ð);¸E¸7À!ÐDÔEô ò BˆØ�‰jˆØ!,Ö<˜A°°!±¸³’aÐ<ˆ
Ð<Ü jÖ9 °A°a³D˜˜!›Ò9Ó:ˆÙ49˜˜a™�z  U¨2¡Y KÑ0¸uˆ
Üˆb���Rœ˜J›Ð(¨°J°<Ð@ÕAðBð&ÛØ�‰œwÓ'ˆÜò 	ˆBØ�V‘*ˆCØ�L‰Lð ð
 �õð	ð 	�‰ŒØ�
‰
ŒÜÐÕ ùò' =ùÚ9øô& ò &ÜÐ   Ð$×%Ñ%ûð&ús1   Â%E4Â3E4Ã
E9
ÃE9
ÄA'E> Å>	FÆFÆFÚ__main__)Úgaoan)!Ú__doc__ÚosrV   r�   r,   r(   Úbs4r   Úurllib.parser   r   r   Úconcurrent.futuresr   r   Úenvironr)   ÚpathÚ
expanduserrp   r'   rs   rŸ   r*   re   r   r?   rb   rl   ry   r•   rª   Ú__name__© r   r   ú<module>r·      sÝ   ðñó$ 
Û 	Û 
Û Û Ý Ý  ß (ß ?ð �*‰*�.‰.˜¨¯©×(:Ñ(:Øó)ó €ð
 %€Ø#€	ð Ø$Ø;Øñ	ð ØØ=Øñ	ð€ð  ð 3ð€ð €óòò@&)òR
òò0/òd&&ðR ˆzÒÙ…Fð r   