Ë
    m }j"  ã                   ó\  — d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
m
Z
mZ ddlmZmZ ej                  j!                  dej                  j!                  dd	«      «      Zd
ZdZdZdZdZe› d�ZdddddddœZddde› �dœZdZdZd„ Zd„ Zd„ Zd „ Zd!„ Z d"„ Z!e"d#k(  r e!«        yy)$um  
crawl_tx.py â€” æ¡�ä¹¡å¸‚äººæ°‘æ”¿åºœÂ·å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·ä¿¡æ�¯å…¬ç¤º
============================================================
CMS: JCMS (å¤§æ±‰ç‰ˆé€š)ï¼Œä¿¡æ�¯å…¬å¼€ API é©±åŠ¨ï¼Œåˆ†ç±»è¿‡æ»¤ code=I01-A01

åˆ—è¡¨ï¼šAPI /api-gateway/jpaas-publish-server/front/page/build/unit
      å¸¦ paramJson æ�œç´¢å�‚æ•°ï¼Œ15æ�¡/é¡µï¼Œå…± 3é¡µ / 37æ�¡ï¼ˆå·²çŸ¥ï¼‰
è¯¦æƒ…ï¼š<div class="column-zjnr"> æ­£æ–‡ | <meta name="ArticleTitle"> /
      <meta name="PubDate"> / <meta name="ContentSource">

ç”¨æ³•:
    python3 crawl_tx.py               # å…¨é‡�
    python3 crawl_tx.py 1             # å¢žé‡�ï¼ˆä»…æœ€æ–°é¡µï¼‰
é    N)ÚBeautifulSoup)Úurljoin)ÚdatetimeÚ	timedelta)ÚThreadPoolExecutorÚas_completedÚ	SEARCH_DBÚGOV_DB_PATHz/root/search.dbzhttps://www.tx.gov.cnu   æ¡�ä¹¡å¸‚äººæ°‘æ”¿åºœu*   å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·ä¿¡æ�¯å…¬ç¤ºzI01-A01u   é‡�ç‚¹é¢†åŸŸä¿¡æ�¯z7/api-gateway/jpaas-publish-server/front/page/build/unitÚbulidstaticÚ3159ÚPumzAFWTgBHhWx3260d5tÚcolumnu   ç»„é…�åˆ†ç±»listÚ
1229856913)Ú	parseTypeÚwebIdÚtplSetIdÚpageTypeÚtagIdÚpageIdzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36zapplication/jsonz:https://www.tx.gov.cn/col/col1229856913/index.html?number=)z
User-AgentÚAcceptÚRefereré   é   c                 ó    — t        d| › �«       y )Nz[tx] )Úprint)Úmsgs    ú/root/gov_crawler/crawl_tx.pyÚlogr   9   s   € Ü	ˆE�#�ˆ-Õó    c                 óÂ  — t        j                  t        dt        dœd¬«      }t        j                  | t        |dœd¬«      }t        t        «      }||d<   	 t        j                  t        t        |d¬«      }|j                  «       }|j                  d	i «      j                  d
d«      }g }d}	t        j                  d|«      }
|
rt        |
j!                  d«      «      }	t        j"                  d|«      D ]¸  }
|
j!                  d«      }|
j!                  d«      j%                  «       }||
j'                  «       |
j)                  «       dz    }t        j                  d|«      }|r|j!                  d«      nd}|j+                  d«      r|nt-        t.        |«      }|j1                  |||f«       Œº ||	fS # t        $ r}t        d| › d|› �«       g dfcY d}~S d}~ww xY w)zTFetch one page of article list from the API. Returns list of (url, title, date_str).Ú )ÚxxgkIdÚxxgkTypeÚ	classNameF)Úensure_ascii)ÚpageNoÚpageSizeÚsearchÚ	paramJsoné   )ÚheadersÚparamsÚtimeoutÚdataÚhtmlz  ERROR API page ú: r   Nzcount="(\d+)"é   zJ<a\s+class="fl"\s+href="([^"]+)"\s+target="_blank"\s+title="([^"]+)"[^>]*>é   iô  z+<span class="fr">(\d{4}-\d{2}-\d{2})</span>Úhttp)ÚjsonÚdumpsÚCOLUMN_CODEÚ
GROUP_NAMEÚ	PAGE_SIZEÚdictÚAPI_BASE_PARAMSÚrequestsÚgetÚAPI_URLÚHEADERSÚ	Exceptionr   Úrer(   ÚintÚgroupÚfinditerÚstripÚstartÚendÚ
startswithr   ÚBASE_URLÚappend)Úpage_noÚsearch_jsonÚ
param_jsonr,   Úrespr.   r/   ÚeÚitemsÚtotalÚmÚhrefÚtitleÚ
full_entryÚdate_mÚdate_strÚfull_urls                    r   Úfetch_articles_pagerX   =   s¾  € ä—*‘*ÜØÜñð ô	€Kô —‘ØÜØñð ô	€Jô ”/Ó"€FØ$€Fˆ;ÑðÜ�|‰|œG¬W¸VÈRÔPˆØ�y‰y‹{ˆØ�x‰x˜ Ó#×'Ñ'¨°Ó3ˆð
 €EØ€Eô 	�	‰	Ð" DÓ)€AÙÜ�A—G‘G˜A“J“ˆô �[‰[ØUØóò 2ˆð �w‰w�q‹zˆØ—‘˜“
× Ñ Ó"ˆð ˜!Ÿ'™'›) A§E¡E£G¨c¡MÐ2ˆ
Ü—‘ÐIÈ:ÓVˆÙ&,�6—<‘< ”?°"ˆàŸ?™?¨6Ô2‘4¼ÄÈ$Ó8OˆØ�‰�h  xÐ0Õ1ð2ð �%ˆ<Ðøô9 ò ÜÐ ˜y¨¨1¨#Ð.Ô/Ø�1ˆu�ûðús   ÁAF6 Æ6	GÆ?GÇGÇGc                 óÎ  — 	 t        j                  | t        d¬«      }d|_        |j                  }t        |d«      }|j                  d	d
di¬«      }|r$|j                  d«      r|d   j                  «       nd}d}|j                  d	d
di¬«      }|rK|j                  d«      r:t        j                  d|d   j                  «       «      }	|	r|	j                  d«      }|j                  dd„ ¬«      }
|
rt        |
«      nd}|s#|j                  dd„ ¬«      }
|
rt        |
«      nd}|st        d| › �«       d||fS |||fS # t
        $ r}t        d| › d|› �«       Y d}~yd}~ww xY w)z=Return (content_html, date_str, title) or (None, None, None).r*   )r+   r-   zutf-8z  ERROR r0   N©NNNzhtml.parserÚmetaÚnameÚArticleTitle)ÚattrsÚcontentr!   ÚPubDatez(\d{4}-\d{2}-\d{2})r1   Údivc                 óT   — | xr% dt        | t        «      r| ndj                  | «      v S )Nzcolumn-zjnrú ©Ú
isinstanceÚstrÚjoin©Úcs    r   ú<lambda>zfetch_detail.<locals>.<lambda>Š   s+   € °AÒ4u¸-ÔQ[Ð\]Ô_bÔQcÉAÐil×iqÑiqÐrsÓitÐ:u€ r   )Úclass_c                 óT   — | xr% dt        | t        «      r| ndj                  | «      v S )NÚ
bt_contentrc   rd   rh   s    r   rj   zfetch_detail.<locals>.<lambda>�   s+   € ¸Ò8x¸lÔT^Ð_`ÔbeÔTfÉqÐlo×ltÑltÐuvÓlwÐ>x€ r   z  WARNING: no content for )r;   r<   r>   ÚencodingÚtextr?   r   r   ÚfindrD   r@   ÚmatchrB   rf   )ÚurlrM   r/   rN   ÚsoupÚ
meta_titlerS   rV   Ú	meta_daterQ   Úcontent_divÚcontent_htmls               r   Úfetch_detailrx   q   sn  € ð Ü�|‰|˜C¬¸"Ô=ˆØˆŒØ�y‰yˆô
 ˜˜}Ó-€Dð —‘˜6¨&°.Ð)A�ÓB€JÙ-7¸J¿N¹NÈ9Ô<UˆJ�yÑ!×'Ñ'Ô)Ð[]€Eð €HØ—	‘	˜&¨°Ð(;�	Ó<€IÙ�Y—]‘] 9Ô-Ü�H‰HÐ+¨Y°yÑ-A×-GÑ-GÓ-IÓJˆÙØ—w‘w˜q“zˆHð —)‘)˜EÑ*u�)Óv€KÙ'2”3�{Ô#¸€Lñ Ø—i‘i Ñ.x�iÓyˆÙ+6”s˜;Ô'¸BˆáÜÐ(¨¨Ð.Ô/Ø�X˜uÐ$Ð$à˜ 5Ð(Ð(øô? ò  Üˆh�s�e˜2˜a˜SÐ!Ô"Üûð ús   ‚/E  Å 	E$Å	EÅE$c                 ó2  — i }t        t        ¬«      5 }| D �ci c]  }|j                  t        |«      |“Œ }}t	        |«      D ]  }||   }	 |j                  «       ||<   Œ 	 d d d «       |S c c}w # t        $ r}d||<   Y d }~ŒAd }~ww xY w# 1 sw Y   |S xY w)N)Úmax_workersrZ   )r   ÚMAX_WORKERSÚsubmitrx   r   Úresultr?   )ÚurlsÚresultsÚexrr   Úfut_mapÚfutrN   s          r   Úfetch_detail_batchrƒ   ™   s¤   € Ø€GÜ	¬Ô	4ð 2¸Ø@DÖE¸�2—9‘9œ\¨3Ó/°Ñ4ÐEˆÐEÜ Ó(ò 	2ˆCØ˜#‘,ˆCð2Ø"Ÿz™z›|�˜’ñ	2÷2ð €Nùò Føô
 ò 2Ø1�˜–ûð2ú÷2ð €NúsE   “B˜A,¶BÁA1ÁBÁ,BÁ1	B	Á:BÁ?BÂB	Â	BÂBc           
      óT  — ddl }| sy|j                  t        «      }|j                  «       }d}| D ]-  \  }}}}	 |j	                  dt
        ||||t        f«       |dz  }Œ/ |j                  «        |j                  «        |S # t        $ r}	t        d|› d|	› �«       Y d}	~	Œqd}	~	ww xY w)z3items: list of (url, title, date_str, content_html)r   NzœINSERT OR REPLACE INTO gov_raw
                   (site_name, page_url, title, publish_date, content, category)
                   VALUES (?, ?, ?, ?, ?, ?)r1   z  DB error for r0   )Úsqlite3ÚconnectÚDB_PATHÚcursorÚexecuteÚ	SITE_NAMEÚCOLUMN_NAMEr?   r   ÚcommitÚclose)
rO   r…   Úconnri   Úinsertedrr   rS   rV   rw   rN   s
             r   Ú
save_to_dbr�   ¦   s·   € ãÙØØ�?‰?œ7Ó#€DØ�‰‹€AØ€HØ.3ò 
.Ñ*ˆˆU�H˜lð		.Ø�I‰Ið0ô ˜C ¨°,ÄÐLô	ð ˜‰M‰Hð
.ð 	‡K�K„MØ‡J�J„LØ€Oøô	 ò 	.Ü�/ #  b¨¨Ð,×-Ñ-ûð	.ús   º%BÂ	B'ÂB"Â"B'c                  óB  — t        t        j                  «      dkD  xr t        j                  d   dk(  } t        j                  «       t        d¬«      z
  j                  d«      }t        d|› �«       g }d}d}	 t        |«      \  }}|dk(  r|}t        d|› �«       |s|dk(  rt        d	«       y n–|D ���	cg c]  \  }}}	|	sŒ
|	|k\  sŒ|||	f‘Œ }
}}}	|j                  |
«       | rt        d
t        |
«      › d�«       nG|
r|
|k7  rt        |«      t        |
«      k(  rn|
|k7  rt        d|› d�«       n|t        z  |k\  rn|dz  }ŒÍt        dt        |«      › �«       g }t        dz  }t        dt        |«      |«      D ]�  }||||z    }|D ��cg c]  \  }}}|‘Œ
 }}}t        d|dz   › d|t        |«      z   › dt        |«      › d�«       t        |«      }|D ]A  \  }}}|j                  |d«      \  }}}|xs |}|xs |}|j                  ||||xs df«       ŒC ŒŸ t!        |«      }t        dd› �«       t        dt        |«      › d|› d�«       |r9t#        |D �cg c]  }|d   sŒ	|d   ‘Œ c}«      }t        d|d   › d|d   › �«       	 dd l}|j'                  t(        «      }|j+                  dt,        f«       |j/                  «        |j1                  «        t        d «       y c c}	}}w c c}}w c c}w # t2        $ r}t        d!|› �«       Y d }~y d }~ww xY w)"Nr1   Ú1iG  )Údaysz%Y-%m-%dz3-year cutoff: r   zTotal expected: zNo articles foundzIncremental: page 1, z	 articleszPage z$: some items before cutoff, stoppingzTotal articles: é   z  Details [ú-ú/z]...rZ   r!   ú
z2==================================================zDone. Total: z articles (saved: ú)r2   zDate range: z ~ éÿÿÿÿa  
            INSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary)
            SELECT rowid, title, site_name,
                   CASE WHEN length(content) > 200 THEN substr(content, 1, 200) ELSE content END
            FROM gov_raw WHERE category = ?
        zFTS index rebuiltzFTS rebuild note: )ÚlenÚsysÚargvr   Únowr   Ústrftimer   rX   Úextendr8   r{   Úrangerƒ   r<   rI   r�   Úsortedr…   r†   r‡   r‰   r‹   rŒ   r�   r?   ) ÚincrementalÚthree_years_agoÚall_articlesÚtotal_expectedÚpagerO   rP   ÚuÚtÚdÚfilteredr   Ú
batch_sizeÚbatch_startÚbatchÚ_Ú
batch_urlsÚ
detail_maprr   rS   rV   rw   Údetail_dateÚdetail_titleÚfinal_titleÚ
final_dateÚsavedÚrÚdatesr…   rŽ   rN   s                                    r   Úmainr¸   ¾   s]  € Ü”c—h‘h“- !Ñ#Ò:¬¯©°©°sÑ(:€KÜ—|‘|“~¬	°wÔ(?Ñ?×IÑIÈ*ÓU€OÜˆ/˜/Ð*Ð+Ô,ð €LØ€Nà€DØ
Ü*¨4Ó0‰ˆˆuØ�1Š9Ø"ˆNÜÐ" >Ð"2Ð3Ô4áØ�qŠyÜÐ'Ô(ØØð .3×QÐQ¡' ! Q¨²a¸AÀÓ<P�Q˜˜1’IÐQˆÒQØ×Ñ˜HÔ%áÜÐ'¬¨H« °iÐ@ÔAØá˜ EÒ)¬c°%«j¼CÀ»MÒ.IØØ˜ÒÜ�%˜�vÐAÐBÔCØà”)Ñ˜~Ò-Øà�‰	ˆð; ô> Ð
œ3˜|Ó,Ð-Ð.Ô/ð €GÜ˜q‘€JÜ˜Q¤ LÓ 1°:Ó>ò 	OˆØ˜[¨°zÑ)AÐBˆØ',×-™G˜A˜q !’aÐ-ˆ
Ñ-Üˆk˜+ a™-˜¨¨+´c¸%³jÑ*@Ð)AÀÄ3À|ÓCTÐBUÐUYÐZÔ[Ü'¨
Ó3ˆ
Ø$)ò 	OÑ ˆC�˜Ø6@·n±nÀSÐJ\Ó6]Ñ3ˆL˜+ |Ø&Ò/¨%ˆKØ$Ò0¨ˆJØ�N‰N˜C ¨j¸,Ò:LÈ"ÐMÕNñ		Oð	Oô �wÓ€Eäˆ"ˆXˆJˆÔÜˆ-œ˜G›�~Ð%7¸°w¸aÐ@ÔAÙÜ gÖ6 °°1³˜˜!›Ò6Ó7ˆÜˆl˜5 ™8˜* C¨¨b©	 {Ð3Ô4ð&ÛØ�‰œwÓ'ˆØ�‰ð ô
 ˆ^ô	ð 	�‰ŒØ�
‰
ŒÜÐÕ ùôo Rùó2 .ùò 7øô  ò &ÜÐ   Ð$×%Ñ%ûð&ús=   Â7K+ÃK+Ã	K+ÆK2É
K8É)K8ÊAK= Ë=	LÌLÌLÚ__main__)#Ú__doc__Úosr@   r›   r4   r;   Úbs4r   Úurllib.parser   r   r   Úconcurrent.futuresr   r   Úenvironr<   r‡   rH   rŠ   r‹   r6   r7   r=   r:   r>   r{   r8   r   rX   rx   rƒ   r�   r¸   Ú__name__© r   r   ú<module>rÂ      sç   ðñó 
Û 	Û 
Û Û Ý Ý  ß (ß ?à
�*‰*�.‰.˜ b§j¡j§n¡n°]ÐDUÓ&VÓ
W€à"€Ø#€	Ø:€Ø€Ø!€
àˆJÐMÐ
N€àØØ'ØØØñ€ð3ð !ØKÈKÈ=ÐYñ€ð €Ø€	òò1òh%)òP
òò0P&ðf ˆzÒÙ…Fð r   