Ë
    M¢j�(  ã                   óÄ  — d Z ddlZddlZddlZddlZddlZddlZddlZ	ddl
Z	ddlZddlZddlZddlmZmZ ddl
mZ ej"                  e_        dZdZedz   Zedz   Zd	Zej0                  j3                  ej0                  j5                  ej0                  j7                  e«      «      d
d«      ZddiZdddddœZ ej@                  ejB                  «      jE                  «       dz
  Z#d,d„Z$d-d„Z%d„ Z&d„ Z'd„ Z(d„ Z)d-d„Z*d„ Z+e,dk(  rÜddl-Z- e-j\                  de› �¬«      Z/e/ja                  de1d ¬!«       e/ja                  d"d#d$¬%«       e/je                  «       Z3e3jh                  r e+«         ejj                  d«        e6d&«        e+«         ej
                  «       Z7 e*e3jp                  ¬'«      \  Z9Z: e6d( ej
                  «       e7z
  d)›d*�«        e6d+«        e+«        yy).uŒ  
crawl_huanpingbao.py â€” çŽ¯è¯„å®�çˆ¬è™«
=================================
http://www.huanpingbao.cn/jcb-portal/publicity/indexPublicity

APIï¼š
  åˆ—è¡¨: GET /jcb-rest/publicityList?limit=25&offset={offset}
  è¯¦æƒ…: åˆ—è¡¨è‡ªå¸¦ contentï¼ˆå®Œæ•´HTMLï¼‰ï¼Œæ— éœ€é¢�å¤–è¯·æ±‚
  æ€»æ•°: ~40427 æ�¡

è§„åˆ™ï¼š
  - å…¨é‡�çˆ¬å�–ï¼ˆAPI åˆ†é¡µï¼Œæ— é¡µæ•°é™�åˆ¶ï¼‰
  - æ­£æ–‡ä¿�ç•™ HTML æ ¼å¼�
  - ç»“æžœå†™ search.dbï¼ˆgov_raw è¡¨ï¼Œè§¦å�‘å™¨è‡ªåŠ¨å»º FTSï¼‰

ç”¨æ³•ï¼š
  python3 crawl_huanpingbao.py             # å…¨é‡�çˆ¬å�–
  python3 crawl_huanpingbao.py --limit=10  # æµ‹è¯•è·‘ 10 æ�¡
  python3 crawl_huanpingbao.py --stats     # çœ‹ search.db ç»Ÿè®¡
é    N)ÚdatetimeÚtimezone)Úurljoinu	   çŽ¯è¯„å®�zhttp://www.huanpingbao.cnz/jcb-rest/publicityListz./jcb-portal/publicity/publicity_detail?id={id}z/root/search.dbÚoutputzhuanpingbao.jsonlz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36u   çŽ¯è¯„å…¬ç¤ºu   éªŒæ”¶å…¬ç¤ºu   å…¶ä»–å…¬ç¤ºu   åº”æ€¥é¢„æ¡ˆå…¬ç¤º)r   é   é   é   i€š£c                 óÌ   — t         j                  j                  | t        ¬«      }t         j                  j	                  ||¬«      j                  «       j                  dd¬«      S )N)Úheaders©Útimeoutúutf-8Úreplace)Úerrors)ÚurllibÚrequestÚRequestÚHEADERSÚurlopenÚreadÚdecode)Úurlr   Úreqs      ú&/root/gov_crawler/crawl_huanpingbao.pyÚhttp_getr   0   sO   € Ü
�.‰.×
 Ñ
  ¬gÐ
 Ó
6€CÜ�>‰>×!Ñ! #¨wÐ!Ó7×<Ñ<Ó>×EÑEÀgÐV_ÐEÓ`Ð`ó    c                 óö   — |r't         j                  j                  |«      }| › d|› �}n| }	 t        |«      }|rt	        j
                  |«      S dS # t        $ r}t        d|dd › d|› �«       Y d}~yd}~ww xY w)u!   GET è¯·æ±‚ï¼Œè¿”å›ž JSON æˆ– Noneú?Nu     âš ï¸� è¯·æ±‚å¤±è´¥: éP   z -> )r   ÚparseÚ	urlencoder   ÚjsonÚloadsÚ	ExceptionÚprint)r   ÚparamsÚqsÚfull_urlÚrespÚes         r   Ú
fetch_pager+   5   s�   € áÜ�\‰\×#Ñ# FÓ+ˆØ�U˜!˜B˜4�=‰àˆðÜ˜Ó!ˆÙ#'Œt�z‰z˜$ÓÐ1¨TÐ1øÜò ÜÐ'¨°°"¨ °d¸1¸#Ð>Ô?Üûðús   ­!A ÁA Á	A8ÁA3Á3A8c                 óº   — | syt        j                  dt        | «      «      }|r7|j                  d«      › d|j                  d«      › d|j                  d«      › �S y)u)   è§£æž� YYYY-MM-DD HH:MM:SS â†’ YYYY-MM-DDÚ z(\d{4})-(\d{2})-(\d{2})r   ú-r   r	   )ÚreÚsearchÚstrÚgroup)Údate_strÚms     r   Ú
parse_dater5   D   sR   € áØÜ
�	‰	Ð,¬c°(«mÓ<€AÙØ—'‘'˜!“*�˜Q˜qŸw™w q›z˜l¨!¨A¯G©G°A«J¨<Ð8Ð8Ør   c                 ó˜  — | j                  d«      }| j                  d«      xs dj                  «       }| j                  d«      xs d}t        j                  |«      }t        j                  |¬«      }t        | j                  dd«      «      }| j                  d«      xs dj                  «       }t        j                  dd	|«      }t        j                  d
d	|«      j                  «       }|dd }	| j                  d«      xs d}
|
rS|
j                  «       rCt        j                  j                  |
«      }|
j                  d«      sd|
› �n|
}|d|› d|› d�z  }| j                  d«      }t        j                  |d«      }| j                  d«      xs dj                  «       }|}|r|d|› �z  }t        t        dz   ||||	|d||dœ
S )u4   å°† API è¿”å›žçš„å�•æ�¡æ•°æ�®è½¬ä¸º search.db æ ¼å¼�ÚidÚtitler-   Úcontent)r7   Ú
createTimeÚcreateusernamez<[^>]+>ú z\s+Ni,  ÚdocumentUrlÚhttpz"http://newfile.123jcb.com/reports/z
<p><a href="u   " target="_blank">ðŸ“Ž é™„ä»¶: z</a></p>ÚpublicityTypeu   å…¶ä»–ÚindustryNameú,z$/jcb-portal/publicity/indexPublicityÚactive)
Ú	site_nameÚ
source_urlÚpage_urlr8   Úpublish_dateÚsummaryr9   ÚstatusÚcategoryÚtags)ÚgetÚstripÚhtml_modÚunescapeÚDETAIL_URL_TPLÚformatr5   r/   Úsubr   r    ÚunquoteÚ
startswithÚPUBLICITY_TYPE_MAPÚ	SITE_NAMEÚBASE_URL)Úapi_itemÚitem_idr8   Úraw_contentr9   rE   Úpub_dateÚcreate_userÚtextrG   Údoc_urlÚdoc_nameÚfull_doc_urlÚptypeÚptype_labelÚindustryrJ   s                    r   Ú
build_itemrc   N   sÍ  € à�l‰l˜4Ó €GØ�\‰\˜'Ó"Ò( b×/Ñ/Ó1€EØ—,‘,˜yÓ)Ò/¨R€Kô ×Ñ Ó,€Gô ×$Ñ$¨Ð$Ó0€Hô ˜(Ÿ,™, |°RÓ8Ó9€Hð —<‘<Ð 0Ó1Ò7°R×>Ñ>Ó@€Kô �6‰6�*˜c 7Ó+€DÜ�6‰6�&˜#˜tÓ$×*Ñ*Ó,€DØ�4�Cˆj€Gð �l‰l˜=Ó)Ò/¨R€GÙ�7—=‘=”?ä—<‘<×'Ñ'¨Ó0ˆàMT×M_ÑM_Ð`fÔMgÐ;¸G¸9ÑEÐmtˆØ�^ L >Ð1PÐQYÐPZÐZbÐcÑcˆð �L‰L˜Ó)€EÜ$×(Ñ(¨°Ó9€KØ—‘˜^Ó,Ò2°×9Ñ9Ó;€Hð €DÙØ�!�H�:�Ñˆô ÜÐ!GÑGØØØ ØØØØØñð r   c                 ó  — t        j                  t         j                  j                  t        «      d¬«       t        t        dd¬«      5 }|j                  t        j                  | d¬«      dz   «       d	d	d	«       y	# 1 sw Y   y	xY w)
u   ä¿�å­˜åˆ°æœ¬åœ° JSONL å¤‡ä»½T)Úexist_okÚar   )ÚencodingF)Úensure_asciiú
N)	ÚosÚmakedirsÚpathÚdirnameÚLOCAL_OUTPUTÚopenÚwriter"   Údumps)ÚitemÚfs     r   Úsave_to_localrt   †   s\   € ä‡K�K”—‘—‘¤Ó-¸Õ=Ü	Œl˜C¨'Ô	2ð =°aØ	�‰”—
‘
˜4¨eÔ4°tÑ;Ô<÷=÷ =ñ =ús   Á*A>Á>Bc                 óê   — 	 | j                  d|d   |d   |d   |d   dd |d   |d	   dd |d
   |d   |d   |d   f
«       | j                  dkD  S # t        $ r}t        d|› �«       Y d}~yd}~ww xY w)u    å†™å…¥ search.db çš„ gov_raw è¡¨zç
            INSERT OR IGNORE INTO gov_raw
                (site_name, source_url, page_url, title, publish_date,
                 summary, content, status, category, tags)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        rC   rD   rE   r8   Néô  rF   rG   r9   rH   rI   rJ   r   u     âš ï¸� å†™å…¥å¤±è´¥: F)ÚexecuteÚtotal_changesr$   r%   )Úconnrr   r*   s      r   Úwrite_to_searchdbrz   �   s®   € ðØ�‰ð ð �ÑØ�ÑØ�ÑØ�‰M˜$˜3ÐØ�Ñ Ø�‰O˜D˜SÐ!Ø�‰OØ�‰NØ�ÑØ�‰Lð
ô	ð" ×!Ñ! AÑ%Ð%øÜò ÜÐ'¨ sÐ+Ô,Üûðús   ‚AA Á	A2ÁA-Á-A2c                 óž  — t        dt        › d�«       d}d}d}ddl}t        j                  dd¬«      }|j                  d«       d}d	}d}d
}	|	�r|dz  }| r|| k\  r�n||dœ}
t        t        |
«      }|st        d|› d�«       nß|j                  d«      xs |j                  d«      }|j                  dd«      }|sn¦|D ]{  }| r|| k\  r nrt        |«      }t        |«       	 |j                  d|d   |d   |d   |d   |d   |d   |d   |d   |d   |d   f
«       |j                  dkD  r|dz  }n|dz  }|dz  }Œ} ||z  }||k  }	t        d|› d|› d|› d |› d!�	«       |	r�Œd}d}	 t        j                  t        d¬«      }|j                  d"«       |j                  d#«      j                  «       }|D ]d  }|j                  d$|d   |d   |d%   |d&   xs d'dd( |d)   |d*   xs d'dd( |d+   |d,   |d-   |d.   f
«       |j                  dkD  r|dz  }Œ`|dz  }Œf |j                  «        |j                  «        t        d/|› d0|› d1�«       |j                  «        t        d3|› d4|› �«       t        d5«       t        d6t"        › d7�«       t        d8«       ||fS #  |dz  }Y �ŒpxY w# t         $ r}t        d2|› �«       Y d}~Œwd}~ww xY w)9u   å…¨é‡�çˆ¬å�–u   
ðŸ“¡ [u   ] APIçˆ¬å�– (limit=25/é¡µ)r   Nz:memory:é<   r   zû
        CREATE TABLE IF NOT EXISTS temp_items (
            site_name TEXT, source_url TEXT, page_url TEXT UNIQUE,
            title TEXT, publish_date TEXT, summary TEXT,
            content TEXT, status TEXT, category TEXT, tags TEXT
        )
    é   Tr   )ÚlimitÚoffsetu     âš ï¸� ç¬¬u   é¡µæ— å“�åº”ï¼Œç»ˆæ­¢ÚrowÚrowsÚtotalz€
                    INSERT OR IGNORE INTO temp_items
                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                rC   rD   rE   r8   rF   rG   r9   rH   rI   rJ   u     [ç¬¬u   é¡µ] ú/u
    æ�¡ | âœ…u    (æœ¬åœ°æš‚å­˜)zPRAGMA busy_timeout=60000zuSELECT site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags FROM temp_itemsz›INSERT OR IGNORE INTO gov_raw (site_name, source_url, page_url, title, publish_date, summary, content, status, category, tags) VALUES (?,?,?,?,?,?,?,?,?,?)r   r	   r-   rv   é   é   é   é   é   é	   u      ðŸ“¥ è‡ªåŠ¨å¯¼å…¥ search.db: +u    (è·³è¿‡ú)u     âš ï¸� è‡ªåŠ¨å¯¼å…¥å¤±è´¥: u   
  ðŸ“Š é‡‡é›†å®Œæˆ�: å…±u   æ�¡, æ–°å¢žu+   
  ðŸ“¤ å¯¼å…¥å‘½ä»¤ï¼ˆSSH å¯†é’¥è®¤è¯�ï¼‰:z
     scp 'z*' root@1.94.217.116:/tmp/huanpingbao.jsonlzP     ssh root@1.94.217.116 'python3 /tmp/import_jsonl.py /tmp/huanpingbao.jsonl')r%   rU   Ú
subprocessÚsqlite3Úconnectrw   r+   ÚAPI_URLrK   rc   rt   rx   ÚSERVER_SEARCH_DBÚfetchallÚcommitÚcloser$   rn   )Ú	max_itemsÚtotal_fetchedÚokÚskipr‹   ry   r   r~   ÚpageÚhas_morer&   Údatar�   r‚   rW   rr   Ú_autoÚ
_auto_skipÚ_dbcÚ_rowsÚ_rÚ_es                         r   Úcrawlr    §   s†  € ä	ˆH”Y�KÐ9Ð
:Ô;ð €MØ	
€BØ€Dó ô �?‰?˜:¨rÔ2€DØ‡L�Lð ô 	ð €FØ€EØ€DØ€Hâ
Ø�‰	ˆÙ˜¨)Ò3Ùà ¨FÑ3ˆÜœ' 6Ó*ˆáÜ�L  Ð&;Ð<Ô=Øà�x‰x˜‹Ò2 $§(¡(¨6Ó"2ˆØ—‘˜ !Ó$ˆáØàò 	ˆHÙ˜]¨iÒ7Ùä˜hÓ'ˆDÜ˜$ÔðØ—‘ð ð ˜Ñ% t¨LÑ'9¸4À
Ñ;KØ˜‘M 4¨Ñ#7¸¸i¹Ø˜‘O T¨(¡^°T¸*Ñ5EÀtÈFÁ|ðôð ×%Ñ%¨Ò)Ø˜!‘G‘Bà˜A‘I�Dð ˜QÑ‰Mð/	ð2 	�%‰ˆØ  5Ñ(ˆÜ��t�f˜E - °°%°¸
À2À$ÀoÐVÔWó[ ð` €EØ€Jð4Ü�‰Ô/¸Ô<ˆØ�‰Ð0Ô1Ø—‘ð  Uó  V÷  _ñ  _ó  aˆØò 	 ˆBØ�L‰Lð nØ�A‘˜˜1™˜r !™u r¨!¡u¢{°°D°SÐ&9¸2¸a¹5À2ÀaÁ5Â;ÈBÐPTÐQTÐBUÐWYÐZ[ÑW\Ð^`ÐabÑ^cÐegÐhiÑejÐlnÐopÑlqÐrôtð ×!Ñ! AÒ%Ø˜‘
‘à˜a‘‘
ð	 ð 	�‰ŒØ�
‰
ŒÜÐ0°°°xÀ
¸|È1ÐMÔNð 	‡J�J„Lä	Ð& } o°[ÀÀÐ
EÔFô 
Ð8Ô:Ü	ˆJ”|�nÐ$NÐ
OÔPÜ	Ð\Ô^àˆtˆ8€OøðOØ˜‘	“ûô4 ò 4ÜÐ-¨b¨TÐ2×3Ñ3ûð4ús&   Ã$AJÅ*C&J+ ÊJ(Ê+	KÊ4KËKc                  ó  — ddl } d}| j                  |dddd¬«      }|j                  j                  «       }t	        d|› d�«       d	}| j                  |dddd¬«      }|j                  j                  «       }t	        d
|› d�«       y)õ   æŸ¥çœ‹ search.db ç»Ÿè®¡r   Nuˆ   ssh -o StrictHostKeyChecking=no root@1.94.217.116 "sqlite3 /root/search.db 'SELECT COUNT(*) FROM gov_raw WHERE site_name=\"çŽ¯è¯„å®�\"'"Té
   )ÚshellÚcapture_outputr\   r   u    ðŸ“Š search.db çŽ¯è¯„å®�æ•°æ�®: u    æ�¡zjssh -o StrictHostKeyChecking=no root@1.94.217.116 "sqlite3 /root/search.db 'SELECT COUNT(*) FROM gov_raw'"u   ðŸ“Š search.db å…¨éƒ¨æ•°æ�®: )r‹   ÚrunÚstdoutrL   r%   )r‹   ÚcmdÚresultÚcountÚcmd2Úresult2r‚   s          r   Úprint_statsr­     sŒ   € ãð [€CØ�^‰^˜C t¸DÀtÐUWˆ^ÓX€FØ�M‰M×ÑÓ!€EÜ	Ð,¨U¨G°4Ð
8Ô9ày€DØ�n‰n˜T¨¸dÈÐWYˆnÓZ€GØ�N‰N× Ñ Ó"€EÜ	Ð)¨%¨°Ð
5Õ6r   Ú__main__u   çˆ¬è™«: )Údescriptionz--limitu   æµ‹è¯•ï¼šå�ªè·‘ N æ�¡)ÚtypeÚhelpz--statsÚ
store_truer¢   )Úactionr±   u   å½“å‰� search.db çŠ¶æ€�ï¼š)r“   u   
â�± è€—æ—¶: z.1fÚsu   
å†™å…¥å�ŽçŠ¶æ€�ï¼š)é   )N);Ú__doc__r/   r"   rj   ÚsysÚtimeÚhtmlrM   Úurllib.requestr   Úurllib.parseÚhashlibÚsslrŒ   r   r   r   Ú_create_unverified_contextÚ_create_default_https_contextrU   rV   rŽ   rO   r�   rl   Újoinrm   ÚabspathÚ__file__rn   r   rT   ÚnowÚutcÚ	timestampÚTHREE_YEARS_AGOr   r+   r5   rc   rt   rz   r    r­   Ú__name__ÚargparseÚArgumentParserÚparserÚadd_argumentÚintÚ
parse_argsÚargsÚstatsÚexitr%   Út0r~   r•   r–   © r   r   ú<module>rÓ      sÚ  ðñ÷* 1× 0× 0ß :× :Ó :ß 'Ý  à$'×$BÑ$B€Ô !ð €	Ø&€Ø
Ð.Ñ
.€ØÐLÑL€Ø$Ð ð �w‰w�|‰|˜BŸG™GŸO™O¨B¯G©G¯O©O¸HÓ,EÓFÈÐReÓf€ð ð  Dð€ð (¨N¸~ÐRfÑgÐ Ø�8—<‘< §¡Ó-×7Ñ7Ó9¸KÑG€óaó
òò5òp=òó4hòV7ð ˆzÒÛØ$ˆX×$Ñ$°8¸I¸;Ð1GÔH€FØ
×Ñ˜	¨Ð2IÐÔJØ
×Ñ˜	¨,Ð=VÐÔWØ×ÑÓ€Dà‡z‚zÙŒØˆ�‰�Œñ 
Ð
&Ô'Ù„Mà	ˆ�‰‹€BÙ˜tŸz™zÔ*�H€BˆÙ	ˆN˜9˜4Ÿ9™9›; r™>¨#Ð.¨aÐ
0Ô1á	Ð
 Ô!Ù…Mð) r   