Ë
    ¸¬ujÖ"  ã                   óv  — d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m	Z	m
Z
 ddlmZ  ej                  dd«      Z e	j                  «        e
d¬«      z
  j!                  d	«      Zd
Zedz   ZdZddddœZddddddœddddddœddddddœgZd„ Zd „ Zd!„ Zd"„ Zd#„ Zd$„ Zd(d%„Zd&„ Zed'k(  r e«        yy))uó   
ä¸´å·�åŒºäººæ°‘æ”¿åºœ - å¤šæ �ç›®çˆ¬è™«
Columns:
  col1559: æ”¿ç­–æ–‡ä»¶ (infotypeId=D00002D00004)
  col4795: çŽ¯å¢ƒä¿�æŠ¤ (infotypeId=D00004D00004)
  col1432: æ”¿åºœä¿¡æ�¯å…¬å¼€ (infotypeId=D00004D00003)
CMS: Hanweb with xxgk/search.jsp API
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupÚ	SEARCH_DBz/root/search.dbiG  )Údaysz%Y-%m-%dzhttps://www.jxlc.gov.cnz/module/xxgk/search.jspé   z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36ÚXMLHttpRequestz0application/x-www-form-urlencoded; charset=UTF-8)ú
User-AgentzX-Requested-WithzContent-Typeu"   ä¸´å·�åŒºäººæ°‘æ”¿åºœ-æ”¿ç­–æ–‡ä»¶ÚD00002D00004zBhttps://www.jxlc.gov.cn/col/col1559/index.html?number=D00002D00004u   æ”¿ç­–æ–‡ä»¶Údiv1432)ÚnameÚ
infotypeIdÚrefererÚcategoryÚdividu"   ä¸´å·�åŒºäººæ°‘æ”¿åºœ-çŽ¯å¢ƒä¿�æŠ¤ÚD00004D00004zHhttps://www.jxlc.gov.cn/col/col4795/index.html?number=D00004D00004D00007u   çŽ¯å¢ƒä¿�æŠ¤u(   ä¸´å·�åŒºäººæ°‘æ”¿åºœ-æ”¿åºœä¿¡æ�¯å…¬å¼€ÚD00004D00003z#http://www.jxlc.gov.cn/col/col1432/u   æ”¿åºœä¿¡æ�¯å…¬å¼€c                  ó¦   — t        j                  t        «      } | j                  d«       | j                  d«       t         j                  | _        | S )NzPRAGMA journal_mode=WALzPRAGMA busy_timeout=10000)Úsqlite3ÚconnectÚDB_PATHÚexecuteÚRowÚrow_factory)Úconns    ú"/root/gov_crawler/crawl_jxlc_v2.pyÚget_connr   3   s;   € Ü�?‰?œ7Ó#€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ü—{‘{€DÔØ€Kó    c                 óH   — | j                  d|f«      j                  «       d uS )Nz'SELECT id FROM gov_raw WHERE page_url=?)r   Úfetchone)r   Úpage_urls     r   Úis_dupr"   :   s%   € Ø�<‰<ÐAÀHÀ;ÓO×XÑXÓZÐbfÐfÐfr   c                 ó¼  — |rd|v rt        |j                  dd«      «      nd}d}|r#t        |d«      }	|	j                  d¬«      }
|
d d }	 | j	                  d||||||||f«      }|j
                  dkD  r 	 | j	                  d	||xs dd d
 |||f«       yy# t        $ r}t        d|› �«       Y d }~yd }~ww xY w# t        $ r}t        d|› �«       Y d }~yd }~ww xY w)Nú-Ú r   úhtml.parserT©ÚstripéÈ   z�INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)zfINSERT INTO gov_search_v3 (title, content, source_url, publish_date, site_name) VALUES (?, ?, ?, ?, ?)i¸  z%    [FTS WARN] gov_search_v3 insert: Fz    [ERR] insert: )ÚintÚreplacer   Úget_textr   ÚrowcountÚ	ExceptionÚprint)r   Ú	site_nameÚtitler!   Úpublish_dateÚcontentr   Ú	date_rankÚsummaryÚ	text_soupÚplainÚcurÚfts_errÚes                 r   Úinsert_itemr;   =   s  € Ù6BÀsÈlÑGZ”�L×(Ñ(¨¨bÓ1Ô2Ð`a€IØ€GÙÜ! '¨=Ó9ˆ	Ø×"Ñ"¨Ð"Ó.ˆØ˜˜�+ˆðØ�l‰lð.ð ˜˜x¨°wÀ	È8ÐU\Ð]ó	
ˆð �<‰<˜!ÒðIØ—‘ð-à˜Wš]¨¨E¨TÐ2°H¸lÈIÐVôð Øøô ò IÜÐ=¸g¸YÐG×HÐHØûðIûô ò ÜÐ" 1 #Ð&Ô'Üûðús<   Á)B: Á6B Â	B7ÂB2Â-B: Â2B7Â7B: Â:	CÃCÃCc                 óÄ  — g }t        j                  d| t         j                  «      }|D ]³  }t        j                  d|«      }t        j                  d|«      }|sŒ2|sŒ5|j	                  d«      }|j	                  d«      j                  «       }|j	                  d«      j                  «       }|j                  d«      r|nt        |z   }	|j                  ||	|f«       Œµ |S )Nz<li>(.*?)</li>z#href='([^']+)'\s+title=\"([^\"]+)\"z <b>\s*(\d{4}-\d{2}-\d{2})\s*</b>é   é   Úhttp)	ÚreÚfindallÚDOTALLÚsearchÚgroupr(   Ú
startswithÚBASEÚappend)
ÚhtmlÚitemsÚ
list_itemsÚliÚmÚdÚhrefr1   Údate_strÚurls
             r   Ú
parse_pagerQ   \   s½   € Ø€EÜ—‘Ð-¨t´R·Y±YÓ?€JØò 1ˆÜ�I‰IÐ<¸bÓAˆÜ�I‰IÐ9¸2Ó>ˆÚ’Ø—7‘7˜1“:ˆDØ—G‘G˜A“J×$Ñ$Ó&ˆEØ—w‘w˜q“z×'Ñ'Ó)ˆHØŸ/™/¨&Ô1‘$´t¸d±{ˆCØ�L‰L˜%  hÐ/Õ0ð1ð €Lr   c                 ó„   — d| › d|› d|› d�}t        j                  t        |t        dd¬«      }d|_        |j
                  S )	NzinfotypeId=z&jdid=5&area=&divid=z&vc_title=&vc_number=&currpage=z)&vc_filenumber=&vc_all=&texttype=&fbtime=é   F)ÚdataÚheadersÚtimeoutÚverifyúutf-8)ÚrequestsÚpostÚ
SEARCH_URLÚHEADERS_TEMPLATEÚencodingÚtext)r   r   Úpage_numrT   Úresps        r   Ú
fetch_pagera   j   sP   € à
�j�\Ð!5°e°WØ
)¨(¨Ø
3ð	5ð 	ô
 �=‰=œ¨$Ô8HÐRTÐ]bÔc€DØ€D„MØ�9‰9Ðr   c                 óR  — t        | d«      }d}|j                  dddi¬«      }|r$|j                  d«      r|d   j                  «       }d}|j                  dd	¬
«      }|rt	        |«      }|s)dD ]$  }|j                  d|¬
«      }|sŒt	        |«      } n d}|j                  dddi¬«      }|rM|j                  d«      r<|d   j                  «       }t        j                  d|«      }	|	r|	j                  d«      }|s%|j                  d«      }
|
r|
j                  d¬«      }|||fS )Nr&   r%   Úmetar   ÚArticleTitle)Úattrsr3   ÚdivÚzoom)Úid)r3   Úarticler^   ÚmainTextÚPubDatez(\d{4}-\d{1,2}-\d{1,2})r=   r1   Tr'   )	r   ÚfindÚgetr(   Ústrr@   ÚmatchrD   r,   )rH   Úsoupr1   rc   r3   rg   Úcidrf   Úpub_dateÚm2Út_tags              r   Úparse_detailru   u   s0  € Ü˜˜}Ó-€DØ€EØ�9‰9�V F¨NÐ#;ˆ9Ó<€DÙ�—‘˜Ô#Ø�Y‘×%Ñ%Ó'ˆà€GØ�9‰9�U˜vˆ9Ó&€DÙÜ�d“)ˆÙØ=ò 	ˆCØ—)‘)˜E c�)Ó*ˆCÚÜ˜c›(�Ùð		ð €HØ�9‰9�V F¨IÐ#6ˆ9Ó7€DÙ�—‘˜Ô#Ø˜	‘?×(Ñ(Ó*ˆÜ�X‰XÐ0°(Ó;ˆÙØ—x‘x “{ˆHáØ—	‘	˜'Ó"ˆÙØ—N‘N¨�NÓ.ˆEà�'˜8Ð#Ð#r   c                 ó  — | d   }| d   }| d   }| d   }| d   t         d<   t        dd› �«       t        d	|› d
�«       t        ||d«      }t        |«      }t	        j
                  d|«      }	|	rt        |	j                  d«      «      n
t        |«      }
|
t        z   dz
  t        z  }t        ||«      }t        d	|› d|
› d|› d|› �«       g }d}t        d|dz   «      D ]š  }|dk(  r|}n	 t        t        |||«      «      }|D ])  \  }}}|t        k\  r|j!                  |||f«       Œ%|dz  }Œ+ |dz  dk(  rt        d|› d|› dt        |«      › d�«       |j#                  «        t        j                  d«       Œœ t        d	|› dt        |«      › d|› d�«       dx}x}}t%        |d«      D ]Ì  \  }\  }}}	 t'        ||«      r|dz  }Œt)        j*                  |ddidd ¬!«      }d"|_        t/        |j0                  «      \  }}}|s|}|s|}t3        |||||||«      r|dz  }n|dz  }|j#                  «        t        j                  d%«       |d&z  dk(  sŒªt        d'|› dt        |«      › d(|› d)|› d*|› �
«       ŒÎ t        d	|› d+|› d,|› d-|› d.�	«       ||||d/œS # t        $ r1}t        d|› d|› �«       t        j                  d«       Y d }~�Œðd }~ww xY w# t        $ r#}|dz  }t        d#|d d$ › d|› �«       Y d }~ŒÖd }~ww xY w)0Nr   r   r   r   r   ÚRefererú
ú<============================================================ú[z] Starting...r=   znTotalCount"\s*value="(\d+)"z	] Total: z
 records, z pages, scanning r   z  ERROR page z: é   é2   z
  Scanned ú/z, z within cutoffg      à?z] Scan done: z within cutoff, z olderr
   zMozilla/5.0rS   F)rU   rV   rW   rX   z  ERROR é(   g333333Ó?é   z	  Detail z: new=z, dup=z, err=z	] Done: +ú new, z dup, z err)ÚsiteÚnewÚdupÚerr)r\   r/   ra   rQ   r@   rC   r*   rD   ÚlenÚPER_PAGEÚminÚranger.   ÚtimeÚsleepÚCUTOFF_DATErG   ÚcommitÚ	enumerater"   rY   rm   r]   ru   r^   r;   )Úcolr   Ú	max_pagesr   r   r   r   rH   Úitems_page1ÚtrÚtotal_recordsÚtotal_pagesÚpages_to_scanÚ
all_activeÚall_skippedÚpagerI   r:   r1   rP   rO   Ú	new_countÚ	dup_countÚerror_countÚidxÚdetail_respÚdetail_titler3   rr   s                                r   Úprocess_columnrž   —   s  € Øˆv‰;€DØ�\Ñ"€JØ�:‰€HØ�‰L€EØ"% i¡.Ô�YÑä	ˆBˆvˆhˆ-ÔÜ	ˆAˆdˆV�=Ð
!Ô"ä�j %¨Ó+€DÜ˜TÓ"€KÜ	�‰Ð2°DÓ	9€BÙ(*”C˜Ÿ™ ›Ô$´°KÓ0@€MØ ¤8Ñ+¨aÑ/´HÑ<€KÜ˜ YÓ/€MÜ	ˆAˆdˆV�9˜]˜O¨:°k°]ÐBSÐTaÐSbÐ
cÔdà€JØ€KÜ�a˜¨Ñ*Ó+ò ˆØ�1Š9Ø‰EðÜ"¤:¨j¸%ÀÓ#FÓG�ð %*ò 	!Ñ ˆE�3˜Øœ;Ò&Ø×!Ñ! 5¨#¨xÐ"8Õ9à˜qÑ ‘ð		!ð �"‰9˜Š>Ü�J˜t˜f A m _°B´s¸:³Ð6GÀ~ÐVÔWà�‰ŒÜ�
‰
�3�ð+ô. 
ˆAˆdˆV�=¤ Z£Ð 1Ð1AÀ+ÀÈfÐ
UÔVà*+Ð+€IÐ+�	˜KÜ'0°¸QÓ'?ò lÑ#ˆÑ#ˆe�S˜(ð	0Ü�d˜CÔ Ø˜Q‘�	Øä"Ÿ,™, s°\À=Ð4QÐ[]ÐfkÔlˆKØ#*ˆKÔ Ü.:¸;×;KÑ;KÓ.LÑ+ˆL˜' 8áØ$�ÙØ#�ä˜4  |°S¸(ÀGÈXÔVØ˜Q‘‘	à˜Q‘�	ð
 	�‰ŒÜ�
‰
�3ŒØ�‰8�q‹=Ü�I˜c˜U !¤C¨
£OÐ#4°F¸9¸+ÀVÈIÈ;ÐV\Ð]hÐ\iÐjÕkð5lô8 
ˆAˆdˆV�9˜Y˜K v¨i¨[¸¸{¸mÈ4Ð
PÔQØ °9À[ÑQÐQøôc ò Ü˜ d V¨2¨a¨SÐ1Ô2Ü—
‘
˜1”ÝûðûôN ò 	0Ø˜1ÑˆKÜ�H˜U 3 B˜Z˜L¨¨1¨#Ð.×/Ñ/ûð	0ús7   Ã(JÆ4KÇAKÊ	KÊ &KËKË	L ËK;Ë;L c                  ó   — t        j                  d¬«      } | j                  dt        dd¬«       | j	                  «       }t        |j                  d«      }t        «       }g }t        D ]"  }	 t        |||¬«      }|j                  |«       Œ$ |j                  «        t        dd› �«       t        d«       |D ]Y  }d|v r)d|j                  dd«      › d|j                  dd«      › d�nd|j                  dd«      › �}t        d|d   d›d|› �«       Œ[ t        t!        j"                  |d¬«      «       y # t        $ r?}t        d	|d
   › d|› �«       |j                  |d
   t        |«      dœ«       Y d }~�Œd }~ww xY w)Nu#   Crawl ä¸´å·�åŒºäººæ°‘æ”¿åºœ columns)Údescriptionz--pagesé   zKMax pages to scan per column (default: 3 for daily, max 999 for full crawl))ÚtypeÚdefaultÚhelpiç  )r�   rz   r   z	] FATAL: )r�   Úerrorrx   ry   zSummary:r‚   ú+r   r€   rƒ   z dupzERROR: r¥   r%   z  r�   Ú30sú F)Úensure_ascii)ÚargparseÚArgumentParserÚadd_argumentr*   Ú
parse_argsr‡   Úpagesr   ÚCOLUMNSrž   rG   r.   r/   rn   Úcloserm   ÚjsonÚdumps)	ÚparserÚargsr�   r   ÚresultsrŽ   Úrr:   Ústatuss	            r   Úmainr¸   å   sƒ  € Ü×$Ñ$Ð1VÔW€FØ
×ÑØÜØØZð	 ô ð ×ÑÓ€DÜ�D—J‘J Ó$€Iä‹:€DØ€GÜò Cˆð	CÜ˜s D°IÔ>ˆAØ�N‰N˜1ÕðCð 	‡J�J„Lä	ˆBˆvˆhˆ-ÔÜ	ˆ*ÔØò -ˆØCHÈAÁ:�1�Q—U‘U˜5 “^Ð$ F¨1¯5©5°°q«>Ð*:¸$Ñ?ÐU\Ð]^×]bÑ]bÐcjÐkmÓ]nÐ\oÐSpˆÜ��1�V‘9˜S�/  6 (Ð+Õ,ð-ô 
Œ$�*‰*�W¨5Ô
1Õ2øô ò 	CÜ�A�c˜&‘k�] )¨A¨3Ð/Ô0Ø�N‰N C¨¡K¼#¸a»&ÑA×BÒBûð	Cús   Á,D5Ä5	E=Ä>4E8Å8E=Ú__main__)r¡   ) Ú__doc__Úosr@   Úsysr   rY   r±   r‰   rª   r   r   Úbs4r   Úgetenvr   ÚnowÚstrftimer‹   rF   r[   r†   r\   r¯   r   r"   r;   rQ   ra   ru   rž   r¸   Ú__name__© r   r   ú<module>rÃ      s  ðñ÷ <× ;× ;× ;ß (Ý à
ˆ"�)‰)�KÐ!2Ó
3€Øˆx�|‰|‹~¡	¨uÔ 5Ñ5×?Ñ?À
ÓK€Ø €ØÐ-Ñ-€
Ø€ð QØ(ØFñÐ ð 5Ø$ØWØ"Øñð 5Ø$Ø]Ø"Øñð ;Ø$Ø8Ø(Øñð€ò2ògòò>òò$óDKRò\3ð< ˆzÒÙ…Fð r   