Ë
    (ˆj¬   ã                   ó  — d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
mZ ddlZ eej                  «      dkD  r2ej                  d   j                  «       r eej                  d   «      ndZe� ed ee«      z   «        ej(                  dd«      Z ej,                  «        e	d	¬
«      z
  j/                  d«      ZdZedz   ZdZddddœZddddddœgZd„ Zd„ Zd„ Z d„ Z!d„ Z"d„ Z#d„ Z$d „ Z%e&d!k(  r e%«        yy)"uõ   


ä¸´å·�åŒºäººæ°‘æ”¿åºœ - å¤šæ �ç›®çˆ¬è™«
Columns:
  col1559: æ”¿ç­–æ–‡ä»¶ (infotypeId=D00002D00004)
  col4795: çŽ¯å¢ƒä¿�æŠ¤ (infotypeId=D00004D00004)
  col1432: æ”¿åºœä¿¡æ�¯å…¬å¼€ (infotypeId=D00004D00003)
CMS: Hanweb with xxgk/search.jsp API
é    N)ÚdatetimeÚ	timedelta)ÚBeautifulSoupé   z[AutoPg] max_pages=Ú	SEARCH_DBz/root/search.dbiG  )Údaysz%Y-%m-%dzhttps://www.jxlc.gov.cnz/module/xxgk/search.jspé   z<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36ÚXMLHttpRequestz0application/x-www-form-urlencoded; charset=UTF-8)ú
User-AgentzX-Requested-WithzContent-Typeu"   ä¸´å·�åŒºäººæ°‘æ”¿åºœ-æ”¿ç­–æ–‡ä»¶ÚD00002D00004zBhttps://www.jxlc.gov.cn/col/col1559/index.html?number=D00002D00004u   æ”¿ç­–æ–‡ä»¶Údiv1432)ÚnameÚ
infotypeIdÚrefererÚcategoryÚdividc                  ó¦   — t        j                  t        «      } | j                  d«       | j                  d«       t         j                  | _        | S )NzPRAGMA journal_mode=WALzPRAGMA busy_timeout=10000)Úsqlite3ÚconnectÚDB_PATHÚexecuteÚRowÚrow_factory)Úconns    ú/root/gov_crawler/crawl_jxlc.pyÚget_connr   ,   s;   € Ü�?‰?œ7Ó#€DØ‡L�LÐ*Ô+Ø‡L�LÐ,Ô-Ü—{‘{€DÔØ€Kó    c                 óH   — | j                  d|f«      j                  «       d uS )Nz'SELECT id FROM gov_raw WHERE page_url=?)r   Úfetchone)r   Úpage_urls     r   Úis_dupr!   3   s%   € Ø�<‰<ÐAÀHÀ;ÓO×XÑXÓZÐbfÐfÐfr   c                 ó2  — |rd|v rt        |j                  dd«      «      nd}d}|r#t        |d«      }	|	j                  d¬«      }
|
d d }	 | j	                  d||||||||f«      }|j
                  dkD  S # t        $ r}t        d	|› �«       Y d }~y
d }~ww xY w)Nú-Ú r   úhtml.parserT©ÚstripéÈ   z�INSERT OR IGNORE INTO gov_raw (site_name, title, page_url, publish_date, content, date_rank, category, summary) VALUES (?, ?, ?, ?, ?, ?, ?, ?)z    [ERR] insert: F)ÚintÚreplacer   Úget_textr   ÚrowcountÚ	ExceptionÚprint)r   Ú	site_nameÚtitler    Úpublish_dateÚcontentr   Ú	date_rankÚsummaryÚ	text_soupÚplainÚcurÚes                r   Úinsert_itemr9   6   sº   € Ù6BÀsÈlÑGZ”�L×(Ñ(¨¨bÓ1Ô2Ð`a€IØ€GÙÜ! '¨=Ó9ˆ	Ø×"Ñ"¨Ð"Ó.ˆØ˜˜�+ˆð
Ø�l‰lð.ð ˜˜x¨°wÀ	È8ÐU\Ð]ó	
ˆð �|‰|˜aÑÐøÜò ÜÐ" 1 #Ð&Ô'Üûðús   Á(A5 Á5	BÁ>BÂBc                 óÄ  — g }t        j                  d| t         j                  «      }|D ]³  }t        j                  d|«      }t        j                  d|«      }|sŒ2|sŒ5|j	                  d«      }|j	                  d«      j                  «       }|j	                  d«      j                  «       }|j                  d«      r|nt        |z   }	|j                  ||	|f«       Œµ |S )Nz<li>(.*?)</li>z#href='([^']+)'\s+title=\"([^\"]+)\"z <b>\s*(\d{4}-\d{2}-\d{2})\s*</b>r   é   Úhttp)	ÚreÚfindallÚDOTALLÚsearchÚgroupr'   Ú
startswithÚBASEÚappend)
ÚhtmlÚitemsÚ
list_itemsÚliÚmÚdÚhrefr0   Údate_strÚurls
             r   Ú
parse_pagerN   J   s½   € Ø€EÜ—‘Ð-¨t´R·Y±YÓ?€JØò 1ˆÜ�I‰IÐ<¸bÓAˆÜ�I‰IÐ9¸2Ó>ˆÚ’Ø—7‘7˜1“:ˆDØ—G‘G˜A“J×$Ñ$Ó&ˆEØ—w‘w˜q“z×'Ñ'Ó)ˆHØŸ/™/¨&Ô1‘$´t¸d±{ˆCØ�L‰L˜%  hÐ/Õ0ð1ð €Lr   c                 ó„   — d| › d|› d|› d�}t        j                  t        |t        dd¬«      }d|_        |j
                  S )	NzinfotypeId=z&jdid=5&area=&divid=z&vc_title=&vc_number=&currpage=z)&vc_filenumber=&vc_all=&texttype=&fbtime=é   F)ÚdataÚheadersÚtimeoutÚverifyúutf-8)ÚrequestsÚpostÚ
SEARCH_URLÚHEADERS_TEMPLATEÚencodingÚtext)r   r   Úpage_numrQ   Úresps        r   Ú
fetch_pager^   X   sP   € à
�j�\Ð!5°e°WØ
)¨(¨Ø
3ð	5ð 	ô
 �=‰=œ¨$Ô8HÐRTÐ]bÔc€DØ€D„MØ�9‰9Ðr   c                 óR  — t        | d«      }d}|j                  dddi¬«      }|r$|j                  d«      r|d   j                  «       }d}|j                  dd	¬
«      }|rt	        |«      }|s)dD ]$  }|j                  d|¬
«      }|sŒt	        |«      } n d}|j                  dddi¬«      }|rM|j                  d«      r<|d   j                  «       }t        j                  d|«      }	|	r|	j                  d«      }|s%|j                  d«      }
|
r|
j                  d¬«      }|||fS )Nr%   r$   Úmetar   ÚArticleTitle)Úattrsr2   ÚdivÚzoom)Úid)r2   Úarticler[   ÚmainTextÚPubDatez(\d{4}-\d{1,2}-\d{1,2})r   r0   Tr&   )	r   ÚfindÚgetr'   Ústrr=   ÚmatchrA   r+   )rE   Úsoupr0   r`   r2   rd   Úcidrc   Úpub_dateÚm2Út_tags              r   Úparse_detailrr   c   s0  € Ü˜˜}Ó-€DØ€EØ�9‰9�V F¨NÐ#;ˆ9Ó<€DÙ�—‘˜Ô#Ø�Y‘×%Ñ%Ó'ˆà€GØ�9‰9�U˜vˆ9Ó&€DÙÜ�d“)ˆÙØ=ò 	ˆCØ—)‘)˜E c�)Ó*ˆCÚÜ˜c›(�Ùð		ð €HØ�9‰9�V F¨IÐ#6ˆ9Ó7€DÙ�—‘˜Ô#Ø˜	‘?×(Ñ(Ó*ˆÜ�X‰XÐ0°(Ó;ˆÙØ—x‘x “{ˆHáØ—	‘	˜'Ó"ˆÙØ—N‘N¨�NÓ.ˆEà�'˜8Ð#Ð#r   c                 ó  — | d   }| d   }| d   }| d   }| d   t         d<   t        dd› �«       t        d	|› d
�«       t        ||d«      }t        |«      }t	        j
                  d|«      }|rt        |j                  d«      «      n
t        |«      }	|	t        z   dz
  t        z  }
t        d	|› d|	› d|
› d�«       g }d}t        dt        |
t        xs |
«      dz   «      D ]š  }|dk(  r|}n	 t        t        |||«      «      }|D ])  \  }}}|t         k\  r|j#                  |||f«       Œ%|dz  }Œ+ |dz  dk(  rt        d|› d|
› dt        |«      › d�«       |j%                  «        t        j                  d«       Œœ t        d	|› dt        |«      › d|› d�«       dx}x}}t'        |d«      D ]Ì  \  }\  }}}	 t)        ||«      r|dz  }Œt+        j,                  |ddidd ¬!«      }d"|_        t1        |j2                  «      \  }}}|s|}|s|}t5        |||||||«      r|dz  }n|dz  }|j%                  «        t        j                  d%«       |d&z  dk(  sŒªt        d'|› dt        |«      › d(|› d)|› d*|› �
«       ŒÎ t        d	|› d+|› d,|› d-|› d.�	«       ||||d/œS # t        $ r1}t        d|› d|› �«       t        j                  d«       Y d }~�Œðd }~ww xY w# t        $ r#}|dz  }t        d#|d d$ › d|› �«       Y d }~ŒÖd }~ww xY w)0Nr   r   r   r   r   ÚRefererú
ú<============================================================ú[z] Starting...r   znTotalCount"\s*value="(\d+)"z	] Total: z
 records, z pagesr   z  ERROR page z: é   é2   z
  Scanned ú/z, z within cutoffg      à?z] Scan done: z within cutoff, z olderr   zMozilla/5.0rP   F)rR   rS   rT   rU   z  ERROR é(   g333333Ó?é   z	  Detail z: new=z, dup=z, err=z	] Done: +ú new, z dup, z err)ÚsiteÚnewÚdupÚerr)rY   r.   r^   rN   r=   r@   r)   rA   ÚlenÚPER_PAGEÚrangeÚminÚ_MAX_PGr-   ÚtimeÚsleepÚCUTOFF_DATErD   ÚcommitÚ	enumerater!   rV   rj   rZ   rr   r[   r9   )Úcolr   r   r   r   r   rE   Úitems_page1ÚtrÚtotal_recordsÚtotal_pagesÚ
all_activeÚall_skippedÚpagerF   r8   r0   rM   rL   Ú	new_countÚ	dup_countÚerror_countÚidxÚdetail_respÚdetail_titler2   ro   s                              r   Úprocess_columnrš   …   s|  € Øˆv‰;€DØ�\Ñ"€JØ�:‰€HØ�‰L€EØ"% i¡.Ô�YÑä	ˆBˆvˆhˆ-ÔÜ	ˆAˆdˆV�=Ð
!Ô"ä�j %¨Ó+€DÜ˜TÓ"€KÜ	�‰Ð2°DÓ	9€BÙ(*”C˜Ÿ™ ›Ô$´°KÓ0@€MØ ¤8Ñ+¨aÑ/´HÑ<€KÜ	ˆAˆdˆV�9˜]˜O¨:°k°]À&Ð
IÔJð €JØ€KÜ�aœ˜[¬'Ò*@°[ÓAÀ!ÑCÓDò ˆØ�1Š9Ø‰EðÜ"¤:¨j¸%ÀÓ#FÓG�ð %*ò 	!Ñ ˆE�3˜Øœ;Ò&Ø×!Ñ! 5¨#¨xÐ"8Õ9à˜qÑ ‘ð		!ð �"‰9˜Š>Ü�J˜t˜f A k ]°"´S¸³_Ð4EÀ^ÐTÔUà�‰ŒÜ�
‰
�3�ð+ô. 
ˆAˆdˆV�=¤ Z£Ð 1Ð1AÀ+ÀÈfÐ
UÔVð +,Ð+€IÐ+�	˜KÜ'0°¸QÓ'?ò lÑ#ˆÑ#ˆe�S˜(ð	0Ü�d˜CÔ Ø˜Q‘�	Øä"Ÿ,™, s°\À=Ð4QÐ[]ÐfkÔlˆKØ#*ˆKÔ Ü.:¸;×;KÑ;KÓ.LÑ+ˆL˜' 8áØ$�ÙØ#�ä˜4  |°S¸(ÀGÈXÔVØ˜Q‘‘	à˜Q‘�	ð
 	�‰ŒÜ�
‰
�3ŒØ�‰8�q‹=Ü�I˜c˜U !¤C¨
£OÐ#4°F¸9¸+ÀVÈIÈ;ÐV\Ð]hÐ\iÐjÕkð5lô8 
ˆAˆdˆV�9˜Y˜K v¨i¨[¸¸{¸mÈ4Ð
PÔQØ °9À[ÑQÐQøôe ò Ü˜ d V¨2¨a¨SÐ1Ô2Ü—
‘
˜1”ÝûðûôP ò 	0Ø˜1ÑˆKÜ�H˜U 3 B˜Z˜L¨¨1¨#Ð.×/Ñ/ûð	0ús7   Ã,JÆ8KÇ
AKÊ	KÊ$&KËKË	LË!K?Ë?Lc                  óR  — t        «       } g }t        D ]   }	 t        || «      }|j                  |«       Œ" | j                  «        t        dd› �«       t        d«       |D ]Y  }d|v r)d	|j                  dd
«      › d|j                  dd
«      › d�nd|j                  dd«      › �}t        d|d   d›d|› �«       Œ[ t        t        j                  |d¬«      «       y # t        $ r?}t        d|d   › d|› �«       |j                  |d   t        |«      dœ«       Y d }~�Œd }~ww xY w)Nrw   r   z	] FATAL: )r~   Úerrorru   rv   zSummary:r   ú+r   r}   r€   z dupzERROR: rœ   r$   z  r~   Ú30sú F)Úensure_ascii)r   ÚCOLUMNSrš   rD   r-   r.   rk   Úcloserj   ÚjsonÚdumps)r   ÚresultsrŒ   Úrr8   Ústatuss         r   Úmainr¨   Ô   s1  € Ü‹:€DØ€GÜò Cˆð	CÜ˜s DÓ)ˆAØ�N‰N˜1ÕðCð 	‡J�J„Lä	ˆBˆvˆhˆ-ÔÜ	ˆ*ÔØò -ˆØCHÈAÁ:�1�Q—U‘U˜5 “^Ð$ F¨1¯5©5°°q«>Ð*:¸$Ñ?ÐU\Ð]^×]bÑ]bÐcjÐkmÓ]nÐ\oÐSpˆÜ��1�V‘9˜S�/  6 (Ð+Õ,ð-ô 
Œ$�*‰*�W¨5Ô
1Õ2øô ò 	CÜ�A�c˜&‘k�] )¨A¨3Ð/Ô0Ø�N‰N C¨¡K¼#¸a»&ÑA×BÒBûð	Cús   —CÃ	D&Ã'4D!Ä!D&Ú__main__)'Ú__doc__Úosr=   Úsysr   rV   r£   r‡   r   r   Úbs4r   Ú_SYSr‚   ÚargvÚisdigitr)   r†   r.   rk   Úgetenvr   ÚnowÚstrftimer‰   rC   rX   rƒ   rY   r¡   r   r!   r9   rN   r^   rr   rš   r¨   Ú__name__© r   r   ú<module>r¶      s,  ðñ	÷ 2× 1× 1Ó 1ß (Ý ã Ù" 4§9¡9›~°Ò1°d·i±iÀ±l×6JÑ6JÔ6L‰#ˆd�i‰i˜‰lÔ
ÐRV€Ø
ÐÙ	Ð
¡# g£,Ñ
.Ô/à
ˆ"�)‰)�KÐ!2Ó
3€Øˆx�|‰|‹~¡	¨uÔ 5Ñ5×?Ñ?À
ÓK€Ø €ØÐ-Ñ-€
Ø€ð QØ(ØFñÐ ð 5Ø$ØWØ"Øñð€òògòò(òò$òDLRò^3ð( ˆzÒÙ…Fð r   