Ë
    ¸¬uj"  ã                   ó¾  — d Z ddlZddlZddlZddlZddlZddlZddlZddlmZm	Z	 ddl
mZmZ ddlmZmZ ddlZ eej$                  «      dkD  r2ej$                  d   j'                  «       r eej$                  d   «      ndZe� ed ee«      z   «       dZd	Zd
diZdZdZdZddlZ ej>                  dd«      Z d„ Z!d„ Z"efd„Z#d„ Z$d„ Z%dd„Z&e'dk(  r e&«        yy)uØ   


crawl_zichuan.py - æ·„å�šå¸‚æ·„å·�åŒºç”Ÿæ€�çŽ¯å¢ƒå±€ä¿¡æ�¯å…¬å¼€çˆ¬è™«
Site: www.zichuan.gov.cn
Channel: è¡Œæ”¿è®¸å�¯å…¬ç¤º (site_zcqsthjj)
Total: 1674 records, 168 pages
CMS: å¤§æ±‰ç‰ˆé€šæ”¿åºœä¿¡æ�¯å…¬å¼€å¹³å�°
é    N)ÚdatetimeÚ	timedelta)ÚRequestÚurlopen)ÚURLErrorÚ	HTTPErroré   z[AutoPg] max_pages=zhttp://www.zichuan.gov.cnzK/gongkai/site_zcqsthjj/channel_c_5f9f6a3dc916f710ccee90f9_n_1605684149.7068z
User-Agentz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36é   é   gš™™™™™¹?Ú	SEARCH_DBz/root/search.dbc                  óZ   — t        j                  t        «      } | j                  d«       | S )NzPRAGMA journal_mode=WAL)Úsqlite3ÚconnectÚDB_PATHÚexecute)Úconns    ú"/root/gov_crawler/crawl_zichuan.pyÚget_connr       s"   € Ü�?‰?œ7Ó#€DØ‡L�LÐ*Ô+Ø€Kó    c                 ó˜   — | j                  d«      j                  «       }|D �cg c]  }|d   ‘Œ	 }}|s| j                  d«       y y c c}w )NzPRAGMA table_info(gov_raw)r	   a›  
            CREATE TABLE gov_raw (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                title TEXT,
                content TEXT,
                publish_date TEXT,
                url TEXT,
                site_name TEXT,
                source TEXT DEFAULT '',
                attachments TEXT DEFAULT '',
                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
            )
        )r   Úfetchall)r   ÚexistingÚrÚ	col_namess       r   Úensure_tabler   %   sO   € à�|‰|Ð8Ó9×BÑBÓD€HØ'Ö(˜!��1“Ð(€IÐ(ÙØ�‰ð õ 	ð ùò )s   ¤Ac                 óÚ  — t        |«      D ]P  }	 t        | t        ¬«      }t        |t        ¬«      5 }|j                  «       }d d d «       j                  d«      c S  y # 1 sw Y   ŒxY w#  j                  dd¬«      cY c S xY w# t        $ rY}||dz
  k  r t        j                  t        |dz   z  «       n't        d| › d	|› �t        j                  ¬
«       Y d }~ y Y d }~ŒÙd }~ww xY w)N)Úheaders)Útimeoutzutf-8ÚgbkÚreplace)Úerrorsr	   z  [WARN] Failed to fetch z: )Úfile)Úranger   ÚHEADERSr   ÚTIMEOUTÚreadÚdecodeÚ	ExceptionÚtimeÚsleepÚSLEEPÚprintÚsysÚstderr)ÚurlÚretriesÚattemptÚreqÚrespÚdataÚes          r   Úfetchr6   8   sÎ   € Ü˜“>ò ˆð	Ü˜#¤wÔ/ˆCÜ˜¤gÔ.ð #°$Ø—y‘y“{�÷#àŸ™ GÓ,Ò,ñ÷#ð #ûð @˜4Ÿ;™; u°Y˜;Ó?Ô?ûÜò 	Ø˜ 1™Ò$Ü—
‘
œ5 G¨a¡KÑ0Õ1äÐ1°#°°b¸¸Ð<Ä3Ç:Á:ÕNÝô 2ûð	úsG   �"B²A ÁBÁA,Á A)	Á%BÁ,BÂBÂBÂ	C*ÂA	C%Ã%C*c                 ó4  — g }t        j                  d| t         j                  «      }|D ]k  \  }}}t        j                  dd|«      j	                  «       }|j                  d«      r	t        |z   n|}|j	                  «       }|j                  |||f«       Œm |S )z2Parse list page: extract (title, url, date) tupleszh<td class="zfxxgk-list-title"><a href="([^"]*)">(.*?)</a></td>\s*<td class="zfxxgk-list-time">(.*?)</td>z<[^>]+>Ú ú/)ÚreÚfindallÚDOTALLÚsubÚstripÚ
startswithÚBASE_URLÚappend)ÚhtmlÚitemsÚrowsÚhrefÚ
title_htmlÚdateÚtitleÚfull_urls           r   Úparse_list_pagerJ   G   s�   € à€Eä�:‰:ØsØŒb�i‰ió€Dð #'ò .Ñˆˆj˜$Ü—‘�z 2 zÓ2×8Ñ8Ó:ˆØ&*§o¡o°cÔ&:”8˜d’?ÀˆØ�z‰z‹|ˆØ�‰�e˜X tÐ,Õ-ð	.ð
 €Lr   c                 ó4  — d}d}d}t        j                  d| «      }|r|j                  d«      j                  «       }t        j                  d| «      }|r"|j                  d«      j                  «       dd }t        j                  d| t         j                  «      }|s%t        j                  d| t         j                  «      }|ro|j                  d«      }t        j
                  d	d|t         j                  ¬
«      }t        j
                  dd|t         j                  ¬
«      }|j                  «       }t        j
                  dd„ |«      }|sFt        j                  d| t         j                  «      }|r|j                  d«      j                  «       }|||fS )z/Parse detail page: extract title, date, contentr8   z+<meta name="ArticleTitle" content="([^"]*)"r	   z&<meta name="PubDate" content="([^"]*)"Né
   zd<div class="details-content " id="details-content">(.*?)</div>\s*</div>\s*(?:<!--endprint-->|</div>)z1<div class="details-content">(.*?)</div>\s*</div>z"<h1 class="details-title">.*?</h1>)Úflagsz)<div class="details-info clear">.*?</div>z-href="(doc_[^"]*\.(?:doc|docx|pdf|xls|xlsx))"c                 ó€   — dt         › | j                  d«      j                  d«      sdnd› | j                  d«      › d�S )Nzhref="r	   r9   r8   ú")r@   Úgroupr?   )Úms    r   ú<lambda>zparse_detail.<locals>.<lambda>w   s?   € �Fœ8˜*°·±¸³
×0EÑ0EÀcÔ0J¡SÐPRÐ$SÐTU×T[ÑT[Ð\]ÓT^ÐS_Ð_`Ða€ r   z$class="zfxxgk-text"[^>]*>(.*?)</div>)r:   ÚsearchrP   r>   r<   r=   )rB   r/   rH   Úpublish_dateÚcontentrQ   Úcontent_htmls          r   Úparse_detailrW   V   sk  € à€EØ€LØ€Gô 	�	‰	Ð@À$ÓG€AÙØ—‘˜“
× Ñ Ó"ˆô 	�	‰	Ð;¸TÓB€AÙØ—w‘w˜q“z×'Ñ'Ó)¨#¨2Ð.ˆô 	�	‰	ÐyÐ{ô  BD÷  BKñ  BKó  	L€AÙä�I‰IÐJÈDÔRT×R[ÑR[Ó\ˆáØ—w‘w˜q“zˆä—v‘vÐCÀRÈÔ]_×]fÑ]fÔgˆä—v‘vÐJÈBÐP\Ôdf×dmÑdmÔnˆØ×$Ñ$Ó&ˆô �f‰fØ8ÙaØó€Gñ Ü�I‰IÐ=¸tÄRÇYÁYÓOˆÙØ—g‘g˜a“j×&Ñ&Ó(ˆGà�, Ð'Ð'r   c                 óÔ  — t        «       }t        |«       d}t        j                  «       t	        | dz  ¬«      z
  j                  d«      }t        d|› d�«       d}d}d}d	}|t        xs |k  �rPt        › t        › d
|› d�}t        d|› d�«       t        d|› �«       t        |«      }	|	st        d«       �nt        |	«      }
|
st        d«       �nít        dt        |
«      › d�«       d}|
D ]  \  }}}|}||k\  sŒd} n |r|dkD  rt        d|› d�«       �n¥|
D �]r  \  }}}||k  rt        d|› d|dd › d|› d�«       |dz  }Œ+|j                  d||f«      j                  «       }|rt        d|› d|dd › d�«       |dz  }Œjt        j                   t"        «       t        |«      }|st        d |› d|dd › d�«       Œ¦t%        ||«      \  }}}|s|}|s|}t'        j(                  |j+                  «       «      j-                  «       dd! }t/        |d!«      d"z  }|r|dd	 nd#}|j                  d$||||||||f«       |j                  d%||||f«       |j1                  «        |dz  }|r	d&|dd › d'�nd#}t        d(|› d|dd) › d|› �«       �Œu |dz  }t        j                   t"        «       |t        xs |k  r�ŒP|j3                  «        t        d*|› d+|› d,|dz
  › d-�«       |S ).zMain crawl functionÚzcqsthjjé   )Údaysz%Y-%m-%dzDate cutoff: z+ (articles older than this will be skipped)r   r	   éÈ   z/?open=fdzdgknr&nr_page=z&nr_per_page=10z

--- Page z ---z
Fetching: z  Failed to fetch, stoppingz  No items found, stoppingz  Found z itemsTFz  All items older than z, stopping paginationz	  [SKIP] ú Né(   z... (older than ú)z7SELECT id FROM gov_raw WHERE page_url=? AND site_name=?z  [EXISTS] z...z	  [FAIL] é   iÿÉš;r8   zžINSERT OR REPLACE INTO gov_raw (id, site_name, title, content, publish_date, source_url, page_url, summary)
                   VALUES (?, ?, ?, ?, ?, ?, ?, ?)zWINSERT OR REPLACE INTO gov_search(rowid, title, site_name, summary) VALUES (?, ?, ?, ?)z [z...]z  [NEW] é2   z
=== Done! New: z, Skipped/Existing: z, Pages crawled: z ===)r   r   r   Únowr   Ústrftimer,   Ú_MAX_PGr@   ÚCHANNEL_PATHr6   rJ   Úlenr   Úfetchoner)   r*   r+   rW   ÚhashlibÚmd5ÚencodeÚ	hexdigestÚintÚcommitÚclose)Úlimit_monthsr   Ú	site_nameÚcutoff_dateÚ	total_newÚtotal_skippedÚpageÚ	max_pagesÚlist_urlrB   rC   Úall_too_oldrH   Ú
detail_urlÚ	list_dateÚdate_to_checkr   Údetail_htmlÚdetail_titleÚpub_daterU   Úurl_hashÚ	record_idÚsummaryÚsource_displays                            r   Ú	crawl_allr‚   ƒ   sŠ  € ä‹:€DÜ�Ôà€Iô —<‘<“>¤I°<À"Ñ3DÔ$EÑE×OÑOÐPZÓ[€KÜ	ˆM˜+˜Ð&QÐ
RÔSà€IØ€MØ€DØ€Ià
”7Ò'˜iÓ
(ä�Z¤˜~Ð-EÀdÀVÈ?Ð[ˆÜ�˜D˜6 Ð&Ô'Ü�
˜8˜*Ð%Ô&ä�X‹ˆÙÜÐ/Ô0Ùä Ó%ˆÙÜÐ.Ô/Ùä�œ˜U›˜ FÐ+Ô,ð ˆØ,1ò 	Ñ(ˆE�:˜yØ%ˆMØ Ó+Ø#�Ùð		ñ ˜4 !š8ÜÐ+¨K¨=Ð8MÐNÔOÙà,1ó 5	PÑ(ˆE�:˜yà˜;Ò&Ü˜	 ) ¨A¨e°C°R¨j¨\Ð9IÈ+ÈÐVWÐXÔYØ Ñ"�Øð —|‘|ØIØ˜YÐ'ó÷ ‰h‹jð ñ Ü˜ I ;¨a°°c°r°
¨|¸3Ð?Ô@Ø Ñ"�Øô �J‰J”uÔÜ 
Ó+ˆKÙÜ˜	 ) ¨A¨e°C°R¨j¨\¸Ð=Ô>Øä.:¸;È
Ó.SÑ+ˆL˜( GÙØ$�áØ$�ô —{‘{ :×#4Ñ#4Ó#6Ó7×AÑAÓCÀCÀRÐHˆHÜ˜H bÓ)¨IÑ5ˆIá'.�g˜d˜s‘m°BˆGð �L‰Lð6à˜I |°W¸hÈ
ÐT^Ð`gÐhôð �L‰LØiØ˜L¨)°WÐ=ôð �K‰KŒMØ˜‰NˆIá8?˜r '¨#¨2 , ¨tÑ4ÀRˆNÜ�H˜X˜J a¨°S°bÐ(9Ð':¸#¸nÐ=MÐNÖOðk5	Pðn 	�‰	ˆÜ�
‰
”5Ôðm ”7Ò'˜iÔ
(ðp 	‡J�J„LÜ	Ð˜i˜[Ð(<¸]¸OÐK\Ð]aÐbcÑ]cÐ\dÐdhÐ
iÔjØÐr   Ú__main__)é$   )(Ú__doc__r-   r:   Újsonr)   ÚosÚsocketrh   r   r   Úurllib.requestr   r   Úurllib.errorr   r   Ú_SYSrf   ÚargvÚisdigitrl   rd   r,   Ústrr@   re   r$   r%   ÚMAX_RETRIESr+   r   Úgetenvr   r   r   r6   rJ   rW   r‚   Ú__name__© r   r   ú<module>r“      sé   ðñ÷ 0× /× /Ó /ß (ß +ß ,ã Ù" 4§9¡9›~°Ò1°d·i±iÀ±l×6JÑ6JÔ6L‰#ˆd�i‰i˜‰lÔ
ÐRV€Ø
ÐÙ	Ð
¡# g£,Ñ
.Ô/à&€Ø\€ØÐWÐ
X€Ø
€Ø€Ø€ó Ø
ˆ"�)‰)�KÐ!2Ó
3€òò
ð& #ó òò+(óZjðX ˆzÒÙ…Kð r   