Ë
    ¸¬uji/  ã                   óÞ  — d Z ddlZddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlmZmZ ddlZdZ ej                  dd«      ZdZd	d
dddddœdddddddœdddddddœdddddddœdddd dddœgZ ej$                  «        ed!¬"«      z
  j'                  d#«      Z ej*                  «       Zej.                  j1                  d$d%i«       d&„ Zd.d'„Zd(„ Zd)„ Zd*„ Zd/d+„Zd,„ Ze d-k(  r e«        yy)0uÊ   
Crawl é�’å²›å¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€ (mbee.qingdao.gov.cn) - å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·å…¬ç¤º + æ”¿åŠ¡æœ�åŠ¡
ASP.NET WebForms + UpdatePanel pagination.
All data written to /root/search.db gov_raw table.
é    N)ÚBeautifulSoup)ÚdatetimeÚ	timedeltazhttp://mbee.qingdao.gov.cn:8082Ú	SEARCH_DBz/root/search.dbu   é�’å²›å¸‚ç”Ÿæ€�çŽ¯å¢ƒå±€u   å®¡æ‰¹å�—ç�†æƒ…å†µå…¬ç¤ºz/m2/ZWGKNew/webgs/list1.aspxz/m2/webgs/list1detaile.aspxie  u   å�—ç�†æ—¶é—´)ÚnameÚ	list_pathÚdetail_prefixÚmÚ	max_pagesÚ
date_fieldu   æ‹Ÿå®¡æ‰¹å‡†æƒ…å†µå…¬ç¤ºz/m2/ZWGKNew/webgs/list2.aspxz/m2/webgs/list2detaile.aspxif  u   æ‹Ÿå®¡æ‰¹æ—¶é—´u   å®¡æ‰¹å†³å®šå…¬ç¤ºz/m2/ZWGKNew/webgs/list3.aspxz/m2/webgs/list3detaile.aspxig  u   å�‘æ–‡æ—¶é—´u   å‘ŠçŸ¥æ‰¿è¯ºå®¡æ‰¹å†³å®šå…¬ç¤ºz"/m2/ZWGKNew/webgs/gzcnsgslist.aspxz/m2/webgs/gzcnsgsdetaile.aspxiÙ  u   éªŒæ”¶å†³å®šå…¬ç¤ºz/m2/ZWGKNew/webys/list3.aspxz/m2/webys/list3detaile.aspxij  iG  )Údaysz%Y-%m-%dz
User-AgentzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36c                 ó¤   — i }dD ]H  }t        j                  d|z   dz   | «      }|sŒ"t        j                  |j	                  d«      «      ||<   ŒJ |S )zDExtract ASP.NET __VIEWSTATE, __VIEWSTATEGENERATOR, __EVENTVALIDATION)Ú__VIEWSTATEÚ__VIEWSTATEGENERATORÚ__EVENTVALIDATIONzid="z"[^>]*value="([^"]*)"é   )ÚreÚsearchÚhtml_modÚunescapeÚgroup)Ú	html_textÚresultr   r
   s       ú'/root/gov_crawler/crawl_qingdao_mbee.pyÚextract_viewstater   A   sV   € à€FØLò 9ˆÜ�I‰I�g ‘nÐ'?Ñ?ÀÓKˆÚÜ#×,Ñ,¨Q¯W©W°Q«ZÓ8ˆF�4ŠLð9ð €Mó    c                 ó   — |dk(  s|€2t         j                  | d¬«      }|j                  }t        |«      }||fS i |¥dt	        |«      ddœ¥}t         j                  | |d¬«      }|j                  }t        |«      }||fS )zoFetch a page, returning (html, new_viewstate). 
    For page 1, does GET. For page 2+, POSTs with __doPostBack.r   é   ©Útimeoutz!ctl00$right_content$Pagination$pgÚ )Ú__EVENTTARGETÚ__EVENTARGUMENTÚ__LASTFOCUS)Údatar    )ÚsessionÚgetÚtextr   ÚstrÚpost)ÚurlÚpage_numÚ	viewstateÚrespÚhtmlÚnew_vsÚ	post_datas          r   Ú
fetch_pager2   J   s“   € ð �1‚}˜	Ð)Ü�{‰{˜3¨ˆ{Ó+ˆØ�y‰yˆÜ" 4Ó(ˆØ�Vˆ|ÐðØ
ðà<Ü˜x›=Øò	€Iô �<‰<˜ )°Rˆ<Ó8€DØ�9‰9€DÜ˜tÓ$€FØ�ˆ<Ðr   c           	      ó¦  — g }t        | d«      }|j                  dd¬«      }|r|j                  d«      }n|j                  dd¬«      }|s|S |j                  d«      }|D �]m  }|j                  d	«      }t        |«      d
k  rŒ$|d   r|d   j                  d«      nd}	|	sŒB|	j	                  d¬«      }
|	j                  dd«      }|r|
sŒk|j                  d«      r
t        |z   }n,|j                  d«      st        dz   |j                  d«      z   }d}t        |«      D ]O  }|j	                  d¬«      }t        j                  d|«      }|sŒ.|j                  d«      j                  dd«      } n g }|dd D ](  }|j	                  d¬«      }|sŒ|j                  |«       Œ* dj                  |«      }|j                  |
||||d   dœ«       �Œp |S )zGParse list page and return list of dicts with title, url, date, summaryúhtml.parserÚdivÚright_content_UpdatePanel1)ÚidÚtablez
table-list©Úclass_ÚtrÚtdé   r   ÚaNT©ÚstripÚhrefr!   ú/Úhttpz(\d{4}[-/]\d{1,2}[-/]\d{1,2})r   ú-éÿÿÿÿz | r   )Útitler+   ÚdateÚsummaryÚsection)r   ÚfindÚfind_allÚlenÚget_textr'   Ú
startswithÚBASE_URLÚlstripÚreversedr   r   r   ÚreplaceÚappendÚjoin)r   rI   ÚrecordsÚsoupÚupdatepanelr8   ÚrowsÚrowÚcellsÚlinkrF   rA   Údate_strÚcellÚtxtÚdmÚsummary_partsrH   s                     r   Úparse_list_pagera   _   sç  € à€GÜ˜ MÓ2€Dð —)‘)˜EÐ&B�)ÓC€KÙØ× Ñ  Ó)‰à—	‘	˜'¨,�	Ó7ˆáØˆà�>‰>˜$Ó€DØó ,ˆØ—‘˜TÓ"ˆÜˆu‹:˜Š>Øð &+¨1¢Xˆu�Q‰x�}‰}˜SÔ!°4ˆÙØà—‘ D�Ó)ˆØ�x‰x˜ Ó#ˆÙ™5Øð �?‰?˜3ÔÜ˜d‘?‰DØ—‘ Ô(Ü˜c‘> D§K¡K°Ó$4Ñ4ˆDð ˆÜ˜U“Oò 	ˆDØ—-‘- d�-Ó+ˆCÜ—‘Ð;¸SÓAˆBÚØŸ8™8 A›;×.Ñ.¨s°CÓ8�Ùð	ð ˆØ˜!˜B�Kò 	*ˆDØ—-‘- d�-Ó+ˆCÚØ×$Ñ$ SÕ)ð	*ð —*‘*˜]Ó+ˆà�‰ØØØØØ˜v‘ñ
ö 	ðM,ð\ €Nr   c                 ód  — 	 t         j                  | d¬«      }d|_        t	        |j
                  d«      }|j                  dd¬	«      }|s|j                  dd
¬	«      }d}d}d}|r±t        |«      }|j                  dd„ ¬«      }|r%|j                  d«      }	|	r|	j                  d¬«      }t        j                  d|j                  «       «      }
|
rE|
j                  d«      }|j                  dd«      j                  dd«      j                  dd«      }|}|||fS # t        $ r
}Y d}~yd}~ww xY w)z>Fetch detail page, return (content_html, date_str, full_title)r   r   zutf-8N)r!   r!   r!   r4   r5   z
detail-boxr9   ztable-detailr!   r<   c                 ó   — | xr d| v S )Nu   é¡¹ç›®å��ç§°© )Úts    r   ú<lambda>zfetch_detail.<locals>.<lambda>·   s   € ¸Ò8Q¸nÐPQÐ>Q€ r   )ÚstringTr?   un   (?:å�—ç�†æ—¶é—´|å�‘æ–‡æ—¶é—´|å…¬ç¤ºæ—¶é—´|æ‹Ÿå®¡æ‰¹æ—¶é—´|å®¡æ‰¹æ—¶é—´)[ï¼š:]\s*(\d{4}å¹´\d{1,2}æœˆ\d{1,2}æ—¥)r   u   å¹´rD   u   æœˆu   æ—¥)r&   r'   ÚencodingÚ	Exceptionr   r(   rJ   r)   Úfind_next_siblingrM   r   r   r   rR   )r+   r.   ÚerV   ÚdetailÚcontent_htmlr\   Ú
full_titleÚ
name_labelÚ	name_cellÚ
date_matchÚdts               r   Úfetch_detailrs   Ÿ   sL  € ðÜ�{‰{˜3¨ˆ{Ó+ˆØˆŒô ˜Ÿ™ MÓ2€Dð �Y‰Y�u \ˆYÓ2€FÙØ—‘˜5¨�Ó8ˆà€LØ€HØ€Jáä˜6“{ˆð —[‘[ Ñ.Q�[ÓRˆ
ÙØ"×4Ñ4°TÓ:ˆIÙØ&×/Ñ/°dÐ/Ó;�
ô —Y‘Yð   Qð  SY÷  Sbñ  Sbó  Sdó  eˆ
ÙØ×!Ñ! !Ó$ˆBØ—‘˜E 3Ó'×/Ñ/°°sÓ;×CÑCÀEÈ2ÓNˆBØˆHà˜ :Ð-Ð-øôA ò Üûðús   ‚D Ä	D/Ä*D/c                 óÐ  — |j                  «       }d}d}| D ]–  }|d   r|d   t        k  r|dz  }Œ	 |j                  d|d   d|d   |j                  dd	«      |j                  d
d	«      |j                  dd	«      t        › d|j                  dd	«      › �f«       |j
                  dkD  r|dz  }Œ˜ |j                  «        ||fS # t        $ r}t        d|› �«       Y d}~ŒÉd}~ww xY w)z!Insert records into gov_raw tabler   rG   r   zÂ
                INSERT OR IGNORE INTO gov_raw 
                (page_url, source_url, title, summary, content, publish_date, site_name)
                VALUES (?, ?, ?, ?, ?, ?, ?)
            r+   zhttp://mbee.qingdao.gov.cnrF   rH   r!   Úcontentz - rI   z  DB error: N)	ÚcursorÚTHREE_YEARS_AGOÚexecuter'   Ú	SITE_NAMEÚrowcountri   ÚprintÚcommit)rU   Úconnrv   ÚinsertedÚskippedÚrecrk   s          r   Úinsert_to_dbr�   Ç   s  € à�[‰[‹]€FØ€HØ€Gàò &ˆàˆvŠ;˜3˜v™;¬Ò8Ø�q‰LˆGØð	&Ø�N‰Nð ð
 �E‘
Ø,Ø�G‘Ø—‘˜	 2Ó&Ø—‘˜	 2Ó&Ø—‘˜ Ó#Ü�+˜S §¡¨°BÓ!7Ð 8Ð9ðô	ð �‰ Ò"Ø˜A‘�øð+&ð2 	‡K�K„MØ�WÐÐøô	 ò 	&Ü�L  Ð$×%Ñ%ûð	&ús   ²A<CÃ	C%ÃC Ã C%c                 ó–  — | d   }t         › | d   › d| d   › �}t        d|› d�«       t        d|› �«       g }d}d	}d
}|rd}	n|dkD  r|}	n| d   dkD  r| d   }	nd}	|�r¯||	k  �r©t        d|› d�dd
¬«       	 t        |||«      \  }
}d|
v rt        d«       �nut        |
| «      }|st        d«       �nZt        t	        |«      › d�dd
¬«       |j                  |«       t        |
d«      }|j                  dd¬«      }|rß|j                  «       }t        j                  d|«      }t        j                  d|«      }|rt        |j                  d«      «      n|}t        j                  d|«      }|r#t        d|j                  d«      › d|› d �d¬!«       | d   dk(  r|dk(  r|dk  r|}	|r't        |j                  d«      «      }||k  xr ||	k  }n||k  xr ||	k  }nd"}t        «        |dz  }t        j                  d#«       |r||	k  r�Œ©t        d%t	        |«      › d&�«       |S # t        $ r}t        d$|› �«       Y d	}~Œ6d	}~ww xY w)'z.Crawl a single section and return record countr   r   z?m=r
   z
=== z ===zURL: r   NTr   r   é   z  Page z...ú )ÚendÚflushzObject movedz*ERROR: page redirected (invalid viewstate)zno records foundz recordsr!   r4   r5   Ú
paginationr9   u   å…±\s*(\d+)\s*é¡µu   ç¬¬\s*(\d+)\s*é¡µu   å…±\s*(\d+)\s*æ�¡z / u   æ�¡ total, u   é¡µ)r…   Fg333333Ó?zERROR: z	  Total: z records collected)rO   r{   r2   ra   rL   Úextendr   rJ   rM   r   r   Úintr   ÚtimeÚsleepri   )rI   r   ÚincrementalÚsection_nameÚlist_urlÚall_recordsÚpager-   Úhas_moreÚpages_to_crawlr   rU   rV   ÚpgÚpg_textÚtotal_pages_matchÚcurrent_page_matchÚtotal_pagesÚrecords_countÚcurrentrk   s                        r   Úcrawl_sectionrš   é   sš  € à˜6‘?€LÜ�˜G KÑ0Ð1°°W¸S±\°NÐC€Hä	ˆF�<�. Ð
%Ô&Ü	ˆE�(�Ð
Ôà€KØ€DØ€IØ€Hñ Ø‰Ø	�QŠØ"‰Ø	�Ñ	 Ò	!Ø  Ñ-‰àˆâ
�t˜~Ó-Ü�˜�v˜SÐ! s°$Õ7ð8	Ü#-¨h¸¸iÓ#HÑ ˆI�yð  Ñ*ÜÐBÔCÙä% i°Ó9ˆGáÜÐ(Ô)Ùä”S˜“\�N (Ð+°¸4Õ@Ø×Ñ˜wÔ'ô ! ¨MÓ:ˆDØ—‘˜5¨�Ó6ˆBÙØŸ+™+›-�Ü$&§I¡IÐ.BÀGÓ$LÐ!Ü%'§Y¡YÐ/CÀWÓ%MÐ"áARœcÐ"3×"9Ñ"9¸!Ó"<Ô=ÐX\�ô !#§	¡	Ð*>ÀÓ H�Ù Ü˜C × 3Ñ 3°AÓ 6Ð7°{À;À-ÈsÐSÐY[Õ\ð ˜;Ñ'¨1Ò,°¸a²à" Q’Ø)4˜ñ &Ü!Ð"4×":Ñ":¸1Ó"=Ó>�GØ&¨Ñ4ÒQ¸À>Ñ9Q‘Hà# kÑ1ÒK°d¸^Ñ6K‘Hà �äŒGØ�A‰IˆDô �J‰J�sŒOño �t˜~Ô-ôz 
ˆI”c˜+Ó&Ð'Ð'9Ð
:Ô;ØÐøô ò 	Ü�G˜A˜3�-Ô Üûð	ús%   Á7H' ÂH' Â3EH' È'	IÈ0IÉIc            
      óâ  — dt         j                  v xs t         j                  d   dk(  } dd l}|j                  t        «      }d}d}t
        D ]è  }t        || ¬«      }|t        |«      z  }|sŒ!t        d«       t        |«      D ]Š  \  }}|dkD  r$|dz  dk(  rt        d|› d	t        |«      › �d
¬«       t        |d   «      \  }	}
}|	r|	|d<   |r,t        |«      t        |j                  dd«      «      kD  r||d<   |
r|d   s|
|d<   Œƒ|
sŒ†|
|d<   ŒŒ t        ||«      \  }}t        d|› d|› �«       ||z  }Œê |j                  «        t        d«       t        d|› �«       t        d|› �«       y )NrŒ   rE   Ú1r   )rŒ   z  Fetching detail pages...é
   z    rB   T)r†   r+   ru   rF   r!   rG   z  Inserted: z, Skipped (old): z
=== SUMMARY ===zTotal records collected: zTotal inserted: )ÚsysÚargvÚsqlite3ÚconnectÚDB_PATHÚSECTIONSrš   rL   r{   Ú	enumeraters   r'   r�   Úclose)rŒ   r    r}   Útotal_insertedÚtotal_recordsrI   rU   Úir€   ru   Údate_from_detailrn   r~   r   s                 r   Úmainrª   @  sŠ  € Ø¤3§8¡8Ð+ÒB¬s¯x©x¸©|¸sÑ/B€KãØ�?‰?œ7Ó#€Dà€NØ€Mäò #ˆÜ °[ÔAˆØœ˜W›Ñ%ˆáØô 	Ð*Ô,Ü Ó(ò 	/‰FˆAˆsØ�1Šu˜˜R™ 1šÜ˜˜Q˜C˜q¤ W£ Ð/°tÕ<ä4@ÀÀUÁÓ4LÑ1ˆGÐ% zÙØ!(��I‘Ùœc *›o´°C·G±G¸GÀRÓ4HÓ0IÒIØ)��G‘Ù¨¨FªØ.��F’Ú!à.��F’ð	/ô  )¨°$Ó7Ñˆ�'Ü�˜X˜JÐ&7¸°yÐAÔBØ˜(Ñ"‰ð7#ð: 	‡J�J„Lä	ÐÔÜ	Ð% m _Ð
5Ô6Ü	Ð˜^Ð,Ð
-Õ.r   Ú__main__)r   N)r   F)!Ú__doc__rž   r   ÚjsonrŠ   Úosr/   r   ÚrequestsÚbs4r   r   r   rO   Úgetenvr¢   ry   r£   ÚnowÚstrftimerw   ÚSessionr&   ÚheadersÚupdater   r2   ra   rs   r�   rš   rª   Ú__name__rd   r   r   ú<module>r¸      sO  ðñ÷ 1× 0× 0Û Ý ß (Û 	à,€Ø
ˆ"�)‰)�KÐ!2Ó
3€Ø&€	ð
 +Ø3Ø6Ø˜qØ$ñð +Ø3Ø6Ø˜qØ'ñð %Ø3Ø6Ø˜qØ$ñð 1Ø9Ø8Ø˜qØ$ñð %Ø3Ø6Ø˜qØ$ñð;$€ðN  �8—<‘<“>¡I°5Ô$9Ñ9×CÑCÀJÓO€à
ˆ(×
Ñ
Ó
€Ø ‡�× Ñ Øð  Dðô òóò*>ò@%.òP óDUòn*/ðX ˆzÒÙ…Fð r   