Ë
    Ç#™j§  ã                   ó–  — d Z ddlZddlZddlZddlZddlZddlmZmZmZ ddl	Z	ddl
Z
ddlZ e
j                  e
j                  j                  «       ddlmZ dZdZ ej$                  dd«      Zd	Z ej*                  ej,                  «       ed
¬«      z
  j/                  d«      ZddddœZddddddddddddœZd„ Zd„ Zd„ Zd „ Zd.d!„Ze d"k(  rzd#ejB                  v Z"e"rd$nd%Z# e$«         e$d&e› d'e#› �«        ej                  «       Z% ee"¬(«      \  Z&Z' e$d) e( ej                  «       e%z
  d*«      › d+�«        e$d,e&› d-e'› �«       yy)/u·  
å®šè¿œåŽ¿-å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»· (www.dingyuan.gov.cn)
============================================
CMS: æ»�å·žå¸‚ç»Ÿä¸€ä¿¡æ�¯å…¬å¼€å¹³å�° (AJAX API)
åˆ—è¡¨API: /chuzhou/site/label/8888?labelName=publicInfoList&pageIndex=N
è¯¦æƒ…: /public/161054677/{id}.html, <div class="xxgk_contnet"> å�«æ­£æ–‡

ç”¨æ³•:
    python3 crawl_dingyuan_hjbh.py          # å…¨é‡�(æœ€å¤š4é¡µ)
    python3 crawl_dingyuan_hjbh.py --test   # æµ‹è¯• 5 æ�¡
é    N)ÚdatetimeÚ	timedeltaÚtimezone)ÚBeautifulSoupu(   å®šè¿œåŽ¿-å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·z3https://www.dingyuan.gov.cn/chuzhou/site/label/8888Ú	SEARCH_DBz/root/search.dbé   iG  )Údaysz%Y-%m-%dz<Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36ÚXMLHttpRequestzVhttps://www.dingyuan.gov.cn/public/column/161054677?type=4&catId=170070671&action=list)z
User-AgentzX-Requested-WithÚRefererÚpublicInfoListÚ2653861Ú20Ú1ÚlistÚtruez
yyyy-MM-ddÚ80Ú	161054677Ú	170070671Ú4)Ú	labelNameÚsiteIdÚpageSizeÚ	pageIndexÚactionÚisDateÚ
dateFormatÚlengthÚorganIdÚcatIdÚtypec           	      ó  — t        t        «      }t        | «      |d<   	 t        j                  t
        |t        dd¬«      }d|_        |j                  S # t        $ r'}t        d| › dt        |«      d d › �«       Y d }~y d }~ww xY w)	Nr   é   F)ÚparamsÚheadersÚtimeoutÚverifyúutf-8z  ! list page z fail: é<   )ÚdictÚ
API_PARAMSÚstrÚrequestsÚgetÚAPI_URLÚHEADERSÚencodingÚtextÚ	ExceptionÚprint)Úpager#   ÚrÚes       ú(/root/gov_crawler/crawl_dingyuan_hjbh.pyÚ
fetch_listr8   .   sx   € Ü”*Ó€FÜ˜d›)€Fˆ;ÑðÜ�L‰Lœ¨¼È"ÐUZÔ[ˆØˆŒ
Ø�v‰vˆøÜò Ü�˜t˜f G¬C°«F°3°B¨K¨=Ð9Ô:Üûðús   Ÿ4A Á	BÁA?Á?Bc                 ó¨  — g }t        j                  d| t         j                  «      D ]§  }|j                  d«      }t        j                  dd|j                  d«      «      j                  «       }|j                  d«      j                  «       }|j                  d«      s|j                  d«      rd	|z   nd
|z   }|j                  |||dœ«       Œ© |S )NzT<li class="mc">\s*<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)</a>.*?<li class="rq">([^<]+)é   z\s+ú é   é   Úhttpú/úhttps://www.dingyuan.gov.cnzhttps://www.dingyuan.gov.cn/)ÚtitleÚurlÚpub_date)ÚreÚfinditerÚDOTALLÚgroupÚsubÚstripÚ
startswithÚappend)ÚhtmlÚitemsÚmrB   rA   rC   s         r7   Ú
parse_listrO   9   sº   € Ø€EÜ�[‰[Ø_ØŒb�i‰ióò 	Iˆð �g‰g�a‹jˆÜ—‘�v˜s A§G¡G¨A£JÓ/×5Ñ5Ó7ˆØ—7‘7˜1“:×#Ñ#Ó%ˆØ�~‰~˜fÔ%Ø9<¿¹ÈÔ9LÐ/°#Ò5ÐRpÐsvÑRvˆCØ�‰˜u¨S¸hÑGÕHð	Ið €Ló    c                 óª  — 	 t        j                  | t        dd¬«      }d|_        |j                  }t        |d«      }|j                  dd„ ¬«      }|s|j                  dd	„ ¬«      }|s(|j                  dd
¬«      xs |j                  dd
¬«      }|s<|j                  dd¬«      }|sy|j                  d«      D ]  }|j                  «        Œ |j                  d«      D ]  }|j                  «        Œ | d | j                  d«      dz    }|j                  dd¬«      D ]d  }|d   j                  «       }|j                  d«      r	d|z   |d<   Œ0|j                  d«      s|j                  d«      rŒSddlm}	  |	||«      |d<   Œf |j                  dd¬«      D ]/  }
|
d   j                  «       }|j                  d«      sŒ(d|z   |
d<   Œ1 t        |«      }t!        j"                  dd|t         j$                  ¬«      }t!        j"                  d d!|«      }|j                  «       S # t&        $ r$}t)        d"t        |«      d d# › �«       Y d }~yd }~ww xY w)$Nr"   F)r$   r%   r&   r'   zhtml.parserÚdivc                 ó6   — | xr t        | t        «      xr d| v S )NÚgkwz_contnet©Ú
isinstancer+   ©Úcs    r7   ú<lambda>zfetch_detail.<locals>.<lambda>N   s   € ¨aÒ.^´J¸qÄ#Ó4FÒ.^È>Ð]^ÐK^€ rP   )Úclass_c                 ó6   — | xr t        | t        «      xr d| v S )NÚxxgkcontentrU   rW   s    r7   rY   zfetch_detail.<locals>.<lambda>P   s   € °!Ò2a¼
À1ÄcÓ8JÒ2aÈ}Ð`aÐOa€ rP   Ú
contentbox)ÚidÚxxgk_contnetÚ zx.xxgk_newtit, .wzewm, .newsinfo, .clear, .ls-article-menu, [class*=share], [class*=print], [class*=ewm], [class*=qrcode]zVscript, style, .xxgk_zclist, .wzewm, [class*=qrcode], [class*=share], .ls-article-menur?   r:   ÚaT)Úhrefrb   r@   )z./z../)r>   Ú
javascriptÚmailtoú#r   )ÚurljoinÚimg)Úsrcrh   z
<!--.*?-->)Úflagsz\s{2,}r;   z  ! detail fail: r(   )r,   r-   r/   r0   r1   r   ÚfindÚselectÚ	decomposeÚrfindÚfind_allrI   rJ   Úurllib.parserf   r+   rD   rH   ÚSr2   r3   )rB   r5   rL   ÚsoupÚdÚelÚbasera   Úhrf   rg   ÚsÚoutr6   s                 r7   Úfetch_detailrx   G   s3  € ð(Ü�L‰L˜¤g°rÀ%ÔHˆØˆŒ
Ø�v‰vˆÜ˜T =Ó1ˆà�I‰I�eÑ$^ˆIÓ_ˆÙØ—	‘	˜%Ñ(a�	ÓbˆAÙØ—	‘	˜% L�	Ó1ÒZ°T·Y±Y¸uÈ\°YÓ5ZˆAÙà—	‘	˜%¨�	Ó7ˆAÙØØ—h‘hð   Zó  [ò �Ø—‘•ðð —(‘(ÐsÓtò 	ˆBØ�L‰L�Nð	ð Ð'�S—Y‘Y˜s“^ aÑ'Ð(ˆØ—‘˜C d�Ó+ò 	-ˆAØ�&‘	—‘Ó!ˆAØ�|‰|˜CÔ Ø9¸AÑ=��&’	Ø—‘˜mÔ,°Q·\±\ÐBgÕ5hÝ0Ù# D¨!Ó,��&’	ð	-ð —:‘:˜e¨�:Ó.ò 	?ˆCØ�E‘
× Ñ Ó"ˆAØ�|‰|˜CÕ Ø:¸QÑ>��E’
ð	?ô �!‹fˆÜ�f‰f�] B¨´2·4±4Ô8ˆÜ�f‰f�Y  SÓ)ˆØ�y‰y‹{ÐøÜò ÜÐ!¤# a£&¨¨" + Ð/Ô0Üûðús-   ‚B'H% Â*CH% Å3AH% ÇA"H% È%	IÈ.IÉIc                 óJ  — | sydd l }|j                  t        «      }|j                  d«       |j                  d«       d\  }}| D ]†  }	 |j                  dt        |j                  d«      xs dd d |j                  d	d«      |j                  d
d«      |j                  d«      xs dd d ddf«       |j                  dkD  r|dz  }n|dz  }Œˆ |j                  dt        f«       |j                  «        |j                  «        ||fS # t        $ r}|dz  }Y d }~Œ×d }~ww xY w)N©r   r   r   zPRAGMA journal_mode=WALzPRAGMA synchronous=NORMALzxINSERT OR REPLACE INTO gov_raw (site_name, title, page_url, content, publish_date, summary, tags) VALUES (?,?,?,?,?,?,?)rA   r`   iô  rB   ÚcontentrC   é
   u   å»ºè®¾é¡¹ç›®çŽ¯å¢ƒå½±å“�è¯„ä»·r:   z·INSERT INTO gov_search(rowid, title, site_name, summary) SELECT r.id, r.title, r.site_name, r.summary FROM gov_raw r WHERE r.id NOT IN (SELECT rowid FROM gov_search) AND r.site_name=?)
Úsqlite3Úconnectr   ÚexecuteÚ	SITE_NAMEr-   Útotal_changesr2   ÚcommitÚclose)rM   r}   ÚdbÚokÚfailÚitr6   s          r7   Úto_dbrˆ   r   s2  € ÙØÛØ	�‰œÓ	#€BØ‡J�JÐ(Ô)Ø‡J�JÐ*Ô+Ø�H€BˆØò ˆð	Ø�J‰Jð)ô Ø—V‘V˜G“_Ò*¨¨D¨SÐ1Ø—F‘F˜5 "Ó%Ø—F‘F˜9 bÓ)Ø—V‘V˜JÓ'Ò-¨2¨s°Ð3ØØ4ðô	ð ×Ñ !Ò#Ø�a‘‘à˜‘	�øð'ð, ‡J�Jð	\ô 
ˆô	ð ‡I�I„KØ‡H�H„JØˆtˆ8€Oøô ò 	Ø�A‰I�Dûð	ús   Á
BD
Ä
	D"ÄDÄD"c                 ó(  — g }t        «       }t        dt        dz   «      D ]¾  }t        |«      }|s n¯t	        |«      }|st        d|› d�«        n‘d}|D ]F  }|d   |v rŒ|j                  |d   «       |d   r|d   t        k  rŒ1|j                  |«       |dz  }ŒH |r|d   d   nd}|r|d   d   nd}	t        d|› d	t        |«      › d
|› d|	› d|› �
«       |dk(  sŒ¾ n t        dt        |«      › d�«       | r|d d }t        dt        |«      › d�«       |syt        |«      D ]Y  \  }
}t        d|
dz   › dt        |«      › d	|d   d d › d�dd¬«       t        |d   «      }||d<   t        t        |«      › d�«       Œ[ t        |«      \  }}||fS )Nr:   z  [Page z] empty, reached endr   rB   rC   ú?éÿÿÿÿú] z items (z ~ z), new: z	  Total: z items within 3yré   z  TEST mode: z itemsrz   z  [r?   rA   é(   z... r`   T)ÚendÚflushr{   ÚB)ÚsetÚrangeÚ	MAX_PAGESr8   rO   r3   ÚaddÚCUTOFFrK   ÚlenÚ	enumeraterx   rˆ   )ÚtestÚ	all_itemsÚ	seen_urlsr4   rL   rM   ÚnewÚitemÚfÚlÚir{   r…   r†   s                 r7   Úcrawlr¡   š   së  € Ø€IÜ“€IÜ�aœ Q™Ó'ò ˆÜ˜$ÓˆÙÙÜ˜4Ó ˆÙÜ�H˜T˜FÐ"6Ð7Ô8ÙØˆØò 	ˆDØ�E‰{˜iÑ'ØØ�M‰M˜$˜u™+Ô&Ø�JÒ D¨Ñ$4´vÒ$=ØØ×Ñ˜TÔ"Ø�1‰H‰Cð	ñ %*ˆE�!‰H�ZÒ ¨sˆÙ%*ˆE�"‰I�jÒ!°ˆÜ�˜˜˜b¤ U£ ¨H°Q°C°s¸1¸#¸XÀcÀUÐKÔLØ�!‹8Ùð+ô, 
ˆI”c˜)“nÐ%Ð%6Ð
7Ô8ÙØ˜b˜q�Mˆ	Ü�œc )›nÐ-¨VÐ4Ô5ÙØÜ˜YÓ'ò "‰ˆˆ4Ü��A�a‘C�5˜œ#˜i›.Ð)¨¨D°©M¸#¸2Ð,>Ð+?¸tÐDÈ"ÐTXÕYÜ˜t E™{Ó+ˆØ!ˆˆY‰Ü”�W“�˜aÐ Õ!ð	"ô
 �YÓ�H€BˆØˆtˆ8€OrP   Ú__main__z--testÚTESTÚFULLú[rŒ   )r™   z  Time: r:   rv   z  New: z Skip: )F))Ú__doc__rD   ÚsysÚosÚtimeÚjsonr   r   r   r,   Úurllib3Údisable_warningsÚ
exceptionsÚInsecureRequestWarningÚbs4r   r€   r.   Úgetenvr   r”   ÚnowÚutcÚstrftimer–   r/   r*   r8   rO   rx   rˆ   r¡   Ú__name__Úargvr™   Úmoder3   Út0r…   r†   Úround© rP   r7   ú<module>rº      sv  ðñ
÷ × Ó ß 2Ñ 2ß Û 	Ø €× Ñ ˜×+Ñ+×BÑBÔ CÝ à7€	ØB€ØˆB�I‰I�kÐ#4Ó5€	Ø€	Øˆh�l‰l˜8Ÿ<™<Ó(©9¸%Ô+@Ñ@×JÑJÈ:ÓV€àPØ(Øgñ€ð "ØØØØØØØØØØñ€
ò	òò)òV&óP%ðN ˆzÒØ�s—x‘xÐ€DÙ‰6˜v€DÙ	„GÙ	ˆAˆiˆ[˜˜4˜&Ð
!Ô"Ø	ˆ�‰‹€BÙ˜$Ô�H€BˆÙ	ˆH‘U˜9˜4Ÿ9™9›; r™>¨1Ó-Ð.¨aÐ
0Ô1Ù	ˆG�B�4�w˜t˜fÐ
%Õ&ð rP   