jBdZddlZddlZddlZddlZddlZej jdej jej je ddl m Z dZ e dzZddiZdd d d d Zdd ZdZdZedk(reZededyy)u crawl_ninghai_hpgs.py - 宁海县生态环境专题子栏目 (JPAAS 平台) 支持 --col 参数: 1229116620=环评批前公示, 1229116621=环评审批结果公示 列表: GET /api-gateway/jpaas-publish-server/front/page/build/unit webId=3575&pageId=&parseType=bulidstatic&pageType=column&tagId=栏目列表&tplSetId=Kj4AQALilei1D88cYSuZt&editType=null paramJson={"pageNo":N,"pageSize":"15"} 分页 详情: /col/col/art/YYYY/art_xxx.html, meta ArticleTitle/PubDate/ContentSource, 正文 id=zoom N)push_to_searchdbzhttp://www.ninghai.gov.cnz7/api-gateway/jpaas-publish-server/front/page/build/unitz User-AgentzIMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0u宁海县-环评批前公示u"宁海县-环评审批结果公示u宁海县-信息公示u宁海县-环评受理公示) 1229116620 1229116621 1229116613 1229116619c Vd|dddddtj|t|dd d } tjt |t d }d |_|j}|j ds!td|j dgdfS|j dxsij ddxsd}tjd|}|rt|jdnd}g} tjd|zdz} | j|D]Y} | jt | jdz| jdj#| jdd[| |fS#t$$r} td| gdfcYd} ~ Sd} ~ wwxYw)N3575 bulidstaticcolumnu 栏目列表Kj4AQALilei1D88cYSuZtnull)pageNopageSizeF) ensure_ascii)webIdpageId parseTypepageTypetagIdtplSetIdeditType paramJson)paramsheaderstimeoututf-8successz [API FAIL] messagerdatahtmlz count="(\d+)"z]+href="(/col/colzY/art/\d+/art_[a-f0-9]+\.html)"[^>]*>([^<]{10,200})\s*(\d{4}-\d{2}-\d{2}))urltitledatez [API ERR] )jsondumpsstrrequestsgetLIST_APIHEADERSencodingprintresearchintgroupcompilefinditerappendBASEstrip Exception) colpage_numrowsqrdr!mtotalitemspatm2es '/root/gov_crawler/crawl_ninghai_hpgs.py fetch_pagerIs"+ZZ8T KZ_` A LL!Wb I FFHuuY M!%% "2!34 5q5Lf #((4: II& -#$AGGAJ!jj2S8O>O>Q[][c[cde[fg h ie|  QC !1u s%A)FC*F F( F#F(#F(c tj|td}d|_|j }d}tjd|}|r|jdj}d}tjd |}|r9tjd |jd}|r|jnd}d}tjd |}|r|jdj}d} |jd } | d kDr|jd| dz} d} | } | t|krJ| d kDrE|| | dzdk(r | dz } | dz } n|| | dzdk(r | dz} | dz } n| dz } | t|kr| d kDrE|| | dz j} tjddtzdz| } tjddtzdz| } |||| fS#t$r}t d|Yd}~yd}~wwxYw)Nr)rrrz [FETCH ERR] )r"r"r"r"r"z/zzhref="/zhref="/zsrc="/zsrc=")r,r-r/r0r;r1textr2r3r5r:findlensubr9)r&r@rGcr'rBpub_datedmsrccontentidxstartdepthposs rH fetch_detailr\<s LLgr :  A E DaHA   "H ?CA YY+QWWQZ 8!#288: C EqIAggaj G && C QwsC 1$CFluqySU|v% q3s1u) qqCFluqyE#a%.&&(&&X_s%:GD&&GdNS$8'B (C ((K  qc"#s#G G>&G99G>c Nddl}|j}|jdd|jdtd|jdd |j }|j }t j|d |}td |d |dg}t}t|d\}}td|d|jr |dzdz} n |j} td| dzD]h} | dkDrt|| \} } n|} td| dt| d| D]/} | d|vs |j| d|j!| 1jg}t#|D]\}} td|dzdt|d| ddddt%| d\}}}}|j's td\|j!|xs| d| d|xs| d||ddtd t|d!|rt)|d"|t|S)#Nrz--colr)defaultz--pagesr#)typer^z--all store_true)actionu 宁海县-u 栏目: z (col)z total=u 条u 第u页: r&z [rNz] r'(z...u [SKIP] 正文空r(r")r'r&rT site_namerWsummaryu 共 u 条有效ninghai_)argparseArgumentParser add_argumentr4 parse_argsr<COL_MAPr-r1setrIallpagesrangerQaddr8 enumerater\r:r)riparserargsr<rf items_allseen first_itemsrC max_pagespgrD_itrecordsrXtrAsrWs rHmainrhsS  $ $ &F 6  Q7  5    D ((C C:cU!34I Jykse1 -.I 5D#C+K HUG4 ! xxRZB& JJ Ay1}% % 6!#r*HE1E bTs5zl$/0 %B%y$E#  $ % %GY'  R CE7!C N+2bk#2.>-?sCD'5 21a}} ( ) %"W+e9RZ"     F3w<. +,HSE"23 w<__main__zDone: z records)rd)__doc__sysosr,r2r)pathinsertdirnameabspath__file__ crawler_librr9r.r/rmrIr\r__name__cntr1rrHrs#""277??277??8#<=>(" K K]  16*0  >))X1h z &C F3%x !r