sjMAdZddlZddlZddlZddlZddlZddlZddl Z ddl Z dZ dZ dZ dZdZ ddlmZej&j)d d Zej&j)d d Zd ZddddZdZeej6D]3\ZZej=ds eejAdddZ5edk\rendZ"dZ#d#dZ$dZ%dZ&dZ'eej6D]:\ZZej=dsejAdddjQZ'<dddZ)dddZ*e'e*vre*e'Z dZ+d Z,d!Z-e.d"k(re-e-yy#e$rdZY,wxYw#e!$rYwxYw)$u 东至县信息公开 (OpennessTarget JS接口) --col=318 重大行政决策预公开(80630) / --col=311 最新公开(0) 列表: /OpennessTarget/{col}/{param}/page_N.html 详情: /OpennessContent/show/{id}.html, 正文 div#zoom Nzhttps://www.dongzhi.gov.cnu%东至县-重大行政决策预公开u安徽zcrawl_dongzhi_xxgk.pyu 环评公示) BeautifulSoupDB_PATHz/mnt/data/search.db JSONL_PATHutf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagez--pages==c2|sytj|}|jddjddjddjdd}tjdd|}tjdd|}|j S) Nru​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts '/root/gov_crawler/crawl_dongzhi_xxgk.py clean_titler*s !A (B''"5==hKSST\^`aA Er1MA vsAA 779cddl}ddl}|j|jjt t }|r||d<d}tdD]d} |j|||d}|jdk(rt|_ |jcSd|jz}tj d ft#d |dd d |d y#t$r} t| dd}Yd} ~ Jd} ~ wwxYw)NrRefererFheaderstimeoutverifyzHTTP %sPz [fetch fail] Zz -> Tflush)requestsurllib3disable_warnings exceptionsInsecureRequestWarningdictHEADERSrangeget status_codeENCencodingtext Exceptionstrtimesleepprint) urlrefererrr%r&hlast_err_res rfetchr>4s W//FFG W A) H 1X  # S!WU KA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #s:CC C0C++C0c %&'|sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}g%i&tjdd|}tjd d|}tr- t |d }|j d d D]U}|j dxsdj}|j ds;|j ds*tjd|tjsd|vss|d}|j dxs|}|jd xsW|j dxsD|j dxs1tjj|jdd}tjj!|}%j#||fdt%&z} d|d|d&| <|j'| X|j ddgD]} | j)|j ddi D]T} | j d d D]}|j dxsdj} d| vs&tjd| tjsP|d}|j dxs|}|jd xs|j dxsd!}tjj!|}%j#||fdt%&z} d|d|d&| <|j'| t+| j,D]"} t/| d"dd#vs| j)$| j'dW|j d$D]_}|j d%xs|j dxsd}d&|jvsd'|jvsP|j)a|j d(D]@}|j d$}|s|jd r+g}|D]}|j d%xs|j dxsd}|s.tjj!|}|j d)xs|j dxsdj1xs5tjj|jddxsd*}|j#d|d|d|sd+t%&z} d,d-j3|zd.z&| <|j'| C|j d$D]8}|j d%xs|j dxsd}|rd&|jvrAtjj!|}|j d)xs|j dxsdj1xs5tjj|jddxsd*}d+t%&z} d/}|j4}|!|j6d(k(rd }n|j4}|!|r d|d|d&| <n d0|d|d1&| <|j'| ;t9|}tjd2d3|tjtjz}%&fd4}tjd5||tjtjz}trQ t |d }|j d D]%}d6D]}|j<j?|d 't9|}g'&'fd7}tjd8|tjrd9nd}tjd:||tjtjz}tjd.d;|tj}tjdd|tj}g}|jd?D]}}|j1}|stjd@d|}tjdAd|}|jAdBdjAdCdD}tjdEdD|}tjdFd|}tCjD|}|jAdGdDjAdHdD}tjdId|}|stjdJd|tj}&jGD]\}}|jA||}tI'D]\} }!|jAdK| z|!}|j#|j1d?j3|}"&jGD]\}}|"jA||}"tI'D]\} }!|"jAdK| z|!}"tjdLd|"}"tjdMd|"}"tjdNd|"}"tjdOd|"}"tCjD|"}"tjdPd=|"}"tjdQd?|"}"tjdEdD|"}"tK}#g}$|"jd?D]X}tjdRdtjdFd|} | s2| |#vr7|#jM| |$j#|Zd?j3|$j1}"|"|%fS#t:$rYwxYw#t:$rYCwxYw)SNrrz flagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】u'发布(?:时间|日期)[:：][^<]{0,30} html.parseraT)hrefrDappendixz data-appendixz\.(pdf|docx?|xlsx?|zip|rar|wps)z /download?oldsrc)r_titletitle?z__ATTACH__%d__ objectembedclasszxu-list)attrs附件name)rMrNimgsrczfileTypeImages/icon_filetypeimagespaltu图片z __IMG__%d__

z


Fz

]*>(.*?)z\1c6|jd}|jdry|jd}tjdd|}t j |j }tjdd|j }|s5tjj|jddxsd }tjj|}j||fd tz}d |d |d |<|S)Nr z javascript:rr!<[^>]+>u^附件[:：]\s*rIrrQz __LINK__%d__rJrKrL)group startswithrrrrrospathbasenamespliturllibparseurljoinappendlen) mrDinnertxtabs_urlkey attachments link_protectpage_urls r _link_replz html_to_text.._link_replswwqz ??= ) ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*s<00ELcR S rz&]*href="([^"]+)"[^>]*>(.*?))stylerOlangdiralignvalignwidthheightborder cellpadding cellspacing indenttextnoextractcontentwzadestabindexcZ|jd}jD]I\}}|jds|j||jddjdd}Kt j dd|tj }t j dd|tj }t j d d|tj }t j d d|tj }j|d td z zS) Nr__IMG__rXrrY(]*>r@z

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

__TBL__%d__r ) r\itemsr]rrrIrerf)rgtblkvrm table_protects r _tbl_replzhtml_to_text.._tbl_replsggaj &&( ODAq||I&kk!QYYub%9%A%A&"%MN Off@"cQSQUQUVff2E3bddKff4fcNff3RBDDIS!M 2Q 677rz ]*>r z]*>.*?z

z  rz z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  r z [ \t]{2,}r[ z z[\s\u200b\u200c\u200d\ufeff]+z

\s*

rz __LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r )'rrSrrfind_allr-lowersearchget_textr^r_r`rarbrcrdrerf replace_with decomposelistchildrengetattrrjoinparentrRr3r2rPpoprrrr enumeratesetadd)(contentrn_souprC_href_lrD real_hrefrirjrkobjxul_hchrSrTrVimgssrcshas_p_ancestor_parrotagattrr has_tablepartsblockbplainrriroutseenfinalrlrmrs( ` @@@r html_to_textrKs 1byff]Brtt"))Lnpwy{y}y}B~COSZCZV9D !h 74I**4*0CAEE(OCquuW~CY[Y`Y`YiYijsjyjyz}j~@AkBZCC$ll228YGG&&~6*S->>CQXZ](^L%NN3' (~~x&9:   ~~Wi,@~A %c5 ,A%%--2446B#r)RYY7Y[]_a_c_c-d y$%EE(O$;t jjtj4ShS8"(,,"6"6x"K#**GS>:.\1BBU\^a,b S)s+ ,s||,'Br6404GG '  $# %$~~e, $wwu~@):@b)SYY[8#KSWWW5E#K"R"R"T#HXZX_X_XhXhilirirsviwxyizX{#H@HC KKQXZ](^_ ` +c,.??,1GLL4F,F,O S)s+ ,~~e, &ggen?(9?R4 C ,,..x=wwu~?)9?RFFH{BGGL\L\]`]f]fgj]klm]nLo{s{#c,&77!&zz&yyC')-;;D & "QXZ](^L%"Y`ad(eL%  %% &&%jGff>wVXVZVZ]_]a]aVabGff> G[][_[_bdbfbf[fgG !'=9E~~d+ .D.DIIMM$-. .%jGM 8YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  FF="artt 4 &&( DAq !QA  . 2FAs -!+S1A 2 QWWY' * ++e C""$ 1kk!Q M*23kk-!+S12 &&!2s +C &&"c *C &&"c *C &&#R -C   C C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&K   :   saBp%Fp%C p%*Bp%1:p%,p%?1p%1B)p%Ep%6:p%.Ap5% p21p25 qqcd}d}d}tjd|}|rt|jd}|sRtjd|tjtj z}|rt|jd}tjd|}|rU|jdd|jdj dd|jdj d}trgt|d }|jd xs$|jd xs|jd }|r"djd |jD}|||fS)Nrz)(.*?)u'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2})-r!rrBzdiv#zoomz div.g-detailzdiv.article-contentc32K|]}t|yw)N)r3).0cs r zparse_detail..s"?a3q6"?s) rrrr\rrzfillr select_onercontents) html_textrnrHdate content_htmlrgdmsoupels r parse_detailr s E DL > JAAGGAJ'  II-y"$$+ F  +E =y IB XXa["((1+*;*;A*> @Q@QRS@TUY 6 __Z ( uDOON,K ut_tOu 77"?2;;"??L $ $$r318z--col=806300)r311u东至县-最新公开c^ddl}ddl}|j|jjt j td}dtt||fz}tt}d|d<tdtd|d|d <|j ||d d }t|_ |jS) Nrrz$%s/OpennessTarget/%s/%s/page_%d.htmlXMLHttpRequestzX-Requested-Withz/OpennessContent/showList//z /page_1.htmlrFr)r%r&r'r(r) COL_PARAMSr-_COLBASE_URLr*r+r/r0r1)pager%r&paramr7r9r<s r fetch_listr(s W//FFG NN4 %E 0HdE43P PC W A,AFNPTV[\AiL S!R >AAJ 66Mrc g}tjd|tjtjzD],}t j |j dj}t j tjdd|j dj}t|}|rt|dkrtjjt|}d}tjd|t!d|j#|d z |j#|d z}|r|j d}|j%|||f/|S) Nz>]*href="(/OpennessContent/show/\d+\.html)"[^>]*>(.*?)r r[rr!z(\d{4}-\d{2}-\d{2})ri,d)rfinditerrrrrr\rrrrfrbrcrdrrmaxfindre) rrrgrDrirHrjrrs r parse_listr3s' E [[Z\egigkgknpnrnrgr s -  ,224z2qwwqz BCIIKC E Q ,,&&x6 YY-yQ t@TUX@X9YZcZhZhimZnorZr/s t 88A;D gud+, - Lrcd}d}t}g}g}tdtdzD]} t|}t |}|st d|zdnet d|t|fzd|D]\} } } | |vr |j| t| } | s|dz }1t| | \} }}| s| } |s| }t|| \}}}ttjdd |j}tjd |tj  xr2tjd |tj  xr | xr|d k}|d ks|rt d| ddzd|dz }t"| | | |d |t$d t&|rdj)d|Dnd d }t*r/|j-|t d| ddd|dd|dz }k|j-||dz }t d| ddd|ddt/j0dt/j0dt*r|st2j4j7dt8j4j;t8j4j=t>ddl m!}||tDg}t*rltGt*dd5}|D]+}|jItKjL|dd z- dddt d!t*t|fzdnt|rrt2j4j7dt8j4j;t8j4j=t>ddl m!}||tDt d"|zdt d#|zdt d$t"zdy#t $r}t d||fzdYd}~=d}~wwxYw#1swYxYw)%Nrr zPage %d fetch error: %sTr#zPage %d: empty, stopzPage %d: found %d itemsr[rz]z]r z skip empty/placeholder: %s(z; c30K|]\}}|d|yw)|N)rurs rrzmain..ms(KdaAq)9(Ks) site_name source_urlr7rHpub_datesummaryrcategorytags group_namerlz JSONL + z ()z + g?g?)push_to_searchdb) batch_labelwr)r0F) ensure_asciirzJSONL written: %s (%d rows)u 新增: %du 跳过: %dz=== %s done ===)'rr,_PAGESrrr2r6rfrr>rrrrrrr SITE_NAMECATEGORY GROUP_NAMErrrer4r5sysr_insertr^dirnameabspath__file__ crawler_libr SCRIPT_NAMEopenwritejsondumps) new_count skip_count seen_urls jsonl_rowsbatchrrrr=rjrHrdhtmldtitleddaterrratts plain_lenno_paraitemrfrows rmainr BsIJIJ Ea!$5 "4(Iy)E (4/t <  '4U*<rrrrr COL_NAMESrrr __name__rrrrs HGGG ' 3  % ! **..$9 : ZZ^^L" -  @O'   chh DAq||J QWWS!_Q/0J "QA.{'|%,  chh *DAq||HwwsAq!'')*S ) ;D\ ] 9$I J5Z zFFy M$   s#E!E$E! E!$E,+E,