'sj[9dZddlZddlZddlZddlZddlZddlZddl Z ddl Z dZ dZ dZ dZdZ ddlmZej&j)d d Zej&j)d d Zd ZddddZdZeej6D]3\ZZej=ds eejAdddZ5edk\rendZ"dZ#ddZ$dZ%dZ&dZ'dZ(dZ)e*dk(re)e)yy#e$rdZYwxYw#e!$rY~wxYw)u 道县-公示公告 (湖南永州道县) 列表: /dx/gsgg/list{N}.shtml (465条/24页) 详情: /dx/gsgg/202603/{md5}.shtml, 正文 div#zoom Nzhttps://www.dx.gov.cnu道县-公示公告u湖南zcrawl_dx_gsgg.pyu 环评公示) BeautifulSoupDB_PATHz/mnt/data/search.db JSONL_PATHutf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagez--pages==c2|sytj|}|jddjddjddjdd}tjdd|}tjdd|}|j S) Nru​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts "/root/gov_crawler/crawl_dx_gsgg.py clean_titler)s !A (B''"5==hKSST\^`aA Er1MA vsAA 779cddl}ddl}|j|jjt t }|r||d<d}tdD]d} |j|||d}|jdk(rt|_ |jcSd|jz}tj d ft#d |dd d |d y#t$r} t| dd}Yd} ~ Jd} ~ wwxYw)NrRefererFheaderstimeoutverifyzHTTP %sPz [fetch fail] Zz -> Tflush)requestsurllib3disable_warnings exceptionsInsecureRequestWarningdictHEADERSrangeget status_codeENCencodingtext Exceptionstrtimesleepprint) urlrefererrr%r&hlast_err_res rfetchr>3s W//FFG W A) H 1X  # S!WU KA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #s:CC C0C++C0c 4 !"|sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}g i!tjdd|}tjd d|}tr t |d }|j d d D]>}|j dsG|j ds6tjd|j ddtjs\|d}|j dxs|}|jd xsW|j dxsD|j dxs1tjj|jdd}tjj|} j!||fdt#!z}d|d|d!|<|j%|A|j dD]_} | j dxs| j dxsd} d| j'vsd| j'vsP| j)a|j dD]@} | j d} | s| jd r+g} | D]} | j dxs| j dxsd} | s.tjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}| j!d|d|d| sd"t#!z}d#d$j-| zd%z!|<| j%|C|j dD]8} | j dxs| j dxsd} | rd| j'vrAtjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}d"t#!z}d&}| j.}|!|j0dk(rd }n|j.}|!|r d|d|d!|<n d'|d|d(!|<| j%|;t3|}tjd)d*|tjtjz} !fd+}tjd,||tjtjz}trQ t |d }|j d D]%}d-D]}|j6j9|d 't3|}g"!"fd.}tjd/|tjrd0nd}tjd1||tjtjz}tjd%d2|tj}tjd3d4|tj}tjd5d|tj}g}|jd6D]}}|j+}|stjd7d|}tjd8d|}|j;d9dj;d:d;}tjd|}|j;d>d;j;d?d;}tjd@d|}|stjdAd|tj}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}|j!|j+d6j-|}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}tjdCd|}tjdDd|}tjdEd|}tjdFd|}t=j>|}tjdGd4|}tjdHd6|}tjdflagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】u'发布(?:时间|日期)[:：][^<]{0,30} html.parseraT)hrefappendixz data-appendixz\.(pdf|docx?|xlsx?|zip|rar|wps)rDoldsrc)r_titletitle?z__ATTACH__%d__ imgsrczfileTypeImages/icon_filetypeimagespaltu图片z __IMG__%d__

z


Fz

]*>(.*?)z\1c6|jd}|jdry|jd}tjdd|}t j |j }tjdd|j }|s5tjj|jddxsd }tjj|}j||fd tz}d |d |d |<|S)Nr z javascript:rr!<[^>]+>u^附件[:：]\s*rIru附件z __LINK__%d__rJrKrL)group startswithrrrrrospathbasenamespliturllibparseurljoinappendlen) mrDinnertxtabs_urlkey attachments link_protectpage_urls r _link_replz html_to_text.._link_replswwqz ??= ) ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*s<00ELcR S rz&]*href="([^"]+)"[^>]*>(.*?))styleclasslangdiralignvalignwidthheightborder cellpadding cellspacing indenttextnoextractcontentwzadestabindexcZ|jd}jD]I\}}|jds|j||jddjdd}Kt j dd|tj }t j dd|tj }t j d d|tj }t j d d|tj }j|d td z zS) Nr__IMG__rRrrS(]*>r@z

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

__TBL__%d__r ) rVitemsrWrrrIr_r`)ratblkvrg table_protects r _tbl_replzhtml_to_text.._tbl_replsggaj &&( ODAq||I&kk!QYYub%9%A%A&"%MN Off@"cQSQUQUVff2E3bddKff4fcNff3RBDDIS!M 2Q 677rz ]*>r z]*>.*?z

z  r{z z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  r z [ \t]{2,}rU z z[\s\u200b\u200c\u200d\ufeff]+z

\s*

r|z __LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r )$rrSr~rfind_allr-searchget_textrXrYrZr[r\r]r^r_r` replace_withlower decomposerjoinparentnamer3r2attrspoprrrr} enumeratesetadd)#contentrh_souprCrD real_hrefrcrdrerMrNrPimgssrcshas_p_ancestor_parritagattrr has_tablepartsblockbplainrriroutseenfinalrfrgrs# ` @@@r html_to_textrJsR 1byff]Brtt"))Lnpqpupuv|AqBDFDHDHCIV9D !h 74I**4*0CAEE(OCquuW~CY[Y`Y`YiYijsjyjyz}j~@AkBZCC$ll228YGG&&~6*S->>CQXZ](^L%NN3' (~~e, $wwu~@):@b)SYY[8#KSWWW5E#K"R"R"T#HXZX_X_XhXhilirirsviwxyizX{#H@HC KKQXZ](^_ ` +c,.??,1GLL4F,F,O S)s+ ,~~e, &ggen?(9?R4 C ,,..x=wwu~?)9?RFFH{BGGL\L\]`]f]fgj]klm]nLo{s{#c,&77!&zz&yyC')-;;D & "QXZ](^L%"Y`ad(eL%  %% &&%jGff>wVXVZVZ]_]a]aVabGff> G[][_[_bdbfbf[fgG !'=9E~~d+ .D.DIIMM$-. .%jGM 8YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  FF="artt 4 &&( DAq !QA  . 2FAs -!+S1A 2 QWWY' * ++e C""$ 1kk!Q M*23kk-!+S12 &&!2s +C &&"c *C &&"c *C &&#R -C   C C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&K   :   sSA:i:?Ei::i:i:1i:B)i:1Ei: :i:Aj : jj jjcd}d}d}tjd|}|rt|jd}|sRtjd|tjtj z}|rt|jd}tjd|}|rU|jdd|jdj dd|jdj d}trgt|d }|jd xs$|jd xs|jd }|r"djd |jD}|||fS)Nrz)(.*?)u'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2})-r!rrBz div#zoomconzdiv#zoomzdiv.article-contentc32K|]}t|yw)N)r3).0cs r zparse_detail..s"?a3q6"?s) rrrrVrr~zfillr select_onercontents) html_textrhrHdate content_htmlradmsoupels r parse_detailrs E DL > JAAGGAJ'  II-y"$$+ F  +E =y IB XXa["((1+*;*;A*> @Q@QRS@TUY 6 __] + ttz/J tdoo^sNt 77"?2;;"??L $ $$rcddl}ddl}|j|jj|dk(r dt z}ndt |dzfz}|j |tdd}t|_ |jS)Nrr z%s/dx/gsgg/list2.shtmlz%s/dx/gsgg/list%d.shtmlFr) r%r&r'r(r)BASE_URLr-r+r/r0r1)pager%r&r7r<s r fetch_listr si W//FFG qy&1'8TAX*>> S'2e DAAJ 66Mrc g}tjd|tjtjzD],}t j |j dj}t j tjdd|j dj}t|}|rt|dkrtjjt|}d}tjd|t!d|j#|d z |j#|d z}|r|j d}|j%|||f/|S) NzA]*href="(/dx/gsgg/\d{6}/[0-9a-f]{32}\.shtml)"[^>]*>(.*?)r rUrr!z(\d{4}-\d{2}-\d{2})ri,d)rfinditerrr~rrrVrrrr`r\r]r^rrmaxfindr_) rr}rarDrcrHrdrrs r parse_listrs' E [[]_hjljnjnqsququju v -  ,224z2qwwqz BCIIKC E Q ,,&&x6 YY-yQ t@TUX@X9YZcZhZhimZnorZr/s t 88A;D gud+, - Lrcd}d}t}g}g}tdtdzD]} t|}t |}|st d|zdn]t d|t|fzd|D]\} } } | |vr |j| t| } | s|dz }1t| | \} }}| s| } |s| }t|| \}}}ttjdd |j}tjd |tj  xr+tjd |tj  xr| }|d ks|rt d | ddzd|dz }t"| | | |d |t$d t&|rdj)d|Dnd d }t*r/|j-|t d| ddd|dd|dz }c|j-||dz }t d| ddd|ddt/j0dt/j0dt*r|st2j4j7dt8j4j;t8j4j=t>ddl m!}||tDg}t*rltGt*dd5}|D]+}|jItKjL|ddz- dddt d t*t|fzdnt|rrt2j4j7dt8j4j;t8j4j=t>ddl m!}||tDt d!|zdt d"|zdt d#t"zdy#t $r}t d||fzdYd}~=d}~wwxYw#1swYxYw)$Nrr zPage %d fetch error: %sTr#zPage %d: empty, stopzPage %d: found %d itemsrUrz]z] z skip empty/placeholder: %s(z; c30K|]\}}|d|yw)|N)rurs rrzmain..Ls(KdaAq)9(Ks) site_name source_urlr7rHpub_datesummaryrcategorytags group_namerfz JSONL + z ()z + g?g?)push_to_searchdb) batch_labelwr)r0F) ensure_asciirzJSONL written: %s (%d rows)u 新增: %du 跳过: %dz=== %s done ===)'rr,_PAGESrrr2r6r`rr>rrrrrrr~ SITE_NAMECATEGORY GROUP_NAMErrr_r4r5sysrYinsertrXdirnameabspath__file__ crawler_libr SCRIPT_NAMEopenwritejsondumps) new_count skip_count seen_urls jsonl_rowsbatchrrr}r=rdrHrdhtmldtitleddaterrratts plain_lenno_paraitemrfrows rmainr"sIJIJ Ea!$4 "4(Iy)E (4/t <  '4U*<rrrrr__name__rrrrsC HGGG " !   ! **..$9 : ZZ^^L" -  @O'   chh DAq||J QWWS!_Q/0J "QA.f'R%, I5X zFFy M$   s#C2!C?2C<;C<?DD