sjwEpdZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z e je jjdZdZdZdZdZ ddlmZej0j3d d Zej0j3d d Zd ZddddZdZeej@D]3\Z!Z"e"jGds e$e"jKdddZ5edk\rendZ'dZ(ddZ)dZ*dZ+da,da-dZ.dZ/dZ0dZ1e2dk(re1yy#e$rdZYwxYw#e&$rY~wxYw)uJ 察右后旗-社会公告 (cyhq.gov.cn) 列表: 首页服务端渲染 ul.long_list_col > li > a (/shgg/{docno}.html) 翻页: POST https://www.cyhq.gov.cn/rmt/tmpl/api/site/{siteId}/channelid/{channelId} body: {"currpage":N,"pagesize":10,"params":"<页面内联544字符串>"} 详情: /shgg/{docno}.html, 正文 div#content Nzhttp://www.cyhq.gov.cnu察右后旗-社会公告u 内蒙古zcrawl_cyhq_shgg.pyu 环评公示) BeautifulSoupDB_PATHz/mnt/data/search.db JSONL_PATHutf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagez--pages==c2|sytj|}|jddjddjddjdd}tjdd|}tjdd|}|j S) Nru​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts $/root/gov_crawler/crawl_cyhq_shgg.py clean_titler-s !A (B''"5==hKSST\^`aA Er1MA vsAA 779ctt}|r||d<d}tdD]h} tj|||d}|j dk(rt |_|jcSd|j z}tjdjtd |dd d |d y#t$r}t|dd}Yd}~Jd}~wwxYw)NRefererFheaderstimeoutverifyzHTTP %sPz [fetch fail] Zz -> Tflush)dictHEADERSrangerequestsget status_codeENCencodingtext Exceptionstrtimesleepprint)urlrefererrhlast_err_res rfetchr:7s W A) H 1X  # S!WUKA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #s>B&)B&& C/CCc 4 !"|sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}g i!tjdd|}tjd d|}tr t |d }|j d d D]>}|j dsG|j ds6tjd|j ddtjs\|d}|j dxs|}|jd xsW|j dxsD|j dxs1tjj|jdd}tjj|} j!||fdt#!z}d|d|d!|<|j%|A|j dD]_} | j dxs| j dxsd} d| j'vsd| j'vsP| j)a|j dD]@} | j d} | s| jd r+g} | D]} | j dxs| j dxsd} | s.tjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}| j!d|d|d| sd"t#!z}d#d$j-| zd%z!|<| j%|C|j dD]8} | j dxs| j dxsd} | rd| j'vrAtjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}d"t#!z}d&}| j.}|!|j0dk(rd }n|j.}|!|r d|d|d!|<n d'|d|d(!|<| j%|;t3|}tjd)d*|tjtjz} !fd+}tjd,||tjtjz}trQ t |d }|j d D]%}d-D]}|j6j9|d 't3|}g"!"fd.}tjd/|tjrd0nd}tjd1||tjtjz}tjd%d2|tj}tjd3d4|tj}tjd5d|tj}g}|jd6D]}}|j+}|stjd7d|}tjd8d|}|j;d9dj;d:d;}tjd|}|j;d>d;j;d?d;}tjd@d|}|stjdAd|tj}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}|j!|j+d6j-|}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}tjdCd|}tjdDd|}tjdEd|}tjdFd|}t=j>|}tjdGd4|}tjdHd6|}tjdflagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】u'发布(?:时间|日期)[:：][^<]{0,30} html.parseraT)hrefappendixz data-appendixz\.(pdf|docx?|xlsx?|zip|rar|wps)r@oldsrc)r_titletitle?z__ATTACH__%d__ imgsrczfileTypeImages/icon_filetypeimagespaltu图片z __IMG__%d__

z


Fz

]*>(.*?)z\1c6|jd}|jdry|jd}tjdd|}t j |j }tjdd|j }|s5tjj|jddxsd }tjj|}j||fd tz}d |d |d |<|S)Nr z javascript:rr!<[^>]+>u^附件[:：]\s*rEru附件z __LINK__%d__rFrGrH)group startswithrrrrrospathbasenamespliturllibparseurljoinappendlen) mr@innertxtabs_urlkey attachments link_protectpage_urls r _link_replz html_to_text.._link_replswwqz ??= ) ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*s<00ELcR S rz&]*href="([^"]+)"[^>]*>(.*?))styleclasslangdiralignvalignwidthheightborder cellpadding cellspacing indenttextnoextractcontentwzadestabindexcZ|jd}jD]I\}}|jds|j||jddjdd}Kt j dd|tj }t j dd|tj }t j d d|tj }t j d d|tj }j|d td z zS) Nr__IMG__rNrrO(]*>r<z

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

__TBL__%d__r ) rRitemsrSrrrIr[r\)r]tblkvrc table_protects r _tbl_replzhtml_to_text.._tbl_replsggaj &&( ODAq||I&kk!QYYub%9%A%A&"%MN Off@"cQSQUQUVff2E3bddKff4fcNff3RBDDIS!M 2Q 677rz ]*>r z]*>.*?z

z  rwz z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  r z [ \t]{2,}rQ z z[\s\u200b\u200c\u200d\ufeff]+z

\s*

rxz __LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r )$rrSrzrfind_allr)searchget_textrTrUrVrWrXrYrZr[r\ replace_withlower decomposerjoinparentnamer/r.attrspoprrrry enumeratesetadd)#contentrd_soupr?r@ real_hrefr_r`rarIrJrLimgssrcshas_p_ancestor_parretagattrr has_tablepartsblockbplainr|r}ir{outseenfinalrbrcr~s# ` @@@r html_to_textrJsR 1byff]Brtt"))Lnpqpupuv|AqBDFDHDHCIV9D !h 74I**4*0CAEE(OCquuW~CY[Y`Y`YiYijsjyjyz}j~@AkBZCC$ll228YGG&&~6*S->>CQXZ](^L%NN3' (~~e, $wwu~@):@b)SYY[8#KSWWW5E#K"R"R"T#HXZX_X_XhXhilirirsviwxyizX{#H@HC KKQXZ](^_ ` +c,.??,1GLL4F,F,O S)s+ ,~~e, &ggen?(9?R4 C ,,..x=wwu~?)9?RFFH{BGGL\L\]`]f]fgj]klm]nLo{s{#c,&77!&zz&yyC')-;;D & "QXZ](^L%"Y`ad(eL%  %% &&%jGff>wVXVZVZ]_]a]aVabGff> G[][_[_bdbfbf[fgG !'=9E~~d+ .D.DIIMM$-. .%jGM 8YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  FF="artt 4 &&( DAq !QA  . 2FAs -!+S1A 2 QWWY' * ++e C""$ 1kk!Q M*23kk-!+S12 &&!2s +C &&"c *C &&"c *C &&#R -C   C C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&K   :   sSA:i:?Ei::i:i:1i:B)i:1Ei: :i:Aj : jj jjcd}d}d}tjd|}|rt|jd}|sRtjd|tjtj z}|rt|jd}tjd|}|rU|jdd|jdj dd|jdj d}trzt|d }|jd xs7|jd xs$|jd xs|jd }|r"djd|jD}|||fS)Nrz)(.*?)u'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2})-r!rr>zdiv#contentspanz div#contentzdiv#zoomz div.articlec32K|]}t|yw)N)r/).0cs r zparse_detail..s"?a3q6"?s) rrrrRrrzzfillr select_onercontents) html_textrdrDdate content_htmlr]dmsoupels r parse_detailrsN E DL > JAAGGAJ'  II-y"$$+ F  +E =y IB XXa["((1+*;*;A*> @Q@QRS@TUY 6 __. /S4??=3QSUYUdUdeoUpStxuDuDERuS 77"?2;;"??L $ $$rc tjtdztdd}t|_|j }tjd|tj}|stjd|}|r|jda tjd|}|stjd |}|r{|jd k(r'd |jdd |jd a nE|jda tjd r dtza tjdda t!dtdt#txsddttfS)u'从首页提取 siteId/channelId/params/shgg/Frz?Pagination\.init\s*\(\s*{[^}]*?params\s*:\s*['\"]([^'\"]+)['\"]z.params[\"']?\s*[:=]\s*[\"']([^\"']{100,})[\"']r z0/tmpl/api/site/([0-9a-f]+)/channelid/([0-9a-f]+)z4dataurl[\"']?\s*[:=]\s*[\"']([^\"']*tmpl/api[^\"']*)r!z*https://www.cyhq.gov.cn/rmt/tmpl/api/site/z /channelid//zhttps://www.cyhq.gov.cn/rmtzhttp://zhttps://z api:z | params len:rTr#)r(r)BASE_URLr&r+r,r-rrrrR_PARAMS lastindex_API_URLrSrr2r\)r8rr]m2s r get_api_infors  X('2eTAAJ A TVWY[Y]Y]^A IIG K''!* F JB YYNPQ R <<1 VXV^V^_`VaceckcklmcnoHxx{H""3'88C'' :>H (Hos7=b/AN W rchttttrtsytj|dtdd}t t }d|d<tdz|d <d}td D]W} tjt|jd |d d } tj|jdcSt#d||fzdy#t$r|jcYcSwxYw#t$r}t|dd}Yd}~nd}~wwxYwtj d)Nr )currpagepagesizeparamsF ensure_asciizapplication/jsonz Content-Typerrrr)datarrrdrz! [fetch_list fail] page %d -> %sTr#)rrrjsondumpsr%r&rr'r(postencoder.r-r/r0r1r2)pagebodyr5r6r7r8r9s r fetch_listr+s 7?w ::4R7KZ_ `D W A*Anh&AiLH 1X  $ hT[[-A1VXafgA zz!&&(??  -x0@ @M  vv   $1vds|H $ 1 s64-C:"$CC72C:6C77C:: DDDc  g}|jjdrB tj|}|j dgxsgD]}|j dxs|j dxsd}|j dxsd}|j dxs|j dxsd}|r|snt d |d }d}|rvt jd t|} | rU| jd d | jdjdd | jdjd}|j|t||f|St jd|D]} | jd | jdj}} t|}|sAt j"j%t | }d}t jd|t'd|j)| dz |j)| dz} | r| jd }|j|||f|S#t$r&} tdt| dddgcYd} ~ Sd} ~ wwxYw)N{rdocnoidrrD inputTime createTimerz.htmlz!(\d{4})[-/](\d{1,2})[-/](\d{1,2})r rr!rz json parse err:<Tr#z6]*href="(/shgg/\d+\.html)"[^>]*>([^<]{4,100})z(\d{4}-\d{2}-\d{2})rrr)rrSrloadsr)rrrr/rRrr[rr.r2finditerrXrYrZmaxfind) rryditrrDtsr`rrr9r]rels r parse_listrEs7 E##C(  9%AeeFB'-2 Bw=266$<=2w-2VVK(FBFF<,@FBE/7?#GRQB-/XXa["((1+:K:KA:NPRPXPXYZP[PaPabcPde g{5'94@A BL [[RT] ^ -WWQZ!1!1!3UE" ,,&&x5 YY-yQ s@STW@W9XYbYgYghkYlmpYp/q r 88A;D gud+, - L  %s1vcr{$ ?I sEI JI>8J>Jc,d}d}t}g}g}ttdtdzD]} t |}t |}|std|zdnetd|t|fzd|D]\} } } | |vr |j| t| } | s|dz }1t| | \} }}| s| } |s| }t|| \}}}ttjdd |j}tj d |tj" xr2tj d |tj" xr | xr|d k}|d ks|rtd| ddzd|dz }t$| | | |d |t&d t(|rdj+d|Dnd d }t,r/|j/|td| ddd|dd|dz }k|j/||dz }td| ddd|ddt1j2dt1j2dt,r|st4j6j9dt:j6j=t:j6j?t@ddl!m"}||tFg}t,rltIt,dd5}|D]+}|jKtMjN|dd z- dddtd!t,t|fzdnt|rrt4j6j9dt:j6j=t:j6j?t@ddl!m"}||tFtd"|zdtd#|zdtd$t$zdy#t $r}td||fzdYd}~=d}~wwxYw#1swYxYw)%Nrr zPage %d fetch error: %sTr#zPage %d: empty, stopzPage %d: found %d itemsrQrz]z]rrz skip empty/placeholder: %s(z; c30K|]\}}|d|yw)|N)rurs rrzmain..s(KdaAq)9(Ks) site_name source_urlr3rDpub_datesummaryrcategorytags group_namerbz JSONL + z ()z + g?g?)push_to_searchdb) batch_labelwr)r,FrrzJSONL written: %s (%d rows)u 新增: %du 跳过: %dz=== %s done ===)(rrr'_PAGESrrr.r2r\rr:rrrrrrrz SITE_NAMECATEGORY GROUP_NAMErrr[r0r1sysrUinsertrTdirnameabspath__file__ crawler_libr SCRIPT_NAMEopenwriterr) new_count skip_count seen_urls jsonl_rowsbatchrrryr9r`rDrdhtmldtitleddaterrratts plain_lenno_paraitemrfrows rmainr ksIJIJ ENa!$4 "4(Iy)E (4/t <  '4U*<rsoHGGG++BBC # '  " ! **..$9 : ZZ^^L" -  @O'   chh DAq||J QWWS!_Q/0J "QA&f'R%.  :4#LJ5Z zFE M$   s$D D- D*)D*-D54D5