sjA dZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z e je jjddddddd d d d d dddddddddd d dZdZeej&D]:\ZZej-dsej/dddj1Z<ej3eedZedZedZedZedZed Zed!Z ed"Z!ed#Z"ed$Z#ed%Z$d&Z% dd'l&m'Z'ejRj3d(d)Z*ejRj3d*d+Z+d,Z,d-d.d/d0Z-dZ.eej&D]3\ZZej-d1s e/ej/dddZ.5e.dk\re.ndZ1d2Z2d:d3Z3d4Z4d5Z5d6Z6d7Z7d8Z8e9d9k(re8yy#e($rdZ'YwxYw#e0$rYwwxYw);u JPAAS 平台通用脚本 (岱山/珠海) 用法: python3 crawl_jpaas_generic.py --site=daishan --pages=5 python3 crawl_jpaas_generic.py --site=zhuhai --pages=5 Nu(岱山县-重大建设项目环境审批u浙江zhttps://www.daishan.gov.cnzFhttps://www.daishan.gov.cn/col/col1229600559/index.html?number=a2503062548aC8iOROsWaAgpd69e6duMu 信息列表 1229600559div#zoomu 环评公示) site_name group_namebase_urllist_urlweb_id tpl_set_idtag_idpage_id detail_selcategoryu珠海市-通知公告u广东zhttps://www.zh.gov.cnz4https://www.zh.gov.cn/col/col1229821392/?uid=89138453428sncjq09fJ0PtSd61FHyUBu当前栏目列表1a 1229821392div.bt-content)daishanzhuhairz--site==rrr r r r r rrrzcrawl_jpaas_generic.py) BeautifulSoupDB_PATHz/mnt/data/search.db JSONL_PATHutf-8zkMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zzh-CN,zh;q=0.9)z User-AgentAcceptzAccept-Languagez--pages=c2|sytj|}|jddjddjddjdd}tjdd|}tjdd|}|j S) Nru​u‌u‍uu3^\s*(?:·|·|\u00b7)?\s*(?: |\u00a0)?\s*\s+ )html_libunescapereplaceresubstrip)ts (/root/gov_crawler/crawl_jpaas_generic.py clean_titler*Qs !A (B''"5==hKSST\^`aA Er1MA vsAA 779ctt}|r||d<d}tdD]h} tj|||d}|j dk(rt |_|jcSd|j z}tjdjtd |dd d |d y#t$r}t|dd}Yd}~Jd}~wwxYw)NRefererFheaderstimeoutverifyzHTTP %sPz [fetch fail] Zz -> Tflush)dictHEADERSrangerequestsget status_codeENCencodingtext Exceptionstrtimesleepprint)urlrefererr1hlast_err_res r)fetchrN[s W A) H 1X  # S!WUKA}}#  vv  1==0H 1   s3Bx :$G   #1vcr{H #s>B&)B&& C/CCc 4 !"|sddgfStjdd|tj}tjdd|tj}tjdd|tjtjz}tjdd|tjtjz}g i!tjdd|}tjd d|}tr t |d }|j d d D]>}|j dsG|j ds6tjd|j ddtjs\|d}|j dxs|}|jd xsW|j dxsD|j dxs1tjj|jdd}tjj|} j!||fdt#!z}d|d|d!|<|j%|A|j dD]_} | j dxs| j dxsd} d| j'vsd| j'vsP| j)a|j dD]@} | j d} | s| jd r+g} | D]} | j dxs| j dxsd} | s.tjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}| j!d|d|d| sd"t#!z}d#d$j-| zd%z!|<| j%|C|j dD]8} | j dxs| j dxsd} | rd| j'vrAtjj| }| j d xs| j dxsdj+xs5tjj| jddxsd!}d"t#!z}d&}| j.}|!|j0dk(rd }n|j.}|!|r d|d|d!|<n d'|d|d(!|<| j%|;t3|}tjd)d*|tjtjz} !fd+}tjd,||tjtjz}trQ t |d }|j d D]%}d-D]}|j6j9|d 't3|}g"!"fd.}tjd/|tjrd0nd}tjd1||tjtjz}tjd%d2|tj}tjd3d4|tj}tjd5d|tj}g}|jd6D]}}|j+}|stjd7d|}tjd8d|}|j;d9dj;d:d;}tjd|}|j;d>d;j;d?d;}tjd@d|}|stjdAd|tj}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}|j!|j+d6j-|}!jAD]\}}|j;||}tC"D]\}}|j;dB|z|}tjdCd|}tjdDd|}tjdEd|}tjdFd|}t=j>|}tjdGd4|}tjdHd6|}tjdflagsz ]*>z]*>.*?z]*>.*?u【字号：[^】]*】u'发布(?:时间|日期)[:：][^<]{0,30} html.parseraThrefappendixz data-appendixz\.(pdf|docx?|xlsx?|zip|rar|wps)rUoldsrcr'_titletitle?z__ATTACH__%d__ imgsrczfileTypeImages/icon_filetypeimagespaltu图片z __IMG__%d__

z


Fz

]*>(.*?)z\1c6|jd}|jdry|jd}tjdd|}t j |j }tjdd|j }|s5tjj|jddxsd }tjj|}j||fd tz}d |d |d |<|S)Nrz javascript:rr5<[^>]+>u^附件[:：]\s*r[ru附件z __LINK__%d__r\r]r^)group startswithr%r&r"r#r'ospathbasenamespliturllibparseurljoinappendlen) mrUinnertxtabs_urlkey attachments link_protectpage_urls r) _link_replz html_to_text.._link_replswwqz ??= ) ffZU+$**,ff("c288:''""4::c?1#56B(C,,&&x6GS>*s<00ELcR S r+z&]*href="([^"]+)"[^>]*>(.*?))styleclasslangdiralignvalignwidthheightborder cellpadding cellspacing indenttextnoextractcontentwzadestabindexcZ|jd}jD]I\}}|jds|j||jddjdd}Kt j dd|tj }t j dd|tj }t j d d|tj }t j d d|tj }j|d td z zS) Nr__IMG__rdrre(]*>rPz

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

z

\s*(?:(?:\r?\n)\s*)*

__TBL__%d__r) rhitemsrir$r%r&Irqrr)rstblkvry table_protects r) _tbl_replzhtml_to_text.._tbl_replsggaj &&( ODAq||I&kk!QYYub%9%A%A&"%MN Off@"cQSQUQUVff2E3bddKff4fcNff3RBDDIS!M 2Q 677r+z ]*>rz]*>.*?z

z  rz z(?<=>)\s*[\r\n\t]+\s*z\s*[\r\n\t]+\s*(?=<)  r!z [ \t]{2,}rg z z[\s\u200b\u200c\u200d\ufeff]+z

\s*

rz __LINK__\d+__z __TBL__\d+__z __IMG__\d+__z__ATTACH__\d+__z[ \t]+\nz\n{3,}r )$r%r&Srrfind_allr=searchget_textrjrkrlrmrnrorprqrr replace_withlower decomposer'joinparentnamerCrBattrspopr$r"r#r enumeratesetadd)#contentrz_souprSrU real_hrefrurvrwr_r`rbimgssrcshas_p_ancestor_parr{tagattrr has_tablepartsblockbplainrriroutseenfinalrxryrs# ` @@@r) html_to_textrnsR 1byff]Brtt"))Lnpqpupuv|AqBDFDHDHCIV9D !h 74I**4*0CAEE(OCquuW~CY[Y`Y`YiYijsjyjyz}j~@AkBZCC$ll228YGG&&~6*S->>CQXZ](^L%NN3' (~~e, $wwu~@):@b)SYY[8#KSWWW5E#K"R"R"T#HXZX_X_XhXhilirirsviwxyizX{#H@HC KKQXZ](^_ ` +c,.??,1GLL4F,F,O S)s+ ,~~e, &ggen?(9?R4 C ,,..x=wwu~?)9?RFFH{BGGL\L\]`]f]fgj]klm]nLo{s{#c,&77!&zz&yyC')-;;D & "QXZ](^L%"Y`ad(eL%  %% &&%jGff>wVXVZVZ]_]a]aVabGff> G[][_[_bdbfbf[fgG !'=9E~~d+ .D.DIIMM$-. .%jGM 8YY>AIff/G244RTRVRV;WGffWj'>Gff\4=Gff@"gUWUYUYZG Ev&  KKM  FF+R 3 FF*B 2 IIdB  ' 'c 2 FF<a (z2q)!!%( fc*228SA7UC  FF="artt 4 &&( DAq !QA  . 2FAs -!+S1A 2 QWWY' * ++e C""$ 1kk!Q M*23kk-!+S12 &&!2s +C &&"c *C &&"c *C &&#R -C   C C &&dC (C &&FC (C &&sC (C 5D E YYv ffVR B!:;  $;    Q ++e  " " $C  ; &&K   :   sSA:i:?Ei::i:i:1i:B)i:1Ei: :i:Aj : jj jjcd}d}d}tjd|}|rt|jd}|sRtjd|tjtj z}|rt|jd}tjd|}|rU|jdd|jdj dd|jdj d}tr~t|d }|jtxs7|jd xs$|jd xs|jd }|r"djd |jD}|||fS)Nrz)(.*?)u'(\d{4})[-年/](\d{1,2})[-月/](\d{1,2})-r5r.rRrrz div.contentc32K|]}t|yw)N)rC).0cs r) zparse_detail..)s"?a3q6"?s) r%rr*rhrrzfillr select_one DETAIL_SELrcontents) html_textrzrZdate content_htmlrsdmsoupels r) parse_detailrsB E DL > JAAGGAJ'  II-y"$$+ F  +E =y IB XXa["((1+*;*;A*> @Q@QRS@TUY 6 __Z (ODOOJ,GO4??[kKlOptppANqO 77"?2;;"??L $ $$r+cxddlm}dttdtdt d}|dkDrt j|dd d |d <|j|}td |}tt}t|d<d|d<tj||dd }t|_ |j"S#t$$rYywxYw)z JPAAS GET APIrN bulidstaticcolumnnull) parseTypewebIdtplSetIdpageTypetagIdeditTypepageIdr )pageNopageSizeF ensure_ascii paramJsonz8/api-gateway/jpaas-publish-server/front/page/build/unit?r-zapplication/jsonrr/r) urllib.parseroWEB_ID TPL_SET_IDTAG_IDPAGE_IDjsondumps urlencodeBASE_URLr9r:LIST_URLr<r=r?r@rArB)pageupparamsqsrGrIrLs r) fetch_listr-s"F ax"jjDb)IX]^{ f BLTVX YC W AAiL$AhK S!R>AAJvv s! B-- B98B9cg} tj|}|jdxsi}|jdxsd}|s|Str*t |d}|j ddD]}|d}d |vrt jd |s$|jd xs|jd d }t jdd|xsd}t|}|rt|dkrtjjt|} d} |j!d} | r:t jd| jd d } | r| j#d} |j%| || f |S#t$r|cYSwxYw)NdatahtmlrrRrSTrTrUz/art/z art_\w+\.htmlrZr!rXu^\s*\[?图片\]?\s*spanz(\d{4}-\d{2}-\d{2})r)rloadsrBr=rrr%rrr&r*rrrnrorpr find_nextrhrq) rrdr html_fragrrSrUrurvrsprs r) parse_listrHsp E JJy ! 55= BD &BI  Y 6s. /AV9Dd"2995Et+L%%.?AJJs$J$?C&&/SYB?Cc"C#c(Q,ll**8T:GDV$BYY5r{{3d{7ST88A;D LL'3- .! /" L3  sE:: FFc$d}d}t}g}g}tdtdzD]} t|}t |}|st d|zdnkt d|t|fzd|D]\} } } | |vr |j| t| t} | s|dz }7t| | \} }}| s| } |s| }t|| \}}}ttjd d |j}tj d |tj" xr2tj d |tj" xr | xr|d k}|dks|rt d| ddzd|dz }t$| | | |d |t&d t(|rdj+d|Dnd d }t,r/|j/|t d| ddd|dd|dz }q|j/||dz }t d| ddd|ddt1j2dt1j2dt,r |s$t4j6j9dt:j6j=t:j6j?t@ddl!m"}||tFg}t,rltIt,dd5}|D]+}|jKtMjN|d d!z- dddt d"t,t|fzdnt|rrt4j6j9dt:j6j=t:j6j?t@ddl!m"}||tFt d#|zdt d$|zdt d%t$zdy#t $r}t d||fzdYd}~=d}~wwxYw#1swYxYw)&NrrzPage %d fetch error: %sTr7zPage %d: empty, stopzPage %d: found %d items)rHrgrz]z]rrz skip empty/placeholder: %s(z; c30K|]\}}|d|yw)|N)rur(s r)rzmain..s(KdaAq)9(Ks) r source_urlrGrZpub_datesummaryrrtagsrrxz JSONL + z ()z + g?g?)push_to_searchdb) batch_labelwr)r@FrrzJSONL written: %s (%d rows)u 新增: %du 跳过: %dz=== %s done ===)(rr;_PAGESrrrBrFrrrrNrrrr%r&r'rr SITE_NAMECATEGORY GROUP_NAMErrrqrDrEsysrkinsertrjdirnameabspath__file__ crawler_libr SCRIPT_NAMEopenwriterr) new_count skip_count seen_urls jsonl_rowsbatchrrrrMrvrZrdhtmldtitleddaterrratts plain_lenno_paraitemrfrows r)mainr"hsIJIJ Ea!$4 "4(Iy)E (4/t <  '4U*<>PP__main__)Nr):__doc__r rjr%rDrr"rrnsqlite3rr<urllib3disable_warnings exceptionsInsecureRequestWarningSITES_SITErargvrrSrirmr'r=CFGrr rrrrrrrr rbs4r ImportErrorenvironrrr?r: _MAX_PAGESint ValueErrorrr*rNrrrrr"__name__rr+r)r5s] HGGG++BBC@0\-  " .+J-(&"  6  chh +DAq||IQ"((*+ iiuY'(     z? z? X   X i.   z?& ! **..$9 : ZZ^^L" -  @O'   chh DAq||J QWWS!_Q/0J "QA&f'R%,6@I5X zFu M$   s$G:GG GGG