optimum-neuron/pr_1110/en/_app/immutable/nodes/23.c8be96a3.js

import{s as ue,n as fe,o as he}from"../chunks/scheduler.56725da7.js";import{S as be,i as Me,e as o,s as l,c as r,h as ye,a as i,d as t,b as a,f as ce,g as m,j as B,k as de,l as we,m as s,n as p,t as c,o as d,p as u}from"../chunks/index.18a26576.js";import{C as Te}from"../chunks/CopyLLMTxtMenu.22a092d8.js";import{C as v}from"../chunks/CodeBlock.e01e74e0.js";import{H as te}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.29952ab9.js";function Je(se){let f,C,I,N,h,R,b,z,M,le='This guide explains how to convert, load, and use <a href="https://www.sbert.net/" rel="nofollow">Sentence Transformers (SBERT)</a> models on AWS Inferentia2 using Optimum Neuron, enabling efficient calculation of embeddings. Sentence Transformers are powerful models for generating sentence embeddings. You can use this Sentence Transformers to compute sentence / text embeddings for more than 100 languages. These embeddings can then be compared e.g. with cosine-similarity to find sentences with a similar meaning. This can be useful for semantic textual similarities, semantic search, or paraphrase mining.',G,y,ae="<em>Note: Currently only text models are supported, we are working on vision support for CLIP.</em>",x,w,Q,T,oe="First, you need to convert your Sentence Transformers model to a format compatible with AWS Inferentia2. You can compile Sentence Transformers models with Optimum Neuron using the <code>optimum-cli</code> or <code>NeuronModelForSentenceTransformers</code> class. Below you will find an example for both approaches. We have to make sure <code>sentence-transformers</code> is installed. That’s only needed for exporting the model.",F,J,H,Z,re=`Here we will use the <code>NeuronModelForSentenceTransformers</code>, which can convert any Sntence Transformers model to a format compatible with AWS Inferentia2 or load already converted models.
When exporting models with the <code>NeuronModelForSentenceTransformers</code> you need to set <code>export=True</code> and define the input shape and batch size. The input shape is defined by the <code>sequence_length</code> and the batch size by <code>batch_size</code>.`,L,g,Y,U,ie="Here we will use the <code>optimum-cli</code> to convert the model. Similar to the <code>NeuronModelForSentenceTransformers</code> we need to define our input shape and batch size. The input shape is defined by the <code>sequence_length</code> and the batch size by <code>batch_size</code>. The <code>optimum-cli</code> will automatically convert the model to a format compatible with AWS Inferentia2 and save it to the specified output directory.",E,W,A,j,q,$,me="Once we have a compiled Sentence Transformers model, which we either exported ourselves or is available on the Hugging Face Hub, we can load it and run inference. For loading the model we can use the <code>NeuronModelForSentenceTransformers</code> class, which is an abstraction layer for the <code>SentenceTransformer</code> class. The <code>NeuronModelForSentenceTransformers</code> class will automatically pad the input to the specified <code>sequence_length</code> and run inference on AWS Inferentia2.",P,S,O,_,D,X,K,k,pe='For deploying these models in a production environment, refer to the <a href="https://www.philschmid.de/inferentia2-embeddings" rel="nofollow">Amazon SageMaker Blog</a>.',ee,V,ne;return h=new Te({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),b=new te({props:{title:"Sentence Transformers on AWS Inferentia with Optimum Neuron",local:"sentence-transformers-on-aws-inferentia-with-optimum-neuron",headingTag:"h1"}}),w=new te({props:{title:"Compile Sentence Transformers models for AWS Inferentia2",local:"compile-sentence-transformers-models-for-aws-inferentia2",headingTag:"h2"}}),J=new v({props:{code:"IXBpcCUyMGluc3RhbGwlMjBzZW50ZW5jZS10cmFuc2Zvcm1lcnM=",highlighted:"!pip install sentence-transformers",lang:"python",wrap:!1}}),g=new v({props:{code:"ZnJvbSUyMG9wdGltdW0ubmV1cm9uJTIwaW1wb3J0JTIwTmV1cm9uTW9kZWxGb3JTZW50ZW5jZVRyYW5zZm9ybWVycyUwQSUwQSUwQSUyMyUyMFNlbnRlbmNlJTIwVHJhbnNmb3JtZXJzJTIwbW9kZWwlMjBmcm9tJTIwSHVnZ2luZ0ZhY2UlMEFtb2RlbF9pZCUyMCUzRCUyMCUyMkJBQUklMkZiZ2Utc21hbGwtZW4tdjEuNSUyMiUwQWlucHV0X3NoYXBlcyUyMCUzRCUyMCU3QiUyMmJhdGNoX3NpemUlMjIlM0ElMjAxJTJDJTIwJTIyc2VxdWVuY2VfbGVuZ3RoJTIyJTNBJTIwMzg0JTdEJTIwJTIwJTIzJTIwbWFuZGF0b3J5JTIwc2hhcGVzJTBBJTBBJTIzJTIwTG9hZCUyMFRyYW5zZm9ybWVycyUyMG1vZGVsJTIwYW5kJTIwZXhwb3J0JTIwaXQlMjB0byUyMEFXUyUyMEluZmVyZW50aWEyJTBBbW9kZWwlMjAlM0QlMjBOZXVyb25Nb2RlbEZvclNlbnRlbmNlVHJhbnNmb3JtZXJzLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMGV4cG9ydCUzRFRydWUlMkMlMjAqKmlucHV0X3NoYXBlcyklMEElMEElMjMlMjBTYXZlJTIwbW9kZWwlMjB0byUyMGRpc2slMEFtb2RlbC5zYXZlX3ByZXRyYWluZWQoJTIyYmdlX2VtYl9pbmYyJTJGJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronModelForSentenceTransformers


<span class="hljs-comment"># Sentence Transformers model from HuggingFace</span>
model_id = <span class="hljs-string">&quot;BAAI/bge-small-en-v1.5&quot;</span>
input_shapes = {<span class="hljs-string">&quot;batch_size&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;sequence_length&quot;</span>: <span class="hljs-number">384</span>}  <span class="hljs-comment"># mandatory shapes</span>

<span class="hljs-comment"># Load Transformers model and export it to AWS Inferentia2</span>
model = NeuronModelForSentenceTransformers.from_pretrained(model_id, export=<span class="hljs-literal">True</span>, **input_shapes)

<span class="hljs-comment"># Save model to disk</span>
model.save_pretrained(<span class="hljs-string">&quot;bge_emb_inf2/&quot;</span>)`,lang:"python",wrap:!1}}),W=new v({props:{code:"IW9wdGltdW0tY2xpJTIwZXhwb3J0JTIwbmV1cm9uJTIwLW0lMjBCQUFJJTJGYmdlLXNtYWxsLWVuLXYxLjUlMjAtLXNlcXVlbmNlX2xlbmd0aCUyMDM4NCUyMC0tYmF0Y2hfc2l6ZSUyMDElMjAtLXRhc2slMjBmZWF0dXJlLWV4dHJhY3Rpb24lMjBiZ2VfZW1iX2luZjIlMkY=",highlighted:'!optimum-cli export neuron -m BAAI/bge-small-en-v1<span class="hljs-number">.5</span> --sequence_length <span class="hljs-number">384</span> --batch_size <span class="hljs-number">1</span> --task feature-extraction bge_emb_inf2/',lang:"python",wrap:!1}}),j=new te({props:{title:"Load compiled Sentence Transformers model and run inference",local:"load-compiled-sentence-transformers-model-and-run-inference",headingTag:"h2"}}),S=new v({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEFmcm9tJTIwb3B0aW11bS5uZXVyb24lMjBpbXBvcnQlMjBOZXVyb25Nb2RlbEZvclNlbnRlbmNlVHJhbnNmb3JtZXJzJTBBJTBBJTBBbW9kZWxfaWRfb3JfcGF0aCUyMCUzRCUyMCUyMmJnZV9lbWJfaW5mMiUyRiUyMiUwQXRva2VuaXplcl9pZCUyMCUzRCUyMCUyMkJBQUklMkZiZ2Utc21hbGwtZW4tdjEuNSUyMiUwQSUwQSUyMyUyMExvYWQlMjBtb2RlbCUyMGFuZCUyMHRva2VuaXplciUwQW1vZGVsJTIwJTNEJTIwTmV1cm9uTW9kZWxGb3JTZW50ZW5jZVRyYW5zZm9ybWVycy5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWRfb3JfcGF0aCklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCh0b2tlbml6ZXJfaWQpJTBBJTBBJTIzJTIwUnVuJTIwaW5mZXJlbmNlJTBBcHJvbXB0JTIwJTNEJTIwJTIySSUyMGxpa2UlMjB0byUyMGVhdCUyMGFwcGxlcyUyMiUwQWVuY29kZWRfaW5wdXQlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEFvdXRwdXRzJTIwJTNEJTIwbW9kZWwoKiplbmNvZGVkX2lucHV0KSUwQSUwQXRva2VuX2VtYmVkZGluZ3MlMjAlM0QlMjBvdXRwdXRzLnRva2VuX2VtYmVkZGluZ3MlMEFzZW50ZW5jZV9lbWJlZGRpbmclMjAlM0QlMjBvdXRwdXRzLnNlbnRlbmNlX2VtYmVkZGluZyUwQSUwQXByaW50KGYlMjJ0b2tlbiUyMGVtYmVkZGluZ3MlM0ElMjAlN0J0b2tlbl9lbWJlZGRpbmdzLnNoYXBlJTdEJTIyKSUyMCUyMCUyMyUyMHRvcmNoLlNpemUoJTVCMSUyQyUyMDclMkMlMjAzODQlNUQpJTBBcHJpbnQoZiUyMnNlbnRlbmNlX2VtYmVkZGluZyUzQSUyMCU3QnNlbnRlbmNlX2VtYmVkZGluZy5zaGFwZSU3RCUyMiklMjAlMjAlMjMlMjB0b3JjaC5TaXplKCU1QjElMkMlMjAzODQlNUQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronModelForSentenceTransformers


model_id_or_path = <span class="hljs-string">&quot;bge_emb_inf2/&quot;</span>
tokenizer_id = <span class="hljs-string">&quot;BAAI/bge-small-en-v1.5&quot;</span>

<span class="hljs-comment"># Load model and tokenizer</span>
model = NeuronModelForSentenceTransformers.from_pretrained(model_id_or_path)
tokenizer = AutoTokenizer.from_pretrained(tokenizer_id)

<span class="hljs-comment"># Run inference</span>
prompt = <span class="hljs-string">&quot;I like to eat apples&quot;</span>
encoded_input = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
outputs = model(**encoded_input)

token_embeddings = outputs.token_embeddings
sentence_embedding = outputs.sentence_embedding

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;token embeddings: <span class="hljs-subst">{token_embeddings.shape}</span>&quot;</span>)  <span class="hljs-comment"># torch.Size([1, 7, 384])</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;sentence_embedding: <span class="hljs-subst">{sentence_embedding.shape}</span>&quot;</span>)  <span class="hljs-comment"># torch.Size([1, 384])</span>`,lang:"python",wrap:!1}}),_=new v({props:{code:"",highlighted:"",lang:"python",wrap:!1}}),X=new te({props:{title:"Production Usage",local:"production-usage",headingTag:"h2"}}),{c(){f=o("meta"),C=l(),I=o("p"),N=l(),r(h.$$.fragment),R=l(),r(b.$$.fragment),z=l(),M=o("p"),M.innerHTML=le,G=l(),y=o("p"),y.innerHTML=ae,x=l(),r(w.$$.fragment),Q=l(),T=o("p"),T.innerHTML=oe,F=l(),r(J.$$.fragment),H=l(),Z=o("p"),Z.innerHTML=re,L=l(),r(g.$$.fragment),Y=l(),U=o("p"),U.innerHTML=ie,E=l(),r(W.$$.fragment),A=l(),r(j.$$.fragment),q=l(),$=o("p"),$.innerHTML=me,P=l(),r(S.$$.fragment),O=l(),r(_.$$.fragment),D=l(),r(X.$$.fragment),K=l(),k=o("p"),k.innerHTML=pe,ee=l(),V=o("p"),this.h()},l(e){const n=ye("svelte-u9bgzb",document.head);f=i(n,"META",{name:!0,content:!0}),n.forEach(t),C=a(e),I=i(e,"P",{}),ce(I).forEach(t),N=a(e),m(h.$$.fragment,e),R=a(e),m(b.$$.fragment,e),z=a(e),M=i(e,"P",{"data-svelte-h":!0}),B(M)!=="svelte-fnsyyf"&&(M.innerHTML=le),G=a(e),y=i(e,"P",{"data-svelte-h":!0}),B(y)!=="svelte-opaxfq"&&(y.innerHTML=ae),x=a(e),m(w.$$.fragment,e),Q=a(e),T=i(e,"P",{"data-svelte-h":!0}),B(T)!=="svelte-1cj8iq0"&&(T.innerHTML=oe),F=a(e),m(J.$$.fragment,e),H=a(e),Z=i(e,"P",{"data-svelte-h":!0}),B(Z)!=="svelte-gyp2c1"&&(Z.innerHTML=re),L=a(e),m(g.$$.fragment,e),Y=a(e),U=i(e,"P",{"data-svelte-h":!0}),B(U)!=="svelte-reiugv"&&(U.innerHTML=ie),E=a(e),m(W.$$.fragment,e),A=a(e),m(j.$$.fragment,e),q=a(e),$=i(e,"P",{"data-svelte-h":!0}),B($)!=="svelte-4dcngh"&&($.innerHTML=me),P=a(e),m(S.$$.fragment,e),O=a(e),m(_.$$.fragment,e),D=a(e),m(X.$$.fragment,e),K=a(e),k=i(e,"P",{"data-svelte-h":!0}),B(k)!=="svelte-qum7z"&&(k.innerHTML=pe),ee=a(e),V=i(e,"P",{}),ce(V).forEach(t),this.h()},h(){de(f,"name","hf:doc:metadata"),de(f,"content",Ze)},m(e,n){we(document.head,f),s(e,C,n),s(e,I,n),s(e,N,n),p(h,e,n),s(e,R,n),p(b,e,n),s(e,z,n),s(e,M,n),s(e,G,n),s(e,y,n),s(e,x,n),p(w,e,n),s(e,Q,n),s(e,T,n),s(e,F,n),p(J,e,n),s(e,H,n),s(e,Z,n),s(e,L,n),p(g,e,n),s(e,Y,n),s(e,U,n),s(e,E,n),p(W,e,n),s(e,A,n),p(j,e,n),s(e,q,n),s(e,$,n),s(e,P,n),p(S,e,n),s(e,O,n),p(_,e,n),s(e,D,n),p(X,e,n),s(e,K,n),s(e,k,n),s(e,ee,n),s(e,V,n),ne=!0},p:fe,i(e){ne||(c(h.$$.fragment,e),c(b.$$.fragment,e),c(w.$$.fragment,e),c(J.$$.fragment,e),c(g.$$.fragment,e),c(W.$$.fragment,e),c(j.$$.fragment,e),c(S.$$.fragment,e),c(_.$$.fragment,e),c(X.$$.fragment,e),ne=!0)},o(e){d(h.$$.fragment,e),d(b.$$.fragment,e),d(w.$$.fragment,e),d(J.$$.fragment,e),d(g.$$.fragment,e),d(W.$$.fragment,e),d(j.$$.fragment,e),d(S.$$.fragment,e),d(_.$$.fragment,e),d(X.$$.fragment,e),ne=!1},d(e){e&&(t(C),t(I),t(N),t(R),t(z),t(M),t(G),t(y),t(x),t(Q),t(T),t(F),t(H),t(Z),t(L),t(Y),t(U),t(E),t(A),t(q),t($),t(P),t(O),t(D),t(K),t(k),t(ee),t(V)),t(f),u(h,e),u(b,e),u(w,e),u(J,e),u(g,e),u(W,e),u(j,e),u(S,e),u(_,e),u(X,e)}}}const Ze='{"title":"Sentence Transformers on AWS Inferentia with Optimum Neuron","local":"sentence-transformers-on-aws-inferentia-with-optimum-neuron","sections":[{"title":"Compile Sentence Transformers models for AWS Inferentia2","local":"compile-sentence-transformers-models-for-aws-inferentia2","sections":[],"depth":2},{"title":"Load compiled Sentence Transformers model and run inference","local":"load-compiled-sentence-transformers-model-and-run-inference","sections":[],"depth":2},{"title":"Production Usage","local":"production-usage","sections":[],"depth":2}],"depth":1}';function ge(se){return he(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class _e extends be{constructor(f){super(),Me(this,f,ge,Je,ue,{})}}export{_e as component};