sagemaker/pr_2556/en/_app/immutable/nodes/2.ea86efdf.js

import{s as Tt,n as xt,o as $t}from"../chunks/scheduler.aec39e6a.js";import{S as Ct,i as wt,e as s,s as l,c as u,h as Lt,a as r,d as n,b as i,f as yt,g as f,j as o,k as vt,l as _t,m as a,n as p,t as m,o as h,p as c}from"../chunks/index.4ee0a2d0.js";import{C as Mt,H as d,E as kt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.6ca9a012.js";import{C as Pt}from"../chunks/CodeBlock.424a7a42.js";function Ut(qe){let g,ae,te,le,b,ie,y,se,v,Je="Below you can find a listing of our latest Deep Learning Containers (DLCs) available on AWS.",re,T,Ye="For each supported combination of use-case (training, inference), accelerator type (CPU, GPU, Neuron), and framework (PyTorch, TGI, TEI) containers are created.",oe,x,Xe='Neuron DLCs for training and inference on AWS Trainium and AWS Inferentia instances can be found in the <a href="https://huggingface.co/docs/optimum-neuron/en/containers" rel="nofollow">Optimum Neuron documentation</a>.',ue,$,Be='If you want to keep track of all our available DLCs, you can also check the <a href="https://aws.github.io/deep-learning-containers/reference/available_images#huggingface-pytorch-training" rel="nofollow">AWS Deep Learning Containers releases</a> page.',fe,C,pe,w,ze="For training, the DLCs are available for PyTorch via Transformers. They include GPUs and AWS AI chips support, with libraries such as TRL, Sentence Transformers, or Diffusers.",me,L,Fe='You can also keep track of the latest PyTorch Training DLC releases <a href="https://github.com/aws/deep-learning-containers/releases?q=huggingface-training+AND+NOT+neuronx&amp;expanded=true" rel="nofollow">here</a>.',he,_,Ke="<thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-training:2.9.0-transformers5.3.0-gpu-py312-cu130-ubuntu22.04</td> <td>GPU</td></tr> <tr><td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training-neuronx:2.8.0-transformers4.55.4-neuronx-py310-sdk2.26.0-ubuntu22.04</td> <td>Neuron</td></tr></tbody>",ce,M,ge,k,de,P,Qe="For inference, there is a general-purpose PyTorch inference DLC, for serving models trained with any of those frameworks mentioned before on CPU, GPU, and AWS AI chips.",be,U,Oe="<thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-inference:2.6.0-transformers4.51.3-cpu-py312-ubuntu22.04-</td> <td>CPU</td></tr> <tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-inference:2.6.0-transformers4.51.3-gpu-py312-cu124-ubuntu22.04</td> <td>GPU</td></tr> <tr><td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-inference-neuronx:2.8.0-transformers4.55.4-neuronx-py310-sdk2.26.0-ubuntu22.04</td> <td>Neuron</td></tr></tbody>",ye,H,ve,S,et="In case you want to serve text generation models with vLLM, there are specific DLCs available for GPU and AWS AI chips.",Te,I,tt="<thead><tr><th>vLLM version</th> <th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>0.17.0</td> <td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-vllm:0.17.0-transformers4.57.5-gpu-py312-cu129-ubuntu22.04</td> <td>GPU</td></tr> <tr><td>0.11.0</td> <td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-vllm-inference-neuronx:0.11.0-optimum0.4.5-neuronx-py310-sdk2.26.1-ubuntu22.04</td> <td>Neuron</td></tr></tbody>",xe,A,$e,Z,nt="There is also a specific DLC for serving models with SGLang on GPU.",Ce,j,at="<thead><tr><th>SGLang version</th> <th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>0.5.8</td> <td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-sglang:0.5.8-transformers4.57.3-gpu-py312-cu129-ubuntu24.04</td> <td>GPU</td></tr></tbody>",we,G,Le,W,lt="Finally, there is the Text Embeddings Inference (TEI) DLC for high-performance serving of embedding models on CPU and GPU.",_e,E,it="<thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>683313688378.dkr.ecr.us-east-1.amazonaws.com/2.0.1-tei1.8.2-cpu-py310-ubuntu22.04</td> <td>CPU</td></tr> <tr><td>683313688378.dkr.ecr.us-east-1.amazonaws.com/tei-cpu:2.0.1-tei1.8.2-cpu-py310-ubuntu22.04</td> <td>CPU</td></tr> <tr><td>683313688378.dkr.ecr.us-east-1.amazonaws.com/tei:2.0.1-tei1.8.2-gpu-py310-cu122-ubuntu22.04</td> <td>GPU</td></tr></tbody>",Me,V,ke,D,st="<strong>How to choose the right inference container for my use case?</strong>",Pe,R,rt='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/sagemaker/inference-dlc-decision-tree.png" alt="inference-dlc-decision-tree"/>',Ue,N,ot='<em>Note:</em> See <a href="https://huggingface.co/docs/sagemaker/main/en/reference/inference-toolkit" rel="nofollow">here</a> for the list of supported task in the inference toolkit.',He,q,ut='<em>Note:</em> Browse through the Hub to see if your model is tagged <a href="https://huggingface.co/models?other=text-generation-inference" rel="nofollow">“text-generation-inference”</a> or <a href="https://huggingface.co/models?other=text-embeddings-inference" rel="nofollow">“text-embeddings-inference”</a>.',Se,J,ft="<strong>How to find the URI of my container?</strong>",Ie,Y,pt="The URI is built with an AWS account ID and an AWS region. Those two values need to be replaced depending on your use case.",Ae,X,mt="Let’s say you want to use the training DLC for GPUs:",Ze,B,ht='<li><code>dlc-aws-account-id</code>: The AWS account ID of the account that owns the ECR repository. You can find them in the <a href="https://github.com/aws/sagemaker-python-sdk/blob/e0b9d38e1e3b48647a02af23c4be54980e53dc61/src/sagemaker/image_uri_config/huggingface.json#L21" rel="nofollow">here</a></li> <li><code>region</code>: The AWS region where you want to use it.</li>',je,z,ct="<strong>How to find the URI of my container but simpler?</strong>",Ge,F,gt="The Python SageMaker SDK util functions are not always up to date but it is much simpler than reconstructing the image URI yourself.",We,K,dt='<p>[!WARNING][SageMaker Python SDK v3 has been recently released](<a href="https://github.com/aws/sagemaker-python-sdk" rel="nofollow">https://github.com/aws/sagemaker-python-sdk</a>), so unless specified otherwise, all the documentation and tutorials are still using the <a href="https://github.com/aws/sagemaker-python-sdk/tree/master-v2" rel="nofollow">SageMaker Python SDK v2</a>. We are actively working on updating all the tutorials and examples, but in the meantime make sure to install the SageMaker SDK as <code>pip install &quot;sagemaker&lt;3.0.0&quot;</code>.</p>',Ee,Q,Ve,O,bt="For PyTorch Training and PyTorch Inference DLCs, there is no such utility.",De,ee,Re,ne,Ne;return b=new Mt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),y=new d({props:{title:"Available DLCs on AWS",local:"available-dlcs-on-aws",headingTag:"h1"}}),C=new d({props:{title:"Training",local:"training",headingTag:"h2"}}),M=new d({props:{title:"Inference",local:"inference",headingTag:"h2"}}),k=new d({props:{title:"PyTorch Inference",local:"pytorch-inference",headingTag:"h3"}}),H=new d({props:{title:"vLLM",local:"vllm",headingTag:"h3"}}),A=new d({props:{title:"SGLang",local:"sglang",headingTag:"h3"}}),G=new d({props:{title:"Text Embeddings Inference",local:"text-embeddings-inference",headingTag:"h3"}}),V=new d({props:{title:"FAQ",local:"faq",headingTag:"h2"}}),Q=new Pt({props:{code:"ZnJvbSUyMHNhZ2VtYWtlci5odWdnaW5nZmFjZSUyMGltcG9ydCUyMEh1Z2dpbmdGYWNlTW9kZWwlMkMlMjBnZXRfaHVnZ2luZ2ZhY2VfbGxtX2ltYWdlX3VyaSUwQSUwQXByaW50KGYlMjJUR0klMjBHUFUlM0ElMjAlN0JnZXRfaHVnZ2luZ2ZhY2VfbGxtX2ltYWdlX3VyaSgnaHVnZ2luZ2ZhY2UnKSU3RCUyMiklMEFwcmludChmJTIyVEVJJTIwR1BVJTNBJTIwJTdCZ2V0X2h1Z2dpbmdmYWNlX2xsbV9pbWFnZV91cmkoJ2h1Z2dpbmdmYWNlLXRlaScpJTdEJTIyKSUwQXByaW50KGYlMjJURUklMjBDUFUlM0ElMjAlN0JnZXRfaHVnZ2luZ2ZhY2VfbGxtX2ltYWdlX3VyaSgnaHVnZ2luZ2ZhY2UtdGVpLWNwdScpJTdEJTIyKSUwQXByaW50KGYlMjJUR0klMjBOZXVyb24lM0ElMjAlN0JnZXRfaHVnZ2luZ2ZhY2VfbGxtX2ltYWdlX3VyaSgnaHVnZ2luZ2ZhY2UtbmV1cm9ueCcpJTdEJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> sagemaker.huggingface <span class="hljs-keyword">import</span> HuggingFaceModel, get_huggingface_llm_image_uri

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TGI GPU: <span class="hljs-subst">{get_huggingface_llm_image_uri(<span class="hljs-string">&#x27;huggingface&#x27;</span>)}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TEI GPU: <span class="hljs-subst">{get_huggingface_llm_image_uri(<span class="hljs-string">&#x27;huggingface-tei&#x27;</span>)}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TEI CPU: <span class="hljs-subst">{get_huggingface_llm_image_uri(<span class="hljs-string">&#x27;huggingface-tei-cpu&#x27;</span>)}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TGI Neuron: <span class="hljs-subst">{get_huggingface_llm_image_uri(<span class="hljs-string">&#x27;huggingface-neuronx&#x27;</span>)}</span>&quot;</span>)`,lang:"python",wrap:!1}}),ee=new kt({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/sagemaker/source/dlcs/available.md"}}),{c(){g=s("meta"),ae=l(),te=s("p"),le=l(),u(b.$$.fragment),ie=l(),u(y.$$.fragment),se=l(),v=s("p"),v.textContent=Je,re=l(),T=s("p"),T.textContent=Ye,oe=l(),x=s("p"),x.innerHTML=Xe,ue=l(),$=s("p"),$.innerHTML=Be,fe=l(),u(C.$$.fragment),pe=l(),w=s("p"),w.textContent=ze,me=l(),L=s("p"),L.innerHTML=Fe,he=l(),_=s("table"),_.innerHTML=Ke,ce=l(),u(M.$$.fragment),ge=l(),u(k.$$.fragment),de=l(),P=s("p"),P.textContent=Qe,be=l(),U=s("table"),U.innerHTML=Oe,ye=l(),u(H.$$.fragment),ve=l(),S=s("p"),S.textContent=et,Te=l(),I=s("table"),I.innerHTML=tt,xe=l(),u(A.$$.fragment),$e=l(),Z=s("p"),Z.textContent=nt,Ce=l(),j=s("table"),j.innerHTML=at,we=l(),u(G.$$.fragment),Le=l(),W=s("p"),W.textContent=lt,_e=l(),E=s("table"),E.innerHTML=it,Me=l(),u(V.$$.fragment),ke=l(),D=s("p"),D.innerHTML=st,Pe=l(),R=s("p"),R.innerHTML=rt,Ue=l(),N=s("p"),N.innerHTML=ot,He=l(),q=s("p"),q.innerHTML=ut,Se=l(),J=s("p"),J.innerHTML=ft,Ie=l(),Y=s("p"),Y.textContent=pt,Ae=l(),X=s("p"),X.textContent=mt,Ze=l(),B=s("ul"),B.innerHTML=ht,je=l(),z=s("p"),z.innerHTML=ct,Ge=l(),F=s("p"),F.textContent=gt,We=l(),K=s("blockquote"),K.innerHTML=dt,Ee=l(),u(Q.$$.fragment),Ve=l(),O=s("p"),O.textContent=bt,De=l(),u(ee.$$.fragment),Re=l(),ne=s("p"),this.h()},l(e){const t=Lt("svelte-u9bgzb",document.head);g=r(t,"META",{name:!0,content:!0}),t.forEach(n),ae=i(e),te=r(e,"P",{}),yt(te).forEach(n),le=i(e),f(b.$$.fragment,e),ie=i(e),f(y.$$.fragment,e),se=i(e),v=r(e,"P",{"data-svelte-h":!0}),o(v)!=="svelte-uq6kdu"&&(v.textContent=Je),re=i(e),T=r(e,"P",{"data-svelte-h":!0}),o(T)!=="svelte-nsjrcm"&&(T.textContent=Ye),oe=i(e),x=r(e,"P",{"data-svelte-h":!0}),o(x)!=="svelte-1mi0ysm"&&(x.innerHTML=Xe),ue=i(e),$=r(e,"P",{"data-svelte-h":!0}),o($)!=="svelte-vu21fv"&&($.innerHTML=Be),fe=i(e),f(C.$$.fragment,e),pe=i(e),w=r(e,"P",{"data-svelte-h":!0}),o(w)!=="svelte-kuh6r"&&(w.textContent=ze),me=i(e),L=r(e,"P",{"data-svelte-h":!0}),o(L)!=="svelte-w6v2gz"&&(L.innerHTML=Fe),he=i(e),_=r(e,"TABLE",{"data-svelte-h":!0}),o(_)!=="svelte-e08247"&&(_.innerHTML=Ke),ce=i(e),f(M.$$.fragment,e),ge=i(e),f(k.$$.fragment,e),de=i(e),P=r(e,"P",{"data-svelte-h":!0}),o(P)!=="svelte-dfyqlg"&&(P.textContent=Qe),be=i(e),U=r(e,"TABLE",{"data-svelte-h":!0}),o(U)!=="svelte-1yo3llp"&&(U.innerHTML=Oe),ye=i(e),f(H.$$.fragment,e),ve=i(e),S=r(e,"P",{"data-svelte-h":!0}),o(S)!=="svelte-1ptami9"&&(S.textContent=et),Te=i(e),I=r(e,"TABLE",{"data-svelte-h":!0}),o(I)!=="svelte-1i4erop"&&(I.innerHTML=tt),xe=i(e),f(A.$$.fragment,e),$e=i(e),Z=r(e,"P",{"data-svelte-h":!0}),o(Z)!=="svelte-1fyryv7"&&(Z.textContent=nt),Ce=i(e),j=r(e,"TABLE",{"data-svelte-h":!0}),o(j)!=="svelte-yb92ek"&&(j.innerHTML=at),we=i(e),f(G.$$.fragment,e),Le=i(e),W=r(e,"P",{"data-svelte-h":!0}),o(W)!=="svelte-dig27s"&&(W.textContent=lt),_e=i(e),E=r(e,"TABLE",{"data-svelte-h":!0}),o(E)!=="svelte-2joef3"&&(E.innerHTML=it),Me=i(e),f(V.$$.fragment,e),ke=i(e),D=r(e,"P",{"data-svelte-h":!0}),o(D)!=="svelte-qjgyy3"&&(D.innerHTML=st),Pe=i(e),R=r(e,"P",{"data-svelte-h":!0}),o(R)!=="svelte-86ji6y"&&(R.innerHTML=rt),Ue=i(e),N=r(e,"P",{"data-svelte-h":!0}),o(N)!=="svelte-1w4dc1f"&&(N.innerHTML=ot),He=i(e),q=r(e,"P",{"data-svelte-h":!0}),o(q)!=="svelte-t7yy3s"&&(q.innerHTML=ut),Se=i(e),J=r(e,"P",{"data-svelte-h":!0}),o(J)!=="svelte-1he7r2b"&&(J.innerHTML=ft),Ie=i(e),Y=r(e,"P",{"data-svelte-h":!0}),o(Y)!=="svelte-1rsk6qo"&&(Y.textContent=pt),Ae=i(e),X=r(e,"P",{"data-svelte-h":!0}),o(X)!=="svelte-89omgb"&&(X.textContent=mt),Ze=i(e),B=r(e,"UL",{"data-svelte-h":!0}),o(B)!=="svelte-tq759t"&&(B.innerHTML=ht),je=i(e),z=r(e,"P",{"data-svelte-h":!0}),o(z)!=="svelte-1kgkd98"&&(z.innerHTML=ct),Ge=i(e),F=r(e,"P",{"data-svelte-h":!0}),o(F)!=="svelte-2a5ny7"&&(F.textContent=gt),We=i(e),K=r(e,"BLOCKQUOTE",{"data-svelte-h":!0}),o(K)!=="svelte-p7runb"&&(K.innerHTML=dt),Ee=i(e),f(Q.$$.fragment,e),Ve=i(e),O=r(e,"P",{"data-svelte-h":!0}),o(O)!=="svelte-mm15hx"&&(O.textContent=bt),De=i(e),f(ee.$$.fragment,e),Re=i(e),ne=r(e,"P",{}),yt(ne).forEach(n),this.h()},h(){vt(g,"name","hf:doc:metadata"),vt(g,"content",Ht)},m(e,t){_t(document.head,g),a(e,ae,t),a(e,te,t),a(e,le,t),p(b,e,t),a(e,ie,t),p(y,e,t),a(e,se,t),a(e,v,t),a(e,re,t),a(e,T,t),a(e,oe,t),a(e,x,t),a(e,ue,t),a(e,$,t),a(e,fe,t),p(C,e,t),a(e,pe,t),a(e,w,t),a(e,me,t),a(e,L,t),a(e,he,t),a(e,_,t),a(e,ce,t),p(M,e,t),a(e,ge,t),p(k,e,t),a(e,de,t),a(e,P,t),a(e,be,t),a(e,U,t),a(e,ye,t),p(H,e,t),a(e,ve,t),a(e,S,t),a(e,Te,t),a(e,I,t),a(e,xe,t),p(A,e,t),a(e,$e,t),a(e,Z,t),a(e,Ce,t),a(e,j,t),a(e,we,t),p(G,e,t),a(e,Le,t),a(e,W,t),a(e,_e,t),a(e,E,t),a(e,Me,t),p(V,e,t),a(e,ke,t),a(e,D,t),a(e,Pe,t),a(e,R,t),a(e,Ue,t),a(e,N,t),a(e,He,t),a(e,q,t),a(e,Se,t),a(e,J,t),a(e,Ie,t),a(e,Y,t),a(e,Ae,t),a(e,X,t),a(e,Ze,t),a(e,B,t),a(e,je,t),a(e,z,t),a(e,Ge,t),a(e,F,t),a(e,We,t),a(e,K,t),a(e,Ee,t),p(Q,e,t),a(e,Ve,t),a(e,O,t),a(e,De,t),p(ee,e,t),a(e,Re,t),a(e,ne,t),Ne=!0},p:xt,i(e){Ne||(m(b.$$.fragment,e),m(y.$$.fragment,e),m(C.$$.fragment,e),m(M.$$.fragment,e),m(k.$$.fragment,e),m(H.$$.fragment,e),m(A.$$.fragment,e),m(G.$$.fragment,e),m(V.$$.fragment,e),m(Q.$$.fragment,e),m(ee.$$.fragment,e),Ne=!0)},o(e){h(b.$$.fragment,e),h(y.$$.fragment,e),h(C.$$.fragment,e),h(M.$$.fragment,e),h(k.$$.fragment,e),h(H.$$.fragment,e),h(A.$$.fragment,e),h(G.$$.fragment,e),h(V.$$.fragment,e),h(Q.$$.fragment,e),h(ee.$$.fragment,e),Ne=!1},d(e){e&&(n(ae),n(te),n(le),n(ie),n(se),n(v),n(re),n(T),n(oe),n(x),n(ue),n($),n(fe),n(pe),n(w),n(me),n(L),n(he),n(_),n(ce),n(ge),n(de),n(P),n(be),n(U),n(ye),n(ve),n(S),n(Te),n(I),n(xe),n($e),n(Z),n(Ce),n(j),n(we),n(Le),n(W),n(_e),n(E),n(Me),n(ke),n(D),n(Pe),n(R),n(Ue),n(N),n(He),n(q),n(Se),n(J),n(Ie),n(Y),n(Ae),n(X),n(Ze),n(B),n(je),n(z),n(Ge),n(F),n(We),n(K),n(Ee),n(Ve),n(O),n(De),n(Re),n(ne)),n(g),c(b,e),c(y,e),c(C,e),c(M,e),c(k,e),c(H,e),c(A,e),c(G,e),c(V,e),c(Q,e),c(ee,e)}}}const Ht='{"title":"Available DLCs on AWS","local":"available-dlcs-on-aws","sections":[{"title":"Training","local":"training","sections":[],"depth":2},{"title":"Inference","local":"inference","sections":[{"title":"PyTorch Inference","local":"pytorch-inference","sections":[],"depth":3},{"title":"vLLM","local":"vllm","sections":[],"depth":3},{"title":"SGLang","local":"sglang","sections":[],"depth":3},{"title":"Text Embeddings Inference","local":"text-embeddings-inference","sections":[],"depth":3}],"depth":2},{"title":"FAQ","local":"faq","sections":[],"depth":2}],"depth":1}';function St(qe){return $t(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Gt extends Ct{constructor(g){super(),wt(this,g,St,Ut,Tt,{})}}export{Gt as component};