sagemaker/pr_2097/en/_app/immutable/nodes/2.eb3bc2bb.js

import{s as mt,n as ct,o as ht}from"../chunks/scheduler.aec39e6a.js";import{S as gt,i as dt,e as s,s as l,c as u,h as bt,a as r,d as n,b as i,f as pt,g as p,j as o,k as ft,l as Tt,m as a,n as f,t as m,o as c,p as h}from"../chunks/index.4ee0a2d0.js";import{C as yt,H as K,E as xt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.2422a08e.js";import{C as Ct}from"../chunks/CodeBlock.27357de2.js";function $t(We){let g,ee,O,te,d,ne,b,ae,T,De="Below you can find a listing of all the Deep Learning Containers (DLCs) available on AWS.",le,y,Ee="For each supported combination of use-case (training, inference), accelerator type (CPU, GPU, Neuron), and framework (PyTorch, TGI, TEI) containers are created.",ie,x,Ne='Neuron DLCs for training and inference on AWS Trainium and AWS Inferentia instances can be found in the <a href="https://huggingface.co/docs/optimum-neuron/en/containers" rel="nofollow">Optimum Neuron documentation</a>.',se,C,re,$,Ve="Pytorch Training DLC: For training, our DLCs are available for PyTorch via Transformers. They include support for training on GPUs and AWS AI chips with libraries such as TRL, Sentence Transformers, or Diffusers.",oe,w,Re='You can also keep track of the latest Pytorch Training DLC releases <a href="https://github.com/aws/deep-learning-containers/releases?q=huggingface-training+AND+NOT+neuronx&amp;expanded=true" rel="nofollow">here</a>.',ue,v,Ye="<thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-training:2.8.0-transformers4.56.2-gpu-py312-cu129-ubuntu22.04</td> <td>GPU</td></tr> <tr><td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training-neuronx:2.7.0-transformers4.51.0-neuronx-py310-sdk2.24.1-ubuntu22.04</td> <td>Neuron</td></tr></tbody>",pe,L,fe,_,me,M,qe="For inference, we have a general-purpose PyTorch inference DLC, for serving models trained with any of those frameworks mentioned before on CPU, GPU, and AWS AI chips.",ce,k,Je='You can also keep track of the latest Pytorch Inference DLC releases <a href="https://github.com/aws/deep-learning-containers/releases?q=huggingface-inference+AND+NOT+tgi+AND+NOT+neuronx&amp;expanded=true" rel="nofollow">here</a>.',he,P,Xe="<thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-inference:2.6.0-transformers4.51.3-cpu-py312-ubuntu22.04-</td> <td>CPU</td></tr> <tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-inference:2.6.0-transformers4.51.3-gpu-py312-cu124-ubuntu22.04</td> <td>GPU</td></tr> <tr><td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-inference-neuronx:2.7.1-transformers4.51.3-neuronx-py310-sdk2.24.1-ubuntu22.04</td> <td>Neuron</td></tr></tbody>",ge,U,de,H,Be="There is also the LLM Text Generation Inference (TGI) DLC for high-performance text generation of LLMs on GPU and AWS AI chips.",be,I,ze='You can also keep track of the latest LLM TGI DLC releases <a href="https://github.com/aws/deep-learning-containers/releases?q=tgi+AND+gpu&amp;expanded=true" rel="nofollow">here</a>.',Te,S,Fe="<thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-tgi-inference:2.7.0-tgi3.3.6-gpu-py311-cu124-ubuntu22.04</td> <td>GPU</td></tr> <tr><td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-tgi-inference:2.7.0-optimum3.3.6-neuronx-py310-ubuntu22.04</td> <td>Neuron</td></tr></tbody>",ye,j,xe,A,Ke="Finally, there is a Text Embeddings Inference (TEI) DLC for high-performance serving of embedding models on CPU and GPU.",Ce,Z,Oe="<thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>683313688378.dkr.ecr.us-east-1.amazonaws.com/2.0.1-tei1.8.2-cpu-py310-ubuntu22.04</td> <td>CPU</td></tr> <tr><td>683313688378.dkr.ecr.us-east-1.amazonaws.com/2.0.1-tei1.8.2-gpu-py310-cu122-ubuntu22.04</td> <td>GPU</td></tr></tbody>",$e,G,we,W,Qe="<strong>How to choose the right inference container for my use case?</strong>",ve,D,et='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/sagemaker/inference-dlc-decision-tree.png" alt="inference-dlc-decision-tree"/>',Le,E,tt='<em>Note:</em> See <a href="https://huggingface.co/docs/sagemaker/main/en/reference/inference-toolkit" rel="nofollow">here</a> for the list of supported task in the inference toolkit.',_e,N,nt='<em>Note:</em> Browse through the Hub to see if you model is tagged <a href="https://huggingface.co/models?other=text-generation-inference" rel="nofollow">“text-generation-inference”</a> or <a href="https://huggingface.co/models?other=text-embeddings-inference" rel="nofollow">“text-embeddings-inference”</a>',Me,V,at="<strong>How to find the URI of my container?</strong>",ke,R,lt=`The URI is built with an AWS account ID and an AWS region. Those two values need to be replaced depending on your use case.
Let’s say you want to use the training DLC for GPUs in`,Pe,Y,it='<li><code>dlc-aws-account-id</code>: The AWS account ID of the account that owns the ECR repository. You can find them in the <a href="https://github.com/aws/sagemaker-python-sdk/blob/e0b9d38e1e3b48647a02af23c4be54980e53dc61/src/sagemaker/image_uri_config/huggingface.json#L21" rel="nofollow">here</a></li> <li><code>region</code>: The AWS region where you want to use it.</li>',Ue,q,st="<strong>How to find the URI of my container but simpler?</strong>",He,J,rt="The Python SagemMaker SDK util functions are not always up to date but it is much simpler than reconstructing the image URI yourself.",Ie,X,ot='<p>[!WARNING][SageMaker Python SDK v3 has been recently released](<a href="https://github.com/aws/sagemaker-python-sdk" rel="nofollow">https://github.com/aws/sagemaker-python-sdk</a>), so unless specified otherwise, all the documentation and tutorials are still using the <a href="https://github.com/aws/sagemaker-python-sdk/tree/master-v2" rel="nofollow">SageMaker Python SDK v2</a>. We are actively working on updating all the tutorials and examples, but in the meantime make sure to install the SageMaker SDK as <code>pip install &quot;sagemaker&lt;3.0.0&quot;</code>.</p>',Se,B,je,z,ut="For Pytorch Training and Pytorch Inference DLCs, there is no such utility.",Ae,F,Ze,Q,Ge;return d=new yt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),b=new K({props:{title:"Available DLCs on AWS",local:"available-dlcs-on-aws",headingTag:"h1"}}),C=new K({props:{title:"Training",local:"training",headingTag:"h2"}}),L=new K({props:{title:"Inference",local:"inference",headingTag:"h2"}}),_=new K({props:{title:"Pytorch Inference DLC",local:"pytorch-inference-dlc",headingTag:"h3"}}),U=new K({props:{title:"LLM TGI",local:"llm-tgi",headingTag:"h3"}}),j=new K({props:{title:"Text Embedding Inference",local:"text-embedding-inference",headingTag:"h3"}}),G=new K({props:{title:"FAQ",local:"faq",headingTag:"h2"}}),B=new Ct({props:{code:"ZnJvbSUyMHNhZ2VtYWtlci5odWdnaW5nZmFjZSUyMGltcG9ydCUyMEh1Z2dpbmdGYWNlTW9kZWwlMkMlMjBnZXRfaHVnZ2luZ2ZhY2VfbGxtX2ltYWdlX3VyaSUwQSUwQXByaW50KGYlMjJUR0klMjBHUFUlM0ElMjAlN0JnZXRfaHVnZ2luZ2ZhY2VfbGxtX2ltYWdlX3VyaSgnaHVnZ2luZ2ZhY2UnKSU3RCUyMiklMEFwcmludChmJTIyVEVJJTIwR1BVJTNBJTIwJTdCZ2V0X2h1Z2dpbmdmYWNlX2xsbV9pbWFnZV91cmkoJ2h1Z2dpbmdmYWNlLXRlaScpJTdEJTIyKSUwQXByaW50KGYlMjJURUklMjBDUFUlM0ElMjAlN0JnZXRfaHVnZ2luZ2ZhY2VfbGxtX2ltYWdlX3VyaSgnaHVnZ2luZ2ZhY2UtdGVpLWNwdScpJTdEJTIyKSUwQXByaW50KGYlMjJUR0klMjBOZXVyb24lM0ElMjAlN0JnZXRfaHVnZ2luZ2ZhY2VfbGxtX2ltYWdlX3VyaSgnaHVnZ2luZ2ZhY2UtbmV1cm9ueCcpJTdEJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> sagemaker.huggingface <span class="hljs-keyword">import</span> HuggingFaceModel, get_huggingface_llm_image_uri

<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TGI GPU: <span class="hljs-subst">{get_huggingface_llm_image_uri(<span class="hljs-string">&#x27;huggingface&#x27;</span>)}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TEI GPU: <span class="hljs-subst">{get_huggingface_llm_image_uri(<span class="hljs-string">&#x27;huggingface-tei&#x27;</span>)}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TEI CPU: <span class="hljs-subst">{get_huggingface_llm_image_uri(<span class="hljs-string">&#x27;huggingface-tei-cpu&#x27;</span>)}</span>&quot;</span>)
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;TGI Neuron: <span class="hljs-subst">{get_huggingface_llm_image_uri(<span class="hljs-string">&#x27;huggingface-neuronx&#x27;</span>)}</span>&quot;</span>)`,wrap:!1}}),F=new xt({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/sagemaker/source/dlcs/available.md"}}),{c(){g=s("meta"),ee=l(),O=s("p"),te=l(),u(d.$$.fragment),ne=l(),u(b.$$.fragment),ae=l(),T=s("p"),T.textContent=De,le=l(),y=s("p"),y.textContent=Ee,ie=l(),x=s("p"),x.innerHTML=Ne,se=l(),u(C.$$.fragment),re=l(),$=s("p"),$.textContent=Ve,oe=l(),w=s("p"),w.innerHTML=Re,ue=l(),v=s("table"),v.innerHTML=Ye,pe=l(),u(L.$$.fragment),fe=l(),u(_.$$.fragment),me=l(),M=s("p"),M.textContent=qe,ce=l(),k=s("p"),k.innerHTML=Je,he=l(),P=s("table"),P.innerHTML=Xe,ge=l(),u(U.$$.fragment),de=l(),H=s("p"),H.textContent=Be,be=l(),I=s("p"),I.innerHTML=ze,Te=l(),S=s("table"),S.innerHTML=Fe,ye=l(),u(j.$$.fragment),xe=l(),A=s("p"),A.textContent=Ke,Ce=l(),Z=s("table"),Z.innerHTML=Oe,$e=l(),u(G.$$.fragment),we=l(),W=s("p"),W.innerHTML=Qe,ve=l(),D=s("p"),D.innerHTML=et,Le=l(),E=s("p"),E.innerHTML=tt,_e=l(),N=s("p"),N.innerHTML=nt,Me=l(),V=s("p"),V.innerHTML=at,ke=l(),R=s("p"),R.textContent=lt,Pe=l(),Y=s("ul"),Y.innerHTML=it,Ue=l(),q=s("p"),q.innerHTML=st,He=l(),J=s("p"),J.textContent=rt,Ie=l(),X=s("blockquote"),X.innerHTML=ot,Se=l(),u(B.$$.fragment),je=l(),z=s("p"),z.textContent=ut,Ae=l(),u(F.$$.fragment),Ze=l(),Q=s("p"),this.h()},l(e){const t=bt("svelte-u9bgzb",document.head);g=r(t,"META",{name:!0,content:!0}),t.forEach(n),ee=i(e),O=r(e,"P",{}),pt(O).forEach(n),te=i(e),p(d.$$.fragment,e),ne=i(e),p(b.$$.fragment,e),ae=i(e),T=r(e,"P",{"data-svelte-h":!0}),o(T)!=="svelte-1eptnyh"&&(T.textContent=De),le=i(e),y=r(e,"P",{"data-svelte-h":!0}),o(y)!=="svelte-nsjrcm"&&(y.textContent=Ee),ie=i(e),x=r(e,"P",{"data-svelte-h":!0}),o(x)!=="svelte-1mi0ysm"&&(x.innerHTML=Ne),se=i(e),p(C.$$.fragment,e),re=i(e),$=r(e,"P",{"data-svelte-h":!0}),o($)!=="svelte-dmm2ao"&&($.textContent=Ve),oe=i(e),w=r(e,"P",{"data-svelte-h":!0}),o(w)!=="svelte-4leueb"&&(w.innerHTML=Re),ue=i(e),v=r(e,"TABLE",{"data-svelte-h":!0}),o(v)!=="svelte-1tg5jwt"&&(v.innerHTML=Ye),pe=i(e),p(L.$$.fragment,e),fe=i(e),p(_.$$.fragment,e),me=i(e),M=r(e,"P",{"data-svelte-h":!0}),o(M)!=="svelte-i6peck"&&(M.textContent=qe),ce=i(e),k=r(e,"P",{"data-svelte-h":!0}),o(k)!=="svelte-5mbpze"&&(k.innerHTML=Je),he=i(e),P=r(e,"TABLE",{"data-svelte-h":!0}),o(P)!=="svelte-uooqvv"&&(P.innerHTML=Xe),ge=i(e),p(U.$$.fragment,e),de=i(e),H=r(e,"P",{"data-svelte-h":!0}),o(H)!=="svelte-z6dexv"&&(H.textContent=Be),be=i(e),I=r(e,"P",{"data-svelte-h":!0}),o(I)!=="svelte-4njdnv"&&(I.innerHTML=ze),Te=i(e),S=r(e,"TABLE",{"data-svelte-h":!0}),o(S)!=="svelte-44tt02"&&(S.innerHTML=Fe),ye=i(e),p(j.$$.fragment,e),xe=i(e),A=r(e,"P",{"data-svelte-h":!0}),o(A)!=="svelte-db04rk"&&(A.textContent=Ke),Ce=i(e),Z=r(e,"TABLE",{"data-svelte-h":!0}),o(Z)!=="svelte-yh0mt5"&&(Z.innerHTML=Oe),$e=i(e),p(G.$$.fragment,e),we=i(e),W=r(e,"P",{"data-svelte-h":!0}),o(W)!=="svelte-qjgyy3"&&(W.innerHTML=Qe),ve=i(e),D=r(e,"P",{"data-svelte-h":!0}),o(D)!=="svelte-86ji6y"&&(D.innerHTML=et),Le=i(e),E=r(e,"P",{"data-svelte-h":!0}),o(E)!=="svelte-1w4dc1f"&&(E.innerHTML=tt),_e=i(e),N=r(e,"P",{"data-svelte-h":!0}),o(N)!=="svelte-8p8z10"&&(N.innerHTML=nt),Me=i(e),V=r(e,"P",{"data-svelte-h":!0}),o(V)!=="svelte-1he7r2b"&&(V.innerHTML=at),ke=i(e),R=r(e,"P",{"data-svelte-h":!0}),o(R)!=="svelte-1rzekti"&&(R.textContent=lt),Pe=i(e),Y=r(e,"UL",{"data-svelte-h":!0}),o(Y)!=="svelte-tq759t"&&(Y.innerHTML=it),Ue=i(e),q=r(e,"P",{"data-svelte-h":!0}),o(q)!=="svelte-1kgkd98"&&(q.innerHTML=st),He=i(e),J=r(e,"P",{"data-svelte-h":!0}),o(J)!=="svelte-l6c2ju"&&(J.textContent=rt),Ie=i(e),X=r(e,"BLOCKQUOTE",{"data-svelte-h":!0}),o(X)!=="svelte-p7runb"&&(X.innerHTML=ot),Se=i(e),p(B.$$.fragment,e),je=i(e),z=r(e,"P",{"data-svelte-h":!0}),o(z)!=="svelte-1om219l"&&(z.textContent=ut),Ae=i(e),p(F.$$.fragment,e),Ze=i(e),Q=r(e,"P",{}),pt(Q).forEach(n),this.h()},h(){ft(g,"name","hf:doc:metadata"),ft(g,"content",wt)},m(e,t){Tt(document.head,g),a(e,ee,t),a(e,O,t),a(e,te,t),f(d,e,t),a(e,ne,t),f(b,e,t),a(e,ae,t),a(e,T,t),a(e,le,t),a(e,y,t),a(e,ie,t),a(e,x,t),a(e,se,t),f(C,e,t),a(e,re,t),a(e,$,t),a(e,oe,t),a(e,w,t),a(e,ue,t),a(e,v,t),a(e,pe,t),f(L,e,t),a(e,fe,t),f(_,e,t),a(e,me,t),a(e,M,t),a(e,ce,t),a(e,k,t),a(e,he,t),a(e,P,t),a(e,ge,t),f(U,e,t),a(e,de,t),a(e,H,t),a(e,be,t),a(e,I,t),a(e,Te,t),a(e,S,t),a(e,ye,t),f(j,e,t),a(e,xe,t),a(e,A,t),a(e,Ce,t),a(e,Z,t),a(e,$e,t),f(G,e,t),a(e,we,t),a(e,W,t),a(e,ve,t),a(e,D,t),a(e,Le,t),a(e,E,t),a(e,_e,t),a(e,N,t),a(e,Me,t),a(e,V,t),a(e,ke,t),a(e,R,t),a(e,Pe,t),a(e,Y,t),a(e,Ue,t),a(e,q,t),a(e,He,t),a(e,J,t),a(e,Ie,t),a(e,X,t),a(e,Se,t),f(B,e,t),a(e,je,t),a(e,z,t),a(e,Ae,t),f(F,e,t),a(e,Ze,t),a(e,Q,t),Ge=!0},p:ct,i(e){Ge||(m(d.$$.fragment,e),m(b.$$.fragment,e),m(C.$$.fragment,e),m(L.$$.fragment,e),m(_.$$.fragment,e),m(U.$$.fragment,e),m(j.$$.fragment,e),m(G.$$.fragment,e),m(B.$$.fragment,e),m(F.$$.fragment,e),Ge=!0)},o(e){c(d.$$.fragment,e),c(b.$$.fragment,e),c(C.$$.fragment,e),c(L.$$.fragment,e),c(_.$$.fragment,e),c(U.$$.fragment,e),c(j.$$.fragment,e),c(G.$$.fragment,e),c(B.$$.fragment,e),c(F.$$.fragment,e),Ge=!1},d(e){e&&(n(ee),n(O),n(te),n(ne),n(ae),n(T),n(le),n(y),n(ie),n(x),n(se),n(re),n($),n(oe),n(w),n(ue),n(v),n(pe),n(fe),n(me),n(M),n(ce),n(k),n(he),n(P),n(ge),n(de),n(H),n(be),n(I),n(Te),n(S),n(ye),n(xe),n(A),n(Ce),n(Z),n($e),n(we),n(W),n(ve),n(D),n(Le),n(E),n(_e),n(N),n(Me),n(V),n(ke),n(R),n(Pe),n(Y),n(Ue),n(q),n(He),n(J),n(Ie),n(X),n(Se),n(je),n(z),n(Ae),n(Ze),n(Q)),n(g),h(d,e),h(b,e),h(C,e),h(L,e),h(_,e),h(U,e),h(j,e),h(G,e),h(B,e),h(F,e)}}}const wt='{"title":"Available DLCs on AWS","local":"available-dlcs-on-aws","sections":[{"title":"Training","local":"training","sections":[],"depth":2},{"title":"Inference","local":"inference","sections":[{"title":"Pytorch Inference DLC","local":"pytorch-inference-dlc","sections":[],"depth":3},{"title":"LLM TGI","local":"llm-tgi","sections":[],"depth":3},{"title":"Text Embedding Inference","local":"text-embedding-inference","sections":[],"depth":3}],"depth":2},{"title":"FAQ","local":"faq","sections":[],"depth":2}],"depth":1}';function vt(We){return ht(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends gt{constructor(g){super(),dt(this,g,vt,$t,mt,{})}}export{Pt as component};