Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / sagemaker /pr_2585 /en /dlcs /available.html

HuggingFaceDocBuilder

about 1 month ago

download

raw

25.5 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Available DLCs on AWS","local":"available-dlcs-on-aws","sections":[{"title":"Training","local":"training","sections":[],"depth":2},{"title":"Inference","local":"inference","sections":[{"title":"PyTorch Inference","local":"pytorch-inference","sections":[],"depth":3},{"title":"vLLM","local":"vllm","sections":[],"depth":3},{"title":"SGLang","local":"sglang","sections":[],"depth":3},{"title":"Text Embeddings Inference","local":"text-embeddings-inference","sections":[],"depth":3}],"depth":2},{"title":"FAQ","local":"faq","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/sagemaker/pr_2585/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/entry/start.8bec021f.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/chunks/scheduler.aec39e6a.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/chunks/singletons.dbd6b9f0.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/chunks/paths.98b9d0e7.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/entry/app.2d6e6bb7.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/chunks/preload-helper.a4df83e9.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/chunks/index.4ee0a2d0.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/nodes/0.a7dfdf8a.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/nodes/2.8ae834b5.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.c07de156.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2585/en/_app/immutable/chunks/CodeBlock.22855481.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Available DLCs on AWS","local":"available-dlcs-on-aws","sections":[{"title":"Training","local":"training","sections":[],"depth":2},{"title":"Inference","local":"inference","sections":[{"title":"PyTorch Inference","local":"pytorch-inference","sections":[],"depth":3},{"title":"vLLM","local":"vllm","sections":[],"depth":3},{"title":"SGLang","local":"sglang","sections":[],"depth":3},{"title":"Text Embeddings Inference","local":"text-embeddings-inference","sections":[],"depth":3}],"depth":2},{"title":"FAQ","local":"faq","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="available-dlcs-on-aws" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#available-dlcs-on-aws"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Available DLCs on AWS</span></h1> <p data-svelte-h="svelte-uq6kdu">Below you can find a listing of our latest Deep Learning Containers (DLCs) available on AWS.</p> <p data-svelte-h="svelte-nsjrcm">For each supported combination of use-case (training, inference), accelerator type (CPU, GPU, Neuron), and framework (PyTorch, TGI, TEI) containers are created.</p> <p data-svelte-h="svelte-1mi0ysm">Neuron DLCs for training and inference on AWS Trainium and AWS Inferentia instances can be found in the <a href="https://huggingface.co/docs/optimum-neuron/en/containers" rel="nofollow">Optimum Neuron documentation</a>.</p> <p data-svelte-h="svelte-vu21fv">If you want to keep track of all our available DLCs, you can also check the <a href="https://aws.github.io/deep-learning-containers/reference/available_images#huggingface-pytorch-training" rel="nofollow">AWS Deep Learning Containers releases</a> page.</p> <h2 class="relative group"><a id="training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#training"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Training</span></h2> <p data-svelte-h="svelte-kuh6r">For training, the DLCs are available for PyTorch via Transformers. They include GPUs and AWS AI chips support, with libraries such as TRL, Sentence Transformers, or Diffusers.</p> <p data-svelte-h="svelte-w6v2gz">You can also keep track of the latest PyTorch Training DLC releases <a href="https://github.com/aws/deep-learning-containers/releases?q=huggingface-training+AND+NOT+neuronx&expanded=true" rel="nofollow">here</a>.</p> <table data-svelte-h="svelte-e08247"><thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-training:2.9.0-transformers5.3.0-gpu-py312-cu130-ubuntu22.04</td> <td>GPU</td></tr> <tr><td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training-neuronx:2.8.0-transformers4.55.4-neuronx-py310-sdk2.26.0-ubuntu22.04</td> <td>Neuron</td></tr></tbody></table> <h2 class="relative group"><a id="inference" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inference"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inference</span></h2> <h3 class="relative group"><a id="pytorch-inference" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pytorch-inference"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>PyTorch Inference</span></h3> <p data-svelte-h="svelte-dfyqlg">For inference, there is a general-purpose PyTorch inference DLC, for serving models trained with any of those frameworks mentioned before on CPU, GPU, and AWS AI chips.</p> <table data-svelte-h="svelte-1rt79ei"><thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-inference:2.6.0-transformers4.51.3-cpu-py312-ubuntu22.04</td> <td>CPU</td></tr> <tr><td>763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-inference:2.6.0-transformers4.51.3-gpu-py312-cu124-ubuntu22.04</td> <td>GPU</td></tr> <tr><td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-inference-neuronx:2.8.0-transformers4.55.4-neuronx-py310-sdk2.26.0-ubuntu22.04</td> <td>Neuron</td></tr></tbody></table> <h3 class="relative group"><a id="vllm" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#vllm"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>vLLM</span></h3> <p data-svelte-h="svelte-1ptami9">In case you want to serve text generation models with vLLM, there are specific DLCs available for GPU and AWS AI chips.</p> <table data-svelte-h="svelte-1i4erop"><thead><tr><th>vLLM version</th> <th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>0.17.0</td> <td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-vllm:0.17.0-transformers4.57.5-gpu-py312-cu129-ubuntu22.04</td> <td>GPU</td></tr> <tr><td>0.11.0</td> <td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-vllm-inference-neuronx:0.11.0-optimum0.4.5-neuronx-py310-sdk2.26.1-ubuntu22.04</td> <td>Neuron</td></tr></tbody></table> <h3 class="relative group"><a id="sglang" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#sglang"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>SGLang</span></h3> <p data-svelte-h="svelte-1fyryv7">There is also a specific DLC for serving models with SGLang on GPU.</p> <table data-svelte-h="svelte-yb92ek"><thead><tr><th>SGLang version</th> <th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>0.5.8</td> <td>763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-sglang:0.5.8-transformers4.57.3-gpu-py312-cu129-ubuntu24.04</td> <td>GPU</td></tr></tbody></table> <h3 class="relative group"><a id="text-embeddings-inference" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-embeddings-inference"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text Embeddings Inference</span></h3> <p data-svelte-h="svelte-dig27s">Finally, there is the Text Embeddings Inference (TEI) DLC for high-performance serving of embedding models on CPU and GPU.</p> <table data-svelte-h="svelte-866hp8"><thead><tr><th>Container URI</th> <th>Accelerator</th></tr></thead> <tbody><tr><td>683313688378.dkr.ecr.us-east-1.amazonaws.com/tei-cpu:2.0.1-tei1.8.2-cpu-py310-ubuntu22.04</td> <td>CPU</td></tr> <tr><td>683313688378.dkr.ecr.us-east-1.amazonaws.com/tei:2.0.1-tei1.8.2-gpu-py310-cu122-ubuntu22.04</td> <td>GPU</td></tr></tbody></table> <h2 class="relative group"><a id="faq" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#faq"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>FAQ</span></h2> <p data-svelte-h="svelte-qjgyy3"><strong>How to choose the right inference container for my use case?</strong></p> <p data-svelte-h="svelte-86ji6y"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/sagemaker/inference-dlc-decision-tree.png" alt="inference-dlc-decision-tree"></p> <p data-svelte-h="svelte-1w4dc1f"><em>Note:</em> See <a href="https://huggingface.co/docs/sagemaker/main/en/reference/inference-toolkit" rel="nofollow">here</a> for the list of supported task in the inference toolkit.</p> <p data-svelte-h="svelte-t7yy3s"><em>Note:</em> Browse through the Hub to see if your model is tagged <a href="https://huggingface.co/models?other=text-generation-inference" rel="nofollow">“text-generation-inference”</a> or <a href="https://huggingface.co/models?other=text-embeddings-inference" rel="nofollow">“text-embeddings-inference”</a>.</p> <p data-svelte-h="svelte-1he7r2b"><strong>How to find the URI of my container?</strong></p> <p data-svelte-h="svelte-1rsk6qo">The URI is built with an AWS account ID and an AWS region. Those two values need to be replaced depending on your use case.</p> <p data-svelte-h="svelte-89omgb">Let’s say you want to use the training DLC for GPUs:</p> <ul data-svelte-h="svelte-tq759t"><li><code>dlc-aws-account-id</code>: The AWS account ID of the account that owns the ECR repository. You can find them in the <a href="https://github.com/aws/sagemaker-python-sdk/blob/e0b9d38e1e3b48647a02af23c4be54980e53dc61/src/sagemaker/image_uri_config/huggingface.json#L21" rel="nofollow">here</a></li> <li><code>region</code>: The AWS region where you want to use it.</li></ul> <p data-svelte-h="svelte-1kgkd98"><strong>How to find the URI of my container but simpler?</strong></p> <p data-svelte-h="svelte-1ojpmq0">The <code>image_uris.retrieve</code> helper from <code>sagemaker.core</code> is not always up to date but it is much simpler than reconstructing the image URI yourself. The processor (CPU/GPU/Neuron) is inferred from the <code>instance_type</code>.</p> <blockquote class="note" data-svelte-h="svelte-1drvvb1"><p>These docs and examples use the <a href="https://github.com/aws/sagemaker-python-sdk" rel="nofollow">SageMaker Python SDK v3</a>, which introduces a new framework-agnostic API built around <code>ModelBuilder</code> (inference) and <code>ModelTrainer</code> (training), replacing the v2 <code>HuggingFaceModel</code> and <code>HuggingFace</code> classes. Install it with <code>pip install "sagemaker>=3.0.0"</code>. In v2 these URIs were retrieved with <code>get_huggingface_llm_image_uri</code>, which has been removed.</p></blockquote> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class="language-python "><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> sagemaker.core <span class="hljs-keyword">import</span> image_uris

	region = <span class="hljs-string">"us-east-1"</span> <span class="hljs-comment"># the AWS region where you want to use the container</span>

	<span class="hljs-comment"># TGI (LLMs) on GPU</span>
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"TGI GPU: <span class="hljs-subst">{image_uris.retrieve(framework=<span class="hljs-string">'huggingface-llm'</span>, region=region, image_scope=<span class="hljs-string">'inference'</span>, instance_type=<span class="hljs-string">'ml.g5.2xlarge'</span>)}</span>"</span>)
	<span class="hljs-comment"># TEI (embeddings) on GPU</span>
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"TEI GPU: <span class="hljs-subst">{image_uris.retrieve(framework=<span class="hljs-string">'huggingface-tei'</span>, region=region, image_scope=<span class="hljs-string">'inference'</span>, instance_type=<span class="hljs-string">'ml.g5.2xlarge'</span>)}</span>"</span>)
	<span class="hljs-comment"># TEI (embeddings) on CPU</span>
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"TEI CPU: <span class="hljs-subst">{image_uris.retrieve(framework=<span class="hljs-string">'huggingface-tei-cpu'</span>, region=region, image_scope=<span class="hljs-string">'inference'</span>, instance_type=<span class="hljs-string">'ml.c6i.2xlarge'</span>)}</span>"</span>)
	<span class="hljs-comment"># TGI (LLMs) on AWS Inferentia2 (Neuron)</span>
	<span class="hljs-built_in">print</span>(<span class="hljs-string">f"TGI Neuron: <span class="hljs-subst">{image_uris.retrieve(framework=<span class="hljs-string">'huggingface-llm-neuronx'</span>, region=region, image_scope=<span class="hljs-string">'inference'</span>, instance_type=<span class="hljs-string">'ml.inf2.xlarge'</span>)}</span>"</span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-mm15hx">For PyTorch Training and PyTorch Inference DLCs, there is no such utility.</p> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/sagemaker/source/dlcs/available.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_fg6pm8 = {
	assets: "/docs/sagemaker/pr_2585/en",
	base: "/docs/sagemaker/pr_2585/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/sagemaker/pr_2585/en/_app/immutable/entry/start.8bec021f.js"),
	import("/docs/sagemaker/pr_2585/en/_app/immutable/entry/app.2d6e6bb7.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 2],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 25.5 kB
Xet hash:: 6b2e0e854df6e16fbcb3ec01dfafa036a24cad6d2db91ca7bc96e8bc2734ac17

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.