Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / sagemaker /pr_2709 /en /tutorials /sagemaker-sdk /deploy-sagemaker-sdk.html

HuggingFaceDocBuilder

3 days ago

download

raw

72.1 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Deploy models on Amazon SageMaker with the SageMaker SDK","local":"deploy-models-on-amazon-sagemaker-with-the-sagemaker-sdk","sections":[{"title":"Deploy a model from the 🤗 Hub","local":"deploy-a-model-from-the--hub","sections":[],"depth":2},{"title":"Deploy a 🤗 Transformers model trained in SageMaker","local":"deploy-a--transformers-model-trained-in-sagemaker","sections":[{"title":"Deploy after training","local":"deploy-after-training","sections":[],"depth":3},{"title":"Deploy with model_data","local":"deploy-with-modeldata","sections":[],"depth":3},{"title":"Create a model artifact for deployment","local":"create-a-model-artifact-for-deployment","sections":[],"depth":3}],"depth":2},{"title":"Deploy an LLM with the vLLM DLC","local":"deploy-an-llm-with-the-vllm-dlc","sections":[{"title":"Invoke the endpoint","local":"invoke-the-endpoint","sections":[],"depth":3}],"depth":2},{"title":"Run batch transform with 🤗 Transformers and SageMaker","local":"run-batch-transform-with--transformers-and-sagemaker","sections":[],"depth":2},{"title":"User defined code and modules","local":"user-defined-code-and-modules","sections":[],"depth":2}],"depth":1}"/>
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/entry/start.COiK-bUV.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/chunks/UGDyBJsP.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/chunks/Paddn1qZ.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/entry/app.CjYANwwM.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/chunks/Beft82ym.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/chunks/Ds9Xs-bq.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/chunks/DnXYL5Yp.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/nodes/0.CsY3iCn7.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/chunks/CxJVP3KE.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/nodes/27.UWwGTNRd.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/chunks/Cz9gzD1e.js" rel="modulepreload">
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/chunks/CFAwabh-.js" rel="modulepreload">
	<!--1d9zhle--><meta name="hf:doc:metadata" content="{"title":"Deploy models on Amazon SageMaker with the SageMaker SDK","local":"deploy-models-on-amazon-sagemaker-with-the-sagemaker-sdk","sections":[{"title":"Deploy a model from the 🤗 Hub","local":"deploy-a-model-from-the--hub","sections":[],"depth":2},{"title":"Deploy a 🤗 Transformers model trained in SageMaker","local":"deploy-a--transformers-model-trained-in-sagemaker","sections":[{"title":"Deploy after training","local":"deploy-after-training","sections":[],"depth":3},{"title":"Deploy with model_data","local":"deploy-with-modeldata","sections":[],"depth":3},{"title":"Create a model artifact for deployment","local":"create-a-model-artifact-for-deployment","sections":[],"depth":3}],"depth":2},{"title":"Deploy an LLM with the vLLM DLC","local":"deploy-an-llm-with-the-vllm-dlc","sections":[{"title":"Invoke the endpoint","local":"invoke-the-endpoint","sections":[],"depth":3}],"depth":2},{"title":"Run batch transform with 🤗 Transformers and SageMaker","local":"run-batch-transform-with--transformers-and-sagemaker","sections":[],"depth":2},{"title":"User defined code and modules","local":"user-defined-code-and-modules","sections":[],"depth":2}],"depth":1}"/><!---->
	<link href="/docs/sagemaker/pr_2709/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="deploy-models-on-amazon-sagemaker-with-the-sagemaker-sdk" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deploy-models-on-amazon-sagemaker-with-the-sagemaker-sdk"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Deploy models on Amazon SageMaker with the SageMaker SDK</span></h1><!--]--><!----> <p>Deploying 🤗 Transformers models in SageMaker for inference is as easy as:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> sagemaker.serve <span class="hljs-keyword">import</span> ModelBuilder

	<span class="hljs-comment"># build a Model with ModelBuilder and deploy it as a SageMaker endpoint</span>
	model_builder = ModelBuilder(...)
	model_builder.build()
	endpoint = model_builder.deploy()<!----></pre></div><!----> <p>This guide shows how to deploy models for inference with <code>ModelBuilder</code> from the SageMaker Python SDK. It covers the three serving paths: the zero-code <a href="https://github.com/aws/sagemaker-huggingface-inference-toolkit" rel="nofollow">Inference Toolkit</a> for 🤗 Transformers models (built on the <a href="https://huggingface.co/docs/transformers/main_classes/pipelines" rel="nofollow"><code>pipeline</code> feature</a>), the vLLM DLC for high-performance LLM serving, and batch transform for offline jobs. Make sure you have <a href="./setup-sagemaker-sdk">set up the SageMaker SDK</a> first.</p> <div class="mermaid-chart " style="text-align: center;"></div><!----> <p>Learn how to:</p> <ul><li><a href="#deploy-a-model-from-the--hub">Deploy a 🤗 Transformers model from the Hugging Face Hub</a>.</li> <li><a href="#deploy-a--transformers-model-trained-in-sagemaker">Deploy a 🤗 Transformers model trained in SageMaker</a>, directly after training or later from S3.</li> <li><a href="#deploy-an-llm-with-the-vllm-dlc">Deploy an LLM with the vLLM DLC</a>.</li> <li><a href="#run-batch-transform-with--transformers-and-sagemaker">Run a Batch Transform Job using 🤗 Transformers and Amazon SageMaker</a>.</li> <li><a href="#user-defined-code-and-modules">Create a custom inference module</a>.</li></ul> <!--[1--><h2 class="relative group"><a id="deploy-a-model-from-the--hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deploy-a-model-from-the--hub"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Deploy a model from the 🤗 Hub</span></h2><!--]--><!----> <p>The <a href="./sagemaker-sdk-quickstart">Quickstart</a> walks this same path end to end in a few minutes; this section explains what each piece does.</p> <p>To deploy a model directly from the 🤗 Hub to SageMaker, pass the model ID as the <code>model</code> argument and the task via the <code>HF_TASK</code> environment variable when you create a <code>ModelBuilder</code>:</p> <ul><li><code>model</code> is the model ID, automatically loaded from <a href="http://huggingface.co/models" rel="nofollow">huggingface.co/models</a> when you create a SageMaker endpoint (ModelBuilder sets the <code>HF_MODEL_ID</code> environment variable from it).</li> <li><code>HF_TASK</code> defines the task for the 🤗 Transformers <code>pipeline</code>. A complete list of tasks can be found <a href="https://huggingface.co/docs/transformers/main_classes/pipelines" rel="nofollow">here</a>.</li></ul> <blockquote class="warning"><p>Pipelines are not optimized for parallelism (multi-threading) and tend to consume a lot of RAM. For example, on a GPU-based instance, the pipeline operates on a single vCPU. When this vCPU becomes saturated with the inference requests preprocessing, it can create a bottleneck, preventing the GPU from being fully utilized for model inference. Learn more <a href="https://huggingface.co/docs/transformers/en/pipeline_webserver#using-pipelines-for-a-webserver" rel="nofollow">here</a>.</p></blockquote> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json
	<span class="hljs-keyword">from</span> sagemaker.serve <span class="hljs-keyword">import</span> ModelBuilder, ModelServer
	<span class="hljs-keyword">from</span> sagemaker.serve.builder.schema_builder <span class="hljs-keyword">import</span> SchemaBuilder
	<span class="hljs-keyword">from</span> sagemaker.core <span class="hljs-keyword">import</span> image_uris
	<span class="hljs-keyword">from</span> sagemaker.core.helper.session_helper <span class="hljs-keyword">import</span> Session, get_execution_role

	<span class="hljs-comment"># set up the SageMaker session and execution role</span>
	sess = Session()
	role = get_execution_role()

	<span class="hljs-comment"># model ID from hf.co/models</span>
	model_id = <span class="hljs-string">"cardiffnlp/twitter-roberta-base-sentiment-latest"</span>
	instance_type = <span class="hljs-string">"ml.m5.xlarge"</span>

	<span class="hljs-comment"># Retrieve the Hugging Face PyTorch inference DLC image URI</span>
	inference_image = image_uris.retrieve(
	framework=<span class="hljs-string">"huggingface"</span>,
	region=sess.boto_region_name,
	<span class="hljs-comment"># Transformers version</span>
	version=<span class="hljs-string">"4.51.3"</span>,
	<span class="hljs-comment"># PyTorch version</span>
	base_framework_version=<span class="hljs-string">"pytorch2.6.0"</span>,
	<span class="hljs-comment"># Python version</span>
	py_version=<span class="hljs-string">"py312"</span>,
	image_scope=<span class="hljs-string">"inference"</span>,
	instance_type=instance_type,
	)

	<span class="hljs-comment"># sample input/output used by ModelBuilder to set up request/response serialization</span>
	sample_input = {<span class="hljs-string">"inputs"</span>: <span class="hljs-string">"I love how simple this was!"</span>}
	sample_output = [{<span class="hljs-string">"label"</span>: <span class="hljs-string">"positive"</span>, <span class="hljs-string">"score"</span>: <span class="hljs-number">0.99</span>}]

	<span class="hljs-comment"># Pass the model ID as `model` (ModelBuilder sets HF_MODEL_ID from it) and serve it with the</span>
	<span class="hljs-comment"># Hugging Face Inference Toolkit. `HF_TASK` tells the toolkit which pipeline to build.</span>
	model_builder = ModelBuilder(
	model=model_id,
	model_server=ModelServer.MMS,
	image_uri=inference_image,
	env_vars={<span class="hljs-string">"HF_TASK"</span>: <span class="hljs-string">"text-classification"</span>},
	<span class="hljs-comment"># IAM role with permissions to create an endpoint</span>
	role_arn=role,
	sagemaker_session=sess,
	instance_type=instance_type,
	schema_builder=SchemaBuilder(sample_input=sample_input, sample_output=sample_output),
	)
	model_builder.build()

	<span class="hljs-comment"># deploy model to SageMaker Inference</span>
	predictor = model_builder.deploy(
	initial_instance_count=<span class="hljs-number">1</span>,
	instance_type=<span class="hljs-string">"ml.m5.xlarge"</span>,
	)

	<span class="hljs-comment"># example request: you always need to define "inputs"</span>
	data = {<span class="hljs-string">"inputs"</span>: <span class="hljs-string">"I love how simple this was!"</span>}

	<span class="hljs-comment"># request</span>
	res = predictor.invoke(body=json.dumps(data), content_type=<span class="hljs-string">"application/json"</span>)
	<span class="hljs-built_in">print</span>(json.loads(res.body.read()))<!----></pre></div><!----> <p>The remaining sections on this page reuse <code>sess</code>, <code>role</code>, and <code>inference_image</code> from this example; each snippet shows only what it changes.</p> <p>After you run your request, you can delete the endpoint again with:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># delete endpoint</span>
	predictor.delete()<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="deploy-a--transformers-model-trained-in-sagemaker" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deploy-a--transformers-model-trained-in-sagemaker"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Deploy a 🤗 Transformers model trained in SageMaker</span></h2><!--]--><!----> <p>There are two ways to deploy your Hugging Face model trained in SageMaker:</p> <ul><li>Deploy it after your training has finished.</li> <li>Deploy your saved model at a later time from S3 with the <code>model_data</code>.</li></ul> <!--[2--><h3 class="relative group"><a id="deploy-after-training" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deploy-after-training"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Deploy after training</span></h3><!--]--><!----> <p>To deploy your model directly after training, the training script must save everything the endpoint needs — model and tokenizer. The training script from the <a href="./training-sagemaker-sdk">Train models guide</a> already does this with <code>trainer.save_model()</code> and <code>save_pretrained()</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json
	<span class="hljs-keyword">from</span> sagemaker.serve <span class="hljs-keyword">import</span> ModelBuilder

	<span class="hljs-comment"># model_trainer is the completed ModelTrainer from the Train models guide</span>
	model_data = model_trainer._latest_training_job.model_artifacts.s3_model_artifacts

	<span class="hljs-comment"># build a Model from the trained artifacts and deploy it to SageMaker Inference</span>
	model_builder = ModelBuilder(
	<span class="hljs-comment"># Hugging Face inference DLC, from the Hub example above</span>
	image_uri=inference_image,
	s3_model_data_url=model_data,
	role_arn=role,
	sagemaker_session=sess,
	instance_type=<span class="hljs-string">"ml.m5.xlarge"</span>,
	)
	model_builder.build()
	predictor = model_builder.deploy(initial_instance_count=<span class="hljs-number">1</span>, instance_type=<span class="hljs-string">"ml.m5.xlarge"</span>)

	<span class="hljs-comment"># example request: you always need to define "inputs"</span>
	res = predictor.invoke(body=json.dumps({<span class="hljs-string">"inputs"</span>: <span class="hljs-string">"SageMaker is pretty cool"</span>}), content_type=<span class="hljs-string">"application/json"</span>)
	<span class="hljs-built_in">print</span>(json.loads(res.body.read()))<!----></pre></div><!----> <p>After you run your request you can delete the endpoint as shown:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># delete endpoint</span>
	predictor.delete()<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="deploy-with-modeldata" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deploy-with-modeldata"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Deploy with model_data</span></h3><!--]--><!----> <p>If you’ve already trained your model and want to deploy it at a later time, use the <code>s3_model_data_url</code> argument to specify the location of your tokenizer and model weights.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json
	<span class="hljs-keyword">from</span> sagemaker.serve <span class="hljs-keyword">import</span> ModelBuilder

	<span class="hljs-comment"># reuses sess, role, and inference_image from the Hub example above</span>

	<span class="hljs-comment"># create a ModelBuilder pointing at your trained model artifacts</span>
	model_builder = ModelBuilder(
	image_uri=inference_image,
	<span class="hljs-comment"># path to your trained SageMaker model</span>
	s3_model_data_url=<span class="hljs-string">"s3://models/my-bert-model/model.tar.gz"</span>,
	<span class="hljs-comment"># IAM role with permissions to create an endpoint</span>
	role_arn=role,
	sagemaker_session=sess,
	instance_type=<span class="hljs-string">"ml.m5.xlarge"</span>,
	)
	model_builder.build()

	<span class="hljs-comment"># deploy model to SageMaker Inference</span>
	predictor = model_builder.deploy(
	initial_instance_count=<span class="hljs-number">1</span>,
	instance_type=<span class="hljs-string">"ml.m5.xlarge"</span>,
	)

	<span class="hljs-comment"># example request: you always need to define "inputs"</span>
	data = {<span class="hljs-string">"inputs"</span>: <span class="hljs-string">"SageMaker is pretty cool"</span>}

	<span class="hljs-comment"># request</span>
	res = predictor.invoke(body=json.dumps(data), content_type=<span class="hljs-string">"application/json"</span>)
	<span class="hljs-built_in">print</span>(json.loads(res.body.read()))<!----></pre></div><!----> <p>After you run your request, you can delete the endpoint again with:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># delete endpoint</span>
	predictor.delete()<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="create-a-model-artifact-for-deployment" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#create-a-model-artifact-for-deployment"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Create a model artifact for deployment</span></h3><!--]--><!----> <p>For later deployment, you can create a <code>model.tar.gz</code> file that contains all the required files, such as:</p> <ul><li><code>model.safetensors</code></li> <li><code>config.json</code></li> <li><code>tokenizer.json</code></li> <li><code>tokenizer_config.json</code></li></ul> <p>For example, your file should look like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->model.tar.gz/
	\|- model.safetensors
	\|- config.json
	\|- tokenizer.json
	\|- tokenizer_config.json
	\|- special_tokens_map.json<!----></pre></div><!----> <p>Create your own <code>model.tar.gz</code> from a model from the 🤗 Hub:</p> <ol><li>Download a model:</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->git xet install
	git <span class="hljs-built_in">clone</span> git@hf.co:{repository}<!----></pre></div><!----> <ol start="2"><li>Create a <code>tar</code> file:</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!----><span class="hljs-built_in">cd</span> {repository}
	tar zcvf model.tar.gz *<!----></pre></div><!----> <ol start="3"><li>Upload <code>model.tar.gz</code> to S3:</li></ol> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->aws s3 <span class="hljs-built_in">cp</span> model.tar.gz <s3://{my-s3-path}><!----></pre></div><!----> <p>Now you can provide the S3 URI to the <code>model_data</code> argument to deploy your model later.</p> <!--[1--><h2 class="relative group"><a id="deploy-an-llm-with-the-vllm-dlc" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deploy-an-llm-with-the-vllm-dlc"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Deploy an LLM with the vLLM DLC</span></h2><!--]--><!----> <p>For high-performance LLM serving, use the Hugging Face vLLM DLC. <a href="https://docs.vllm.ai/" rel="nofollow">vLLM</a> serves most text-generation architectures on the Hub with high throughput and memory efficiency, and exposes an OpenAI-compatible API. The DLC is available for GPU and AWS AI chips (Neuron) — browse all images on the <a href="../../get-started/dlcs#available-dlcs">Available DLCs</a> page.</p> <p>Retrieve the vLLM DLC image URI and deploy with <code>ModelBuilder</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> sagemaker.core.image_uris <span class="hljs-keyword">import</span> retrieve
	<span class="hljs-keyword">from</span> sagemaker.serve <span class="hljs-keyword">import</span> ModelBuilder, ModelServer

	<span class="hljs-comment"># reuses sess and role from the Hub example above</span>

	model_id = <span class="hljs-string">"Qwen/Qwen3-8B"</span>
	instance_type = <span class="hljs-string">"ml.g5.xlarge"</span>

	<span class="hljs-comment"># Retrieve the Hugging Face vLLM inference DLC image URI</span>
	image_uri = retrieve(
	<span class="hljs-string">"huggingface-vllm"</span>,
	region=sess.boto_region_name,
	image_scope=<span class="hljs-string">"inference"</span>,
	instance_type=instance_type,
	)

	env_vars = {
	<span class="hljs-comment"># required so the container passes the SageMaker health check</span>
	<span class="hljs-string">"SM_VLLM_HOST"</span>: <span class="hljs-string">"0.0.0.0"</span>,
	<span class="hljs-comment"># required for gated models</span>
	<span class="hljs-comment"># "HF_TOKEN": "hf_...",</span>
	}

	<span class="hljs-comment"># Pass the model ID as `model` (ModelBuilder sets HF_MODEL_ID from it) and select the vLLM server.</span>
	model_builder = ModelBuilder(
	model=model_id,
	model_server=ModelServer.VLLM,
	image_uri=image_uri,
	env_vars=env_vars,
	<span class="hljs-comment"># IAM role with permissions to create an endpoint</span>
	role_arn=role,
	sagemaker_session=sess,
	instance_type=instance_type,
	)
	model_builder.build()

	predictor = model_builder.deploy(initial_instance_count=<span class="hljs-number">1</span>, instance_type=instance_type)<!----></pre></div><!----> <p>Tune vLLM through environment variables: <code>SM_VLLM_MAX_MODEL_LEN</code> for the context length, <code>SM_VLLM_GPU_MEMORY_UTILIZATION</code> for the KV cache budget, and more — each maps to a vLLM engine argument. For a full configuration example, see the <a href="../../examples/sagemaker-sdk-trip-planner-agent-vllm">trip planner agent with vLLM</a> example.</p> <!--[2--><h3 class="relative group"><a id="invoke-the-endpoint" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#invoke-the-endpoint"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Invoke the endpoint</span></h3><!--]--><!----> <p>The vLLM DLC exposes OpenAI-compatible routes. Send requests with the SageMaker runtime client and set the route in <code>CustomAttributes</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> json

	runtime = sess.boto_session.client(<span class="hljs-string">"sagemaker-runtime"</span>)

	response = runtime.invoke_endpoint(
	EndpointName=predictor.endpoint_name,
	ContentType=<span class="hljs-string">"application/json"</span>,
	Body=json.dumps({
	<span class="hljs-string">"model"</span>: model_id,
	<span class="hljs-string">"messages"</span>: [{<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: <span class="hljs-string">"What is the capital of France?"</span>}],
	}),
	CustomAttributes=<span class="hljs-string">"route=/v1/chat/completions"</span>,
	)

	<span class="hljs-built_in">print</span>(json.loads(response[<span class="hljs-string">"Body"</span>].read())[<span class="hljs-string">"choices"</span>][<span class="hljs-number">0</span>][<span class="hljs-string">"message"</span>][<span class="hljs-string">"content"</span>])<!----></pre></div><!----> <p>Once you are done experimenting, delete the endpoint:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->predictor.delete()<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="run-batch-transform-with--transformers-and-sagemaker" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#run-batch-transform-with--transformers-and-sagemaker"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Run batch transform with 🤗 Transformers and SageMaker</span></h2><!--]--><!----> <p>After training a model, you can use <a href="https://docs.aws.amazon.com/sagemaker/latest/dg/how-it-works-batch.html" rel="nofollow">SageMaker batch transform</a> to perform inference with the model. Batch transform accepts your inference data as an S3 URI and then SageMaker will take care of downloading the data, running the prediction, and uploading the results to S3. For more details about batch transform, take a look <a href="https://docs.aws.amazon.com/sagemaker/latest/dg/batch-transform.html" rel="nofollow">here</a>.</p> <blockquote class="warning"><p>The Hugging Face Inference DLC currently only supports <code>.jsonl</code> for batch transform due to the complex structure of textual data.</p></blockquote> <blockquote class="note"><p>Make sure your <code>inputs</code> fit the <code>max_length</code> of the model during preprocessing.</p></blockquote> <p>If you trained a model with a <code>ModelTrainer</code>, build a <code>ModelBuilder</code> from the trained artifacts and call its <code>transformer()</code> method to create a transform job:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> sagemaker.serve <span class="hljs-keyword">import</span> ModelBuilder

	<span class="hljs-comment"># build a Model from the trained artifacts</span>
	model_builder = ModelBuilder(
	image_uri=inference_image,
	s3_model_data_url=model_trainer._latest_training_job.model_artifacts.s3_model_artifacts,
	role_arn=role,
	sagemaker_session=sess,
	)
	model_builder.build()

	batch_job = model_builder.transformer(
	instance_count=<span class="hljs-number">1</span>,
	<span class="hljs-comment"># matches the CPU inference image from the Hub example</span>
	instance_type=<span class="hljs-string">'ml.m5.xlarge'</span>,
	strategy=<span class="hljs-string">'SingleRecord'</span>)


	batch_job.transform(
	data=<span class="hljs-string">'s3://s3-uri-to-batch-data'</span>,
	content_type=<span class="hljs-string">'application/json'</span>,
	split_type=<span class="hljs-string">'Line'</span>)<!----></pre></div><!----> <p>If you want to run your batch transform job later or with a model from the 🤗 Hub, create a <code>ModelBuilder</code> and then call the <code>transformer()</code> method:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> sagemaker.serve <span class="hljs-keyword">import</span> ModelBuilder, ModelServer
	<span class="hljs-keyword">from</span> sagemaker.serve.builder.schema_builder <span class="hljs-keyword">import</span> SchemaBuilder

	<span class="hljs-comment"># reuses sess, role, and inference_image from the Hub example above</span>

	model_id = <span class="hljs-string">"cardiffnlp/twitter-roberta-base-sentiment-latest"</span>
	instance_type = <span class="hljs-string">"ml.m5.xlarge"</span>

	<span class="hljs-comment"># Pass the model ID as `model` (ModelBuilder sets HF_MODEL_ID) and serve it with the</span>
	<span class="hljs-comment"># Hugging Face Inference Toolkit.</span>
	model_builder = ModelBuilder(
	model=model_id,
	model_server=ModelServer.MMS,
	image_uri=inference_image,
	env_vars={<span class="hljs-string">"HF_TASK"</span>: <span class="hljs-string">"text-classification"</span>},
	<span class="hljs-comment"># IAM role with permissions to create an endpoint</span>
	role_arn=role,
	sagemaker_session=sess,
	instance_type=instance_type,
	schema_builder=SchemaBuilder(
	sample_input={<span class="hljs-string">"inputs"</span>: <span class="hljs-string">"this movie is terrible"</span>},
	sample_output=[{<span class="hljs-string">"label"</span>: <span class="hljs-string">"negative"</span>, <span class="hljs-string">"score"</span>: <span class="hljs-number">0.99</span>}],
	),
	)
	model_builder.build()

	<span class="hljs-comment"># create transformer to run a batch job</span>
	batch_job = model_builder.transformer(
	instance_count=<span class="hljs-number">1</span>,
	instance_type=instance_type,
	strategy=<span class="hljs-string">'SingleRecord'</span>
	)

	<span class="hljs-comment"># starts batch transform job and uses S3 data as input</span>
	batch_job.transform(
	data=<span class="hljs-string">'s3://sagemaker-s3-demo-test/samples/input.jsonl'</span>,
	content_type=<span class="hljs-string">'application/json'</span>,
	split_type=<span class="hljs-string">'Line'</span>
	)<!----></pre></div><!----> <p>The <code>input.jsonl</code> looks like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-jsonl "><!----><span class="hljs-punctuation">{</span><span class="hljs-attr">"inputs"</span><span class="hljs-punctuation">:</span><span class="hljs-string">"this movie is terrible"</span><span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">{</span><span class="hljs-attr">"inputs"</span><span class="hljs-punctuation">:</span><span class="hljs-string">"this movie is amazing"</span><span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">{</span><span class="hljs-attr">"inputs"</span><span class="hljs-punctuation">:</span><span class="hljs-string">"SageMaker is pretty cool"</span><span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">{</span><span class="hljs-attr">"inputs"</span><span class="hljs-punctuation">:</span><span class="hljs-string">"SageMaker is pretty cool"</span><span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">{</span><span class="hljs-attr">"inputs"</span><span class="hljs-punctuation">:</span><span class="hljs-string">"this movie is terrible"</span><span class="hljs-punctuation">}</span>
	<span class="hljs-punctuation">{</span><span class="hljs-attr">"inputs"</span><span class="hljs-punctuation">:</span><span class="hljs-string">"this movie is amazing"</span><span class="hljs-punctuation">}</span><!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="user-defined-code-and-modules" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#user-defined-code-and-modules"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>User defined code and modules</span></h2><!--]--><!----> <p>The Hugging Face Inference Toolkit allows the user to override the default methods of the <code>HuggingFaceHandlerService</code>. You will need to create a folder named <code>code/</code> with an <code>inference.py</code> file in it. See <a href="#create-a-model-artifact-for-deployment">here</a> for more details on how to archive your model artifacts. For example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->model.tar.gz/
	\|- model.safetensors
	\|- ....
	\|- code/
	\|- inference.py
	\|- requirements.txt <!----></pre></div><!----> <p>The <code>inference.py</code> file contains your custom inference module, and the <code>requirements.txt</code> file contains additional dependencies that should be added. The custom module can override the following methods:</p> <ul><li><code>model_fn(model_dir)</code> overrides the default method for loading a model. The return value <code>model</code> will be used in <code>predict</code> for predictions. <code>predict</code> receives argument the <code>model_dir</code>, the path to your unzipped <code>model.tar.gz</code>.</li> <li><code>transform_fn(model, data, content_type, accept_type)</code> overrides the default transform function with your custom implementation. You will need to implement your own <code>preprocess</code>, <code>predict</code> and <code>postprocess</code> steps in the <code>transform_fn</code>. This method can’t be combined with <code>input_fn</code>, <code>predict_fn</code> or <code>output_fn</code> mentioned below.</li> <li><code>input_fn(input_data, content_type)</code> overrides the default method for preprocessing. The return value <code>data</code> will be used in <code>predict</code> for predictions. The inputs are: <ul><li><code>input_data</code> is the raw body of your request.</li> <li><code>content_type</code> is the content type from the request header.</li></ul></li> <li><code>predict_fn(processed_data, model)</code> overrides the default method for predictions. The return value <code>predictions</code> will be used in <code>postprocess</code>. The input is <code>processed_data</code>, the result from <code>preprocess</code>.</li> <li><code>output_fn(prediction, accept)</code> overrides the default method for postprocessing. The return value <code>result</code> will be the response of your request (e.g.<code>JSON</code>). The inputs are: <ul><li><code>predictions</code> is the result from <code>predict</code>.</li> <li><code>accept</code> is the return accept type from the HTTP Request, e.g. <code>application/json</code>.</li></ul></li></ul> <p>Here is an example of a custom inference module with <code>model_fn</code>, <code>input_fn</code>, <code>predict_fn</code>, and <code>output_fn</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> sagemaker_huggingface_inference_toolkit <span class="hljs-keyword">import</span> decoder_encoder

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">model_fn</span>(<span class="hljs-params">model_dir</span>):
	<span class="hljs-comment"># implement custom code to load the model</span>
	loaded_model = ...

	<span class="hljs-keyword">return</span> loaded_model

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">input_fn</span>(<span class="hljs-params">input_data, content_type</span>):
	<span class="hljs-comment"># decode the input data (e.g. JSON string -> dict)</span>
	data = decoder_encoder.decode(input_data, content_type)
	<span class="hljs-keyword">return</span> data

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">predict_fn</span>(<span class="hljs-params">data, model</span>):
	<span class="hljs-comment"># call your custom model with the data</span>
	outputs = model(data , ... )
	<span class="hljs-keyword">return</span> predictions

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">output_fn</span>(<span class="hljs-params">prediction, accept</span>):
	<span class="hljs-comment"># convert the model output to the desired output format (e.g. dict -> JSON string)</span>
	response = decoder_encoder.encode(prediction, accept)
	<span class="hljs-keyword">return</span> response<!----></pre></div><!----> <p>Customize your inference module with only <code>model_fn</code> and <code>transform_fn</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> sagemaker_huggingface_inference_toolkit <span class="hljs-keyword">import</span> decoder_encoder

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">model_fn</span>(<span class="hljs-params">model_dir</span>):
	<span class="hljs-comment"># implement custom code to load the model</span>
	loaded_model = ...

	<span class="hljs-keyword">return</span> loaded_model

	<span class="hljs-keyword">def</span> <span class="hljs-title function_">transform_fn</span>(<span class="hljs-params">model, input_data, content_type, accept</span>):
	<span class="hljs-comment"># decode the input data (e.g. JSON string -> dict)</span>
	data = decoder_encoder.decode(input_data, content_type)

	<span class="hljs-comment"># call your custom model with the data</span>
	outputs = model(data , ... )

	<span class="hljs-comment"># convert the model output to the desired output format (e.g. dict -> JSON string)</span>
	response = decoder_encoder.encode(output, accept)

	<span class="hljs-keyword">return</span> response<!----></pre></div><!----> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/sagemaker/source/tutorials/sagemaker-sdk/deploy-sagemaker-sdk.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->

	<script>
	{
	__sveltekit_dwws3f = {
	base: "/docs/sagemaker/pr_2709/en",
	assets: "/docs/sagemaker/pr_2709/en"
	};

	const element = document.currentScript.parentElement;

	Promise.all([
	import("/docs/sagemaker/pr_2709/en/_app/immutable/entry/start.COiK-bUV.js"),
	import("/docs/sagemaker/pr_2709/en/_app/immutable/entry/app.CjYANwwM.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 27],
	data: [null,null],
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 72.1 kB
Xet hash:: e523ef16ed10b48de2d718d13209634b71c02b953bed0e886b92c7a600978694

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.