Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / sagemaker /pr_2188 /en /examples /sagemaker-sdk-neuron-agent-inf.html

rtrm

4 months ago

download

raw

92.2 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Deploy an AI Agent on AWS Inferentia2 with SageMaker","local":"deploy-an-ai-agent-on-aws-inferentia2-with-sagemaker","sections":[{"title":"🎯 Goal","local":"-goal","sections":[{"title":"Configure logging and env","local":"configure-logging-and-env","sections":[],"depth":4}],"depth":3},{"title":"1. Deploy an LLM on a SageMaker Inferentia2 real-time endpoint","local":"1-deploy-an-llm-on-a-sagemaker-inferentia2-real-time-endpoint","sections":[],"depth":3},{"title":"2. Create an Agent with Strands Agent","local":"2-create-an-agent-with-strands-agent","sections":[],"depth":3},{"title":"3. Use my Agent with HF MCP server 🤗","local":"3-use-my-agent-with-hf-mcp-server-","sections":[],"depth":3},{"title":"🧩 Understanding the Agent Output","local":"-understanding-the-agent-output","sections":[],"depth":3}],"depth":1}">
	<link href="/docs/sagemaker/pr_2188/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/entry/start.48a18f09.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/chunks/scheduler.aec39e6a.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/chunks/singletons.8e7a9ddc.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/chunks/paths.a4e52f32.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/entry/app.8481cdda.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/chunks/preload-helper.382cad4e.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/chunks/index.4ee0a2d0.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/nodes/0.52c1f0fb.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/nodes/10.b8e2ee3c.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/chunks/Tip.e2132029.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/chunks/MermaidChart.svelte_svelte_type_style_lang.3c60bfa3.js">
	<link rel="modulepreload" href="/docs/sagemaker/pr_2188/en/_app/immutable/chunks/CodeBlock.543f5448.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Deploy an AI Agent on AWS Inferentia2 with SageMaker","local":"deploy-an-ai-agent-on-aws-inferentia2-with-sagemaker","sections":[{"title":"🎯 Goal","local":"-goal","sections":[{"title":"Configure logging and env","local":"configure-logging-and-env","sections":[],"depth":4}],"depth":3},{"title":"1. Deploy an LLM on a SageMaker Inferentia2 real-time endpoint","local":"1-deploy-an-llm-on-a-sagemaker-inferentia2-real-time-endpoint","sections":[],"depth":3},{"title":"2. Create an Agent with Strands Agent","local":"2-create-an-agent-with-strands-agent","sections":[],"depth":3},{"title":"3. Use my Agent with HF MCP server 🤗","local":"3-use-my-agent-with-hf-mcp-server-","sections":[],"depth":3},{"title":"🧩 Understanding the Agent Output","local":"-understanding-the-agent-output","sections":[],"depth":3}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg></button></div> </div> <h1 class="relative group"><a id="deploy-an-ai-agent-on-aws-inferentia2-with-sagemaker" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#deploy-an-ai-agent-on-aws-inferentia2-with-sagemaker"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Deploy an AI Agent on AWS Inferentia2 with SageMaker</span></h1> <p data-svelte-h="svelte-1neh50t">This notebook demontrates how to deploy an open-ource LLM on a real-time AWS endpoint with Neuron devices using Hugging Face library <code>optimum-neuron</code>, <code>strands-agent</code> and AWS Sagemaker.</p> <h3 class="relative group"><a id="-goal" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#-goal"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>🎯 Goal</span></h3> <p data-svelte-h="svelte-zrb19s">By the end of this notebook, you will know how to:</p> <ul data-svelte-h="svelte-1r2qr0q"><li><strong>Deploy an LLM</strong> optimized for instruction-following, reasoning, and tool-calling — specifically <a href="https://huggingface.co/ibm-granite/granite-3.3-8b-instruct" rel="nofollow"><code>ibm-granite/granite-3.3-8b-instruct</code></a>.</li> <li><strong>Run it on an AWS Inferentia2 inference-optimized instance</strong> (<code>inf2.24xlarge</code>) using a <strong>SageMaker real-time endpoint</strong>.</li> <li><strong>Use the deployed endpoint in an agentic setup</strong>, including:
	<ul><li>Calling external tools such as <strong>web search</strong>.</li> <li>Connecting to and interacting with <strong>MCP servers</strong>, such as the Hugging Face MCP servers, to work with the Hugging Face Hub.</li></ul></li></ul> <p data-svelte-h="svelte-ypfsa4">This notebook guides you from deployment to practical agent-enabled usage.</p> <h4 class="relative group"><a id="configure-logging-and-env" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#configure-logging-and-env"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Configure logging and env</span></h4> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->%%writefile .env
	HF_TOKEN=your_hf_token_here<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
	<span class="hljs-keyword">from</span> dotenv <span class="hljs-keyword">import</span> load_dotenv

	<span class="hljs-comment"># Load secrets from .env if present so the notebook picks up local configuration.</span>
	dotenv_path = Path.cwd() / <span class="hljs-string">".env"</span>
	<span class="hljs-keyword">if</span> dotenv_path.exists():
	load_dotenv(dotenv_path)
	console.<span class="hljs-built_in">print</span>(<span class="hljs-string">f"[green]Loaded environment from <span class="hljs-subst">{dotenv_path}</span>[/green]"</span>)
	<span class="hljs-keyword">else</span>:
	console.<span class="hljs-built_in">print</span>(<span class="hljs-string">f"[yellow]No .env file found at <span class="hljs-subst">{dotenv_path}</span>; using existing environment variables.[/yellow]"</span>)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->!pip install -q rich<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> logging
	<span class="hljs-keyword">from</span> rich.console <span class="hljs-keyword">import</span> Console
	<span class="hljs-keyword">from</span> rich.logging <span class="hljs-keyword">import</span> RichHandler

	console = Console()
	logging.basicConfig(
	level=logging.INFO,
	<span class="hljs-built_in">format</span>=<span class="hljs-string">"%(message)s"</span>,
	datefmt=<span class="hljs-string">"[%X]"</span>,
	handlers=[RichHandler(console=console, rich_tracebacks=<span class="hljs-literal">True</span>, markup=<span class="hljs-literal">True</span>)],
	force=<span class="hljs-literal">True</span>,
	)
	logger = logging.getLogger(<span class="hljs-string">"neuron-agent"</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="1-deploy-an-llm-on-a-sagemaker-inferentia2-real-time-endpoint" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#1-deploy-an-llm-on-a-sagemaker-inferentia2-real-time-endpoint"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>1. Deploy an LLM on a SageMaker Inferentia2 real-time endpoint</span></h3> <p data-svelte-h="svelte-knl01y"><strong><em>🔧 Configure SageMaker Session</em></strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->!pip install sagemaker --upgrade --quiet<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> <span class="hljs-keyword">import</span> sagemaker
	>> <span class="hljs-keyword">import</span> boto3

	>> sess = sagemaker.Session()
	>> <span class="hljs-comment"># sagemaker session bucket -> used for uploading data, models and logs</span>
	>> <span class="hljs-comment"># sagemaker will automatically create this bucket if it not exists</span>
	>> sagemaker_session_bucket = <span class="hljs-literal">None</span>
	>> <span class="hljs-keyword">if</span> sagemaker_session_bucket <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span> <span class="hljs-keyword">and</span> sess <span class="hljs-keyword">is</span> <span class="hljs-keyword">not</span> <span class="hljs-literal">None</span>:
	<span class="hljs-meta">... </span> <span class="hljs-comment"># set to default bucket if a bucket name is not given</span>
	<span class="hljs-meta">... </span> sagemaker_session_bucket = sess.default_bucket()<!-- HTML_TAG_END --></pre></div> <pre data-svelte-h="svelte-1k0tiap">sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml
	sagemaker.config INFO - Not applying SDK defaults from location: /home/ec2-user/.config/sagemaker/config.yaml
	</pre> <p data-svelte-h="svelte-k5o2yu"><strong><em>🔑 Configure SageMaker Role</em></strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->sagemaker_role_name = <span class="hljs-string">"sagemaker-dlcs"</span> <span class="hljs-comment"># <span class="hljs-doctag">TODO:</span> change to your own role name</span>
	iam = boto3.client(<span class="hljs-string">"iam"</span>)
	role = iam.get_role(RoleName=sagemaker_role_name)[<span class="hljs-string">"Role"</span>][<span class="hljs-string">"Arn"</span>]<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> sess = sagemaker.Session(default_bucket=sagemaker_session_bucket)

	>> <span class="hljs-built_in">print</span>(<span class="hljs-string">f"sagemaker role arn: <span class="hljs-subst">{role}</span>"</span>)
	>> <span class="hljs-built_in">print</span>(<span class="hljs-string">f"sagemaker session region: <span class="hljs-subst">{sess.boto_region_name}</span>"</span>)<!-- HTML_TAG_END --></pre></div> <pre data-svelte-h="svelte-11ud7vs">sagemaker role arn: arn:aws:iam::754289655784:role/sagemaker-dlcs
	sagemaker session region: us-east-1
	</pre> <p data-svelte-h="svelte-ch75wo"><strong><em>📦 Prepare the model</em></strong></p> <p data-svelte-h="svelte-124nmkx">Before deploying the model, we need to <strong>compile it for AWS Inferentia2</strong>.</p> <p data-svelte-h="svelte-1r32t58">Inferentia and Trainium devices run models using the <strong>Neuron runtime</strong>, which requires models to be <strong>compiled ahead of time</strong> into an optimized Neuron executable.<br>
	This compilation step transforms the original model (PyTorch or Transformers weights) into a hardware-optimized graph that can run efficiently on <code>inf2</code> instances.</p> <p data-svelte-h="svelte-ipw77i">Because we’re working on a <strong>Neuron-based instance</strong>, compiling the model is a mandatory step before deployment.</p> <p data-svelte-h="svelte-p275m2">SageMaker will then use the <strong>Optimum-Neuron vLLM container</strong> to load the compiled artifacts and run them efficiently on Inferentia2.<br>
	Instead of compiling the model at inference time, the container expects a <strong>pre-compiled Neuron executable</strong>, which it can serve directly through a real-time endpoint for fast and stable performance.</p> <p data-svelte-h="svelte-hwswrp"><strong><em>🧩 Optimum Neuron Cache</em></strong></p> <p data-svelte-h="svelte-gfe9bq">Hugging Face makes working with Neuron hardware much easier through the <strong>Optimum Neuron Cache</strong> (<a href="https://huggingface.co/aws-neuron/optimum-neuron-cache" rel="nofollow"><code>aws-neuron/optimum-neuron-cache</code></a>), a public repository that hosts <strong>pre-compiled Neuron artifacts</strong> for many popular models.<br>
	Instead of spending time compiling models yourself—a process that can take minutes to hours—you can simply reuse these ready-made artifacts.</p> <p data-svelte-h="svelte-1p4vv0o">This means:</p> <ul data-svelte-h="svelte-1xmsedh"><li><strong>Instant startup</strong> with no compilation step</li> <li><strong>Consistent, reproducible performance</strong> on Inferentia2 and Trainium</li> <li><strong>Seamless integration</strong> with the Optimum-Neuron vLLM containers, which can load these cached executables directly</li></ul> <p data-svelte-h="svelte-mfv7ok">If a matching compiled artifact is available, you can deploy your model on SageMaker immediately.</p> <p data-svelte-h="svelte-17zambt">Learn more about the Optimum Neuron model cache <a href="https://huggingface.co/docs/optimum-neuron/en/guides/cache_system" rel="nofollow">here</a>.</p> <p data-svelte-h="svelte-1cx03ag"><strong><em>⚙️ Model compilation</em></strong></p> <p data-svelte-h="svelte-1v2wimv">The model we selected, <a href="%22https://huggingface.co/ibm-granite/granite-3.3-8b-instruct%22"><code>ibm-granite/granite-3.3-8b-instruct</code></a>, provides strong instruction-following, reasoning, and tool-calling abilities while remaining lightweight enough to run efficiently on a single Inferentia2 with a large input context necessary for agentic use-cases.</p> <p data-svelte-h="svelte-1g0753p">At the time of writing, <strong>granite-3.3-8B-instruct</strong> was not yet available as a pre-compiled Neuron artifact, so we compile it ourselves before deploying it on Inferentia2. We followed the steps from the official Optimum Neuron export tutorial (<a href="https://huggingface.co/docs/optimum-neuron/en/guides/export_model" rel="nofollow">https://huggingface.co/docs/optimum-neuron/en/guides/export_model</a>) to compile the model for Inferentia2.</p> <p data-svelte-h="svelte-1tboauk">To compile the model, we:</p> <ul><li data-svelte-h="svelte-1xb1py4"><p>Launched an <strong>Inferentia2 EC2 instance</strong> using the official <strong>Hugging Face Neuron Deep Learning AMI</strong><br>
	(<a href="https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2" rel="nofollow">https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2</a>).</p></li> <li data-svelte-h="svelte-1mpwa1q"><p>Connected to the instance via <strong>SSH</strong>.</p></li> <li><p data-svelte-h="svelte-1td5nmr">Used <strong>Optimum Neuron</strong> to compile the model with the following command:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron \
	--model ibm-granite/granite-3.3-8b-instruct \
	--sequence_length 16384 \
	--batch_size 1 \
	--num_cores 8 \
	--task text-generation \
	<folder-to-saved-compiled-artifacts><!-- HTML_TAG_END --></pre></div></li> <li data-svelte-h="svelte-p23mf1"><p>Pushed the compiled Neuron artifacts to the Hugging Face Hub:
	<a href="https://huggingface.co/florentgbelidji/granite-3.3-8b-instruct-neuron-bs-1-seq-16384-tp-8" rel="nofollow"><code>florentgbelidji/granite-3.3-8b-instruct-neuron-bs-1-seq-16384-tp-8</code></a></p></li></ul> <p data-svelte-h="svelte-1nfqfnx">At the time of writing, AWS Inferentia2 does not support dynamic shapes for inference, which means that we need to specify our sequence length and batch size ahead of time. We chose these compilation parameters to match our agentic use case and the target hardware:</p> <ul data-svelte-h="svelte-c0cnpr"><li><strong><code>--sequence_length 16384</code></strong>: Enables <em>large-context</em> inference, which is important when prompts include long tool definitions or multi-step reasoning typical in agent workflows.</li> <li><strong><code>--batch_size 1</code></strong>: Prioritizes <em>low latency</em> over throughput, ideal for the demo where requests are sent sequentially rather than in batches.</li> <li><strong><code>--num_cores 8</code></strong>: Maximizes hardware utilization on <code>inf2</code> instances.<br>
	An <code>inf2.24xlarge</code> exposes <em>6 Neuron devices = 12 Neuron cores</em> (learn more <a href="https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf2-arch.html#aws-inf2-arch" rel="nofollow">https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf2-arch.html#aws-inf2-arch</a>
	)</li></ul> <p data-svelte-h="svelte-406pm4">We set <strong><code>--num_cores 8</code></strong> because, at <strong>deployment time</strong>, the Neuron runtime requires that the model’s number of attention heads be divisible by the number of Neuron cores assigned to the model.<br>
	Using 8 cores satisfies this constraint on Inferentia2 and ensures the model can be loaded and served correctly.</p> <p data-svelte-h="svelte-ohmkdu"><strong>🧱 Deployment Server: HuggingFace vLLM Inference Neuronx DLC</strong></p> <p data-svelte-h="svelte-1ni9rl8">For deployment, we use the <strong>HuggingFace vLLM Inference Neuronx</strong> Deep Learning Container from AWS ECR:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-attribute">763104351884</span>.dkr.ecr.us-east-<span class="hljs-number">1</span>.amazonaws.com/huggingface-vllm-inference-neuronx:<span class="hljs-number">0</span>.<span class="hljs-number">11</span>.<span class="hljs-number">0</span>-optimum0.<span class="hljs-number">4</span>.<span class="hljs-number">4</span>-neuronx-py310-sdk2.<span class="hljs-number">26</span>.<span class="hljs-number">1</span>-ubuntu22.<span class="hljs-number">04</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1drgj2j">This container is built on <a href="https://huggingface.co/docs/optimum-neuron" rel="nofollow">Optimum Neuron</a> and is designed to <strong>serve Neuron-compiled models using vLLM on Inferentia2 instances</strong>, combining Neuron runtime optimizations with vLLM’s efficient inference mechanisms.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->llm_image = <span class="hljs-string">"763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-vllm-inference-neuronx:0.11.0-optimum0.4.4-neuronx-py310-sdk2.26.1-ubuntu22.04"</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-7g2kv1"><strong><em>⚙️ Configure the SageMaker Endpoint with the Hugging Face SDK</em></strong></p> <p data-svelte-h="svelte-1rkejri">Now that we have a Neuron-compiled model and a deployment image, we can configure a SageMaker real-time endpoint using the <strong>Hugging Face SageMaker SDK</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> os
	<span class="hljs-keyword">from</span> sagemaker.huggingface <span class="hljs-keyword">import</span> HuggingFaceModel

	<span class="hljs-comment"># sagemaker config</span>
	instance_type = <span class="hljs-string">"ml.inf2.24xlarge"</span>
	health_check_timeout = <span class="hljs-number">3600</span> <span class="hljs-comment"># additional time to load the model</span>

	config = {
	<span class="hljs-string">"SM_ON_MODEL"</span>: <span class="hljs-string">"florentgbelidji/granite-3.3-8b-instruct-neuron-bs-1-seq-16384-tp-8"</span>,
	<span class="hljs-string">"SM_ON_TENSOR_PARALLEL_SIZE"</span>: <span class="hljs-string">"8"</span>,<span class="hljs-comment"># corresponds to the number of Neuron cores</span>
	<span class="hljs-string">"SM_ON_BATCH_SIZE"</span>: <span class="hljs-string">"1"</span>, <span class="hljs-comment"># max batch size for the model</span>
	<span class="hljs-string">"SM_ON_SEQUENCE_LENGTH"</span>: <span class="hljs-string">"16384"</span>, <span class="hljs-comment"># max length of generated text</span>
	<span class="hljs-string">"SM_ON_ENABLE_AUTO_TOOL_CHOICE"</span>: <span class="hljs-string">"true"</span>, <span class="hljs-comment"># enables tool choice</span>
	<span class="hljs-string">"SM_ON_TOOL_CALL_PARSER"</span>: <span class="hljs-string">"granite"</span>, <span class="hljs-comment"># enables tool call parsing with parser configured for granite</span>
	<span class="hljs-string">"SM_ON_MAX_NUM_BATCHED_TOKENS"</span>: <span class="hljs-string">"8192"</span>, <span class="hljs-comment"># max number of tokens in a batch, control prefill chunking</span>
	<span class="hljs-string">"HF_TOKEN"</span>: os.environ[<span class="hljs-string">"HF_TOKEN"</span>],
	}

	endpoint_name = <span class="hljs-string">"granite-8b-instruct-neuron-bs-1-seq-16384-tp-8-demo-endpoint"</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-14x4mn1"><strong><em>Create Sagemaker Model</em></strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->llm_model = HuggingFaceModel(
	role=role,
	image_uri=llm_image,
	env=config,
	name=endpoint_name
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1h6to9w"><strong><em>🚀 Deploy</em></strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> llm_model._is_compiled_model = <span class="hljs-literal">True</span>

	>> llm = llm_model.deploy(
	<span class="hljs-meta">... </span> initial_instance_count=<span class="hljs-number">1</span>,
	<span class="hljs-meta">... </span> instance_type=instance_type,
	<span class="hljs-meta">... </span> container_startup_health_check_timeout=health_check_timeout,
	<span class="hljs-meta">... </span> inference_ami_version=<span class="hljs-string">"al2-ami-sagemaker-inference-neuron-2"</span>,
	<span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <pre data-svelte-h="svelte-b2by0">-------------------!
	</pre> <p data-svelte-h="svelte-dh0xpk">Go to console.aws.amazon.com to check the status of your endpoint and the logs</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-built_in">print</span>(llm.endpoint_name)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-qobbm6"><strong><em>Reload already created endpoint</em></strong></p> <p data-svelte-h="svelte-12gbzkn">If your endpoint is still running run the following:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->existing_endpoint_name = <span class="hljs-string">"granite-8b-instruct-neuron-bs-1-seq-163-2025-11-25-16-35-23-419"</span> <span class="hljs-comment"># <span class="hljs-doctag">TODO:</span> change to your own endpoint name</span><!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> sagemaker.predictor <span class="hljs-keyword">import</span> Predictor
	llm = Predictor(
	endpoint_name=existing_endpoint_name ,
	sagemaker_session=sess
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-l3rlw1"><strong>Test</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> sagemaker.deserializers <span class="hljs-keyword">import</span> JSONDeserializer
	<span class="hljs-keyword">from</span> sagemaker.serializers <span class="hljs-keyword">import</span> JSONSerializer

	messages = [
	{<span class="hljs-string">"role"</span>: <span class="hljs-string">"user"</span>, <span class="hljs-string">"content"</span>: [{<span class="hljs-string">"type"</span>: <span class="hljs-string">"text"</span>, <span class="hljs-string">"text"</span>: <span class="hljs-string">"Hello my name is Huggy"</span>}]},
	]

	parameters = {
	<span class="hljs-string">"max_tokens"</span>: <span class="hljs-number">16</span>,
	<span class="hljs-string">"temperature"</span>: <span class="hljs-number">0.1</span>,
	<span class="hljs-string">"stream"</span>: <span class="hljs-literal">False</span>,
	}

	llm.serializer = JSONSerializer()
	llm.deserializer = JSONDeserializer()

	payload = {<span class="hljs-string">"messages"</span>: messages, **parameters}
	chat = llm.predict(payload)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->chat<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> <span class="hljs-built_in">print</span>(chat[<span class="hljs-string">"choices"</span>][<span class="hljs-number">0</span>][<span class="hljs-string">"message"</span>][<span class="hljs-string">"content"</span>])<!-- HTML_TAG_END --></pre></div> <pre data-svelte-h="svelte-1ixdx6t">Hello Huggy, nice to meet you! How can I assist you today
	</pre> <h3 class="relative group"><a id="2-create-an-agent-with-strands-agent" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#2-create-an-agent-with-strands-agent"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>2. Create an Agent with Strands Agent</span></h3> <p data-svelte-h="svelte-1geb7w3"><a href="https://strandsagents.com/latest/" rel="nofollow">Strands Agents</a> is an open-source Python SDK for building <strong>production-grade AI agents</strong> with minimal code.<br>
	It provides a clean, model-driven framework for defining system prompts, tools, multi-step reasoning, and orchestration.</p> <p data-svelte-h="svelte-spmlue">A key advantage for AWS users: <strong>Strands natively supports SageMaker real-time endpoints</strong>, allowing you to plug in your deployed LLMs directly as agent backends with no custom wrappers.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->!pip install -q <span class="hljs-string">'strands-agents[sagemaker]'</span> strands-agents-tools<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> strands <span class="hljs-keyword">import</span> Agent, tool
	<span class="hljs-keyword">from</span> strands.tools.mcp <span class="hljs-keyword">import</span> MCPClient
	<span class="hljs-keyword">from</span> strands.models.sagemaker <span class="hljs-keyword">import</span> SageMakerAIModel
	<span class="hljs-keyword">from</span> mcp.client.streamable_http <span class="hljs-keyword">import</span> streamablehttp_client<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1f7uw8q"><strong><em>🔗 Link your endpoint to strands-agent</em></strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->model = SageMakerAIModel(
	endpoint_config={
	<span class="hljs-string">"endpoint_name"</span>: endpoint_name,
	<span class="hljs-string">"region_name"</span>: <span class="hljs-string">"us-east-1"</span>, <span class="hljs-comment"># or your region</span>
	},
	payload_config={
	<span class="hljs-string">"max_tokens"</span>: <span class="hljs-number">2048</span>,
	<span class="hljs-string">"temperature"</span>: <span class="hljs-number">0.2</span>,
	<span class="hljs-string">"stream"</span>: <span class="hljs-literal">False</span>,

	}
	)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-33uaik"><strong><em>🕸️ Define a web-search tool for the agent</em></strong></p> <p data-svelte-h="svelte-zsnc9d">To enable our agent to fetch fresh information from the web, we install the <code>duckduckgo-search</code> package, which provides a simple Python API for performing real-time DuckDuckGo queries.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->!pip install -q ddgs<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> ddgs <span class="hljs-keyword">import</span> DDGS

	<span class="hljs-comment"># define a tool to search the web</span>
	<span class="hljs-meta">@tool()</span>
	<span class="hljs-keyword">def</span> <span class="hljs-title function_">web_search</span>(<span class="hljs-params">query:<span class="hljs-built_in">str</span>, max_results=<span class="hljs-number">5</span></span>)-><span class="hljs-built_in">dict</span>:
	results = <span class="hljs-string">""</span>
	<span class="hljs-keyword">with</span> DDGS() <span class="hljs-keyword">as</span> ddgs:
	results = [r <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> ddgs.text(query, max_results=max_results)]
	<span class="hljs-keyword">return</span> results<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1p9p656"><strong><em>📞 My first call to the agent</em></strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> agent = Agent(model=model, tools=[web_search])
	>> response = agent(<span class="hljs-string">"What are AWS Neuron devices?"</span>)<!-- HTML_TAG_END --></pre></div> <pre data-svelte-h="svelte-b3z13l">Tool #1: web_search
	</pre> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> <span class="hljs-keyword">from</span> pprint <span class="hljs-keyword">import</span> pprint
	>> pprint(response.message[<span class="hljs-string">"content"</span>][<span class="hljs-number">0</span>][<span class="hljs-string">"text"</span>])<!-- HTML_TAG_END --></pre></div> <pre data-svelte-h="svelte-hu8bzt">('AWS Neuron devices refer to the accelerated machine learning chips, '
	'specifically AWS Inferentia and AWS Trainium, which are designed to enhance '
	'AI and deep learning workloads on Amazon EC2 instances. These chips are '
	'optimized for high-density, scalable, and cost-effective inference and '
	'training respectively. \n'
	'\n'
	'AWS Neuron is a software development kit (SDK) that facilitates the '
	'optimization of AI and deep learning models for deployment on these Neuron '
	'devices. It transforms models from popular frameworks like PyTorch or '
	'TensorFlow into a specialized, low-level representation for efficient '
	'execution on AWS Inferentia and Trainium.\n'
	'\n'
	'The AWS Neuron Kernel Mode Driver establishes a communication channel (FWIO) '
	'that allows the driver and an application to send specific commands to the '
	'device. \n'
	'\n'
	"Additionally, there's a Neuron backend for Hugging Face's Text Generation "
	'Inference (TGI) that allows deployment on AWS Trainium and Inferentia family '
	'of chips.\n'
	'\n'
	'For more detailed technical information, you can refer to the provided links '
	'in the web search results.')
	</pre> <p data-svelte-h="svelte-zk0te6">Here’s a concise breakdown of what the logs show:</p> <ul data-svelte-h="svelte-6jhalu"><li><p><strong>Initial payload:</strong><br>
	The first block (<code>payload=...</code>) is the request sent to the SageMaker endpoint.<br>
	It includes the user message <em>(“Explain what is a Neuron device”)</em>, the tool schema,<br>
	and generation parameters (<code>max_tokens</code>, <code>temperature</code>, etc.).</p></li> <li><p><strong>Model triggers a tool:</strong><br>
	The first response contains a <code>tool_call</code>, meaning the model decided<br>
	to use the <code>web_search</code> tool.<br>
	It sends arguments like:<br> <code>{"query": "what is a Neuron device", "max_results": 1}</code>.</p></li> <li><p><strong>Tool execution:</strong><br>
	The notebook executes the <code>web_search</code> function locally.<br>
	It runs the DuckDuckGo/Bing query and returns the results to the agent.</p></li> <li><p><strong>Second payload:</strong><br>
	The agent sends a new request to the endpoint that includes:</p> <ul><li>the original user message</li> <li>the tool call</li> <li>the tool result<br>
	This allows the model to produce a final answer with the retrieved information.</li></ul></li> <li><p><strong>Final model response:</strong><br>
	The last block shows the assistant’s answer generated after incorporating<br>
	the tool’s output. (In this example, the model misinterprets <em>Neuron device</em>,<br>
	but the flow is correct.)</p></li></ul> <p data-svelte-h="svelte-u2ndi7">Overall, this demonstrates the complete agent loop:</p> <ol data-svelte-h="svelte-18wsyxd"><li>Model decides to call a tool</li> <li>Tool runs</li> <li>Result is injected back into the model</li> <li>Model produces the final answer</li></ol> <p data-svelte-h="svelte-120afqs">✨ <strong>The entire iteration—model → tool → model—completed in under 10 seconds.</strong></p> <h3 class="relative group"><a id="3-use-my-agent-with-hf-mcp-server-" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#3-use-my-agent-with-hf-mcp-server-"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>3. Use my Agent with HF MCP server 🤗</span></h3> <p data-svelte-h="svelte-18kgojw">The <strong>Hugging Face MCP (Model Context Protocol) Server</strong> is a service that lets your MCP-compatible AI assistant (for example in VS Code, Cursor, Zed, or Claude Desktop) <strong>connect directly to the Hugging Face Hub</strong>.</p> <p data-svelte-h="svelte-14kj8q3">Once connected, your assistant can:</p> <ul data-svelte-h="svelte-qavied"><li>Search and explore Hub resources (models, datasets, Spaces, papers).</li> <li>Invoke community tools (Gradio apps on Spaces) as functions, returning results—metadata, links, context.</li></ul> <p data-svelte-h="svelte-mil2tb">It effectively turns the Hugging Face Hub into a tool-server your agent can use, enhancing its tool-calling and exploration capabilities.</p> <p data-svelte-h="svelte-124e5nw">Learn more about the MCP server <a href="https://huggingface.co/docs/hub/en/hf-mcp-server" rel="nofollow">here</a>.</p> <p data-svelte-h="svelte-ldpm4e"><strong><em>🔧 Configure the MCP client</em></strong></p> <p data-svelte-h="svelte-w2k4re">To use the Hugging Face MCP server with your agent, you must first configure it at:<br>
	👉 <strong><a href="https://huggingface.co/settings/mcp" rel="nofollow">https://huggingface.co/settings/mcp</a></strong></p> <p data-svelte-h="svelte-dmeum2">On this page, you can generate an MCP-compatible access token and configure which built-in tools your agent is allowed to use.</p> <p data-svelte-h="svelte-1lwgf4d">Once enabled, your agent gains access to a rich set of Hub-powered tools, including:</p> <ul data-svelte-h="svelte-t79a02"><li><strong>Spaces Semantic Search</strong> – find the best AI apps on the Hub using natural language.</li> <li><strong>Model Search</strong> – query models by task, library, license, or metadata.</li> <li><strong>Documentation Semantic Search</strong> – search across all HF documentation.</li> <li><strong>Hub Repository Details</strong> – retrieve structured metadata for models, datasets, and Spaces.</li> <li><strong>Papers Semantic Search</strong> – find AI research papers semantically.</li> <li><strong>Dataset Search</strong> – search datasets by author, tags, modality, etc.</li> <li><strong>Run and Manage Jobs</strong> – schedule and monitor jobs on Hugging Face infrastructure.</li> <li><strong>Include README files</strong> – allow agents to read repository READMEs in results.</li></ul> <p data-svelte-h="svelte-d2nn39">This integration lets your agent use the Hugging Face Hub as a powerful knowledge and tooling backend.</p> <p data-svelte-h="svelte-6y84cb"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/sagemaker/mcp-settings.png" alt="MCP Settings"></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> strands.tools.mcp <span class="hljs-keyword">import</span> MCPClient
	<span class="hljs-keyword">from</span> mcp.client.streamable_http <span class="hljs-keyword">import</span> streamablehttp_client
	<span class="hljs-comment">#MCP URL</span>
	hf_mcp_url = <span class="hljs-string">"https://huggingface.co/mcp"</span>
	hf_token = os.environ.get(<span class="hljs-string">"HF_TOKEN"</span>)

	mcp_client = MCPClient(
	<span class="hljs-keyword">lambda</span>: streamablehttp_client(
	url=hf_mcp_url,
	headers={<span class="hljs-string">"Authorization"</span>: <span class="hljs-string">f"Bearer <span class="hljs-subst">{hf_token}</span>"</span>} <span class="hljs-keyword">if</span> hf_token <span class="hljs-keyword">else</span> {},
	timeout=<span class="hljs-number">120</span>,
	sse_read_timeout=<span class="hljs-number">600</span>,
	)
	)<!-- HTML_TAG_END --></pre></div> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-comment">#Use the MCP client as a tool</span>
	agent = Agent(model=model, tools=[mcp_client])<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-dvuyt4">We are now ready to use the MCP client as a tool in our agent. we will test it to:</p> <ul data-svelte-h="svelte-jbxp4e"><li>Search for a Space that can transcribe audio files</li> <li>Search for trending image generation models</li> <li>Search the doc to export a model to neuron with optimum</li></ul> <p data-svelte-h="svelte-dcp6sk"><strong><em>🔉 Space search</em></strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> audio_space_search = agent(<span class="hljs-string">"Search for a Space that can transcribe audio files"</span>)
	>> pprint(audio_space_search.message[<span class="hljs-string">"content"</span>][<span class="hljs-number">0</span>][<span class="hljs-string">"text"</span>])<!-- HTML_TAG_END --></pre></div> <pre data-svelte-h="svelte-ur7fd5">Tool #1: space_search
	</pre> <p data-svelte-h="svelte-ukpoqp"><strong><em>📈 Trending model search</em></strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> image_model_search = agent(<span class="hljs-string">"Search for trending image generation models"</span>)
	>> pprint(image_model_search.message[<span class="hljs-string">"content"</span>][<span class="hljs-number">0</span>][<span class="hljs-string">"text"</span>])<!-- HTML_TAG_END --></pre></div> <pre data-svelte-h="svelte-twdf9x">Tool #2: model_search
	</pre> <p data-svelte-h="svelte-153c0en"><strong><em>📚 Documentation search</em></strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->>> doc_search = agent(<span class="hljs-string">"How to export a model to neuron with optimum"</span>)
	>> pprint(doc_search.message[<span class="hljs-string">"content"</span>][<span class="hljs-number">0</span>][<span class="hljs-string">"text"</span>])<!-- HTML_TAG_END --></pre></div> <pre data-svelte-h="svelte-1b1asbr">To export a Hugging Face model to Neuron using Optimum, you'll need to follow these steps:

	1. Install Optimum: If you haven't already, install Optimum using pip:

	```bash
	pip install optimum
	```

	2. Choose a Model: Select a model from the Hugging Face Model Hub that you want to export. For example, let's use `bert-base-uncased` for a text classification task.

	3. Load the Model: Use the `transformers` library to load your chosen model.

	```python
	from transformers import AutoModelForSequenceClassification, AutoTokenizer

	model_name = "bert-base-uncased"
	model = AutoModelForSequenceClassification.from_pretrained(model_name)
	tokenizer = AutoTokenizer.from_pretrained(model_name)
	```

	4. Prepare for Export: Optimum provides a `neuron_quantized` exporter. First, you need to quantize your model.

	```python
	from optimum.neuron.quantization import NeuronQuantizer

	quantizer = NeuronQuantizer(model)
	quantized_model = quantizer.fit(model, dataset=your_dataset)
	```

	Replace `your_dataset` with a suitable dataset for fine-tuning or evaluation.

	5. Export the Model: Use the `export` method provided by Optimum to export the quantized model to Neuron format.

	```python
	from optimum.neuron.export import export_neuron

	export_neuron(quantized_model, "bert-base-uncased-neuron")
	```

	This will save the exported model in Neuron format in a directory named `bert-base-uncased-neuron`.

	6. Load on Neuron: Finally, load the exported model on Neuron hardware using the Neuron SDK.

	Please note that the exact steps might vary slightly based on the model you choose and the specific Neuron hardware you are targeting. Always refer to the [Optimum documentation](https://huggingface.co/docs/optimum/index) and the [Neuron SDK documentation](https://docs.neuron.app/) for the most accurate and up-to-date instructions.

	If you encounter any specific issues or need further assistance, feel free to ask!("To export a Hugging Face model to Neuron using Optimum, you'll need to "
	'follow these steps:\n'
	'\n'
	"1. Install Optimum: If you haven't already, install Optimum using pip:\n"
	'\n'
	' ```bash\n'
	' pip install optimum\n'
	' ```\n'
	'\n'
	'2. Choose a Model: Select a model from the Hugging Face Model Hub that '
	"you want to export. For example, let's use `bert-base-uncased` for a text "
	'classification task.\n'
	'\n'
	'3. Load the Model: Use the `transformers` library to load your chosen '
	'model.\n'
	'\n'
	' ```python\n'
	' from transformers import AutoModelForSequenceClassification, '
	'AutoTokenizer\n'
	'\n'
	' model_name = "bert-base-uncased"\n'
	' model = AutoModelForSequenceClassification.from_pretrained(model_name)\n'
	' tokenizer = AutoTokenizer.from_pretrained(model_name)\n'
	' ```\n'
	'\n'
	'4. Prepare for Export: Optimum provides a `neuron_quantized` exporter. '
	'First, you need to quantize your model.\n'
	'\n'
	' ```python\n'
	' from optimum.neuron.quantization import NeuronQuantizer\n'
	'\n'
	' quantizer = NeuronQuantizer(model)\n'
	' quantized_model = quantizer.fit(model, dataset=your_dataset)\n'
	' ```\n'
	'\n'
	' Replace `your_dataset` with a suitable dataset for fine-tuning or '
	'evaluation.\n'
	'\n'
	'5. Export the Model: Use the `export` method provided by Optimum to '
	'export the quantized model to Neuron format.\n'
	'\n'
	' ```python\n'
	' from optimum.neuron.export import export_neuron\n'
	'\n'
	' export_neuron(quantized_model, "bert-base-uncased-neuron")\n'
	' ```\n'
	'\n'
	' This will save the exported model in Neuron format in a directory named '
	'`bert-base-uncased-neuron`.\n'
	'\n'
	'6. Load on Neuron: Finally, load the exported model on Neuron hardware '
	'using the Neuron SDK.\n'
	'\n'
	'Please note that the exact steps might vary slightly based on the model you '
	'choose and the specific Neuron hardware you are targeting. Always refer to '
	'the [Optimum documentation](https://huggingface.co/docs/optimum/index) and '
	'the [Neuron SDK documentation](https://docs.neuron.app/) for the most '
	'accurate and up-to-date instructions.\n'
	'\n'
	'If you encounter any specific issues or need further assistance, feel free '
	'to ask!')
	</pre> <h3 class="relative group"><a id="-understanding-the-agent-output" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#-understanding-the-agent-output"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>🧩 Understanding the Agent Output</span></h3> <p data-svelte-h="svelte-dzd972">The long log you see comes from the agent’s <strong>full reasoning loop</strong>, not just the final answer.<br>
	Here’s what’s happening:</p> <ul data-svelte-h="svelte-nyqf2v"><li>The agent keeps a <strong>persistent conversation history</strong>, so past queries and tool calls are always included.</li> <li>When you run <code>agent("…")</code>, the model decides whether it should call a tool (e.g., HF MCP search).</li> <li>If a tool is needed, the agent:
	<ol><li>emits a <code>tool_call</code>,</li> <li>executes the tool,</li> <li>appends the tool result back into the conversation,</li> <li>asks the model again to produce the final answer.</li></ol></li> <li>The printed value (<code>doc_search.message["content"][0]["text"]</code>) only shows the <strong>assistant’s final answer</strong>, but the log shows the <strong>full chain</strong>:<br>
	user message → tool call → tool output → final completion.</li></ul> <hr> <blockquote class="tip"><p data-svelte-h="svelte-1c76scg">📍 Find the complete example on GitHub <a href="https://github.com/huggingface/hub-docs/tree/main/notebooks/sagemaker-sdk/neuron-agent-inf/sagemaker-notebook.ipynb" rel="nofollow">here</a>!</p></blockquote> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/hub-docs/blob/main/docs/sagemaker/source/examples/sagemaker-sdk-neuron-agent-inf.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg> <span data-svelte-h="svelte-zjs2n5"><span class="underline">Update</span> on GitHub</span></a> <p></p>

	<script>
	{
	__sveltekit_16a821l = {
	assets: "/docs/sagemaker/pr_2188/en",
	base: "/docs/sagemaker/pr_2188/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/sagemaker/pr_2188/en/_app/immutable/entry/start.48a18f09.js"),
	import("/docs/sagemaker/pr_2188/en/_app/immutable/entry/app.8481cdda.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 10],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 92.2 kB
Xet hash:: 02e4d28dd296111c8e1b80de57a9389182a3468445635c7e607e88ae1209310a

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.