Buckets:

hf-doc-build
/

doc

Files

xet

hf-doc-build/doc / optimum-neuron /v0.2.0.dev2 /en /guides /export_model.html

rtrm

about 1 month ago

download

raw

69.3 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Export a model to Inferentia","local":"export-a-model-to-inferentia","sections":[{"title":"Summary","local":"summary","sections":[],"depth":2},{"title":"Why compile to Neuron model?","local":"why-compile-to-neuron-model","sections":[],"depth":2},{"title":"Exporting a model to Neuron using the CLI","local":"exporting-a-model-to-neuron-using-the-cli","sections":[{"title":"Exporting standard (non-LLM) models","local":"exporting-standard-non-llm-models","sections":[],"depth":3},{"title":"Exporting Stable Diffusion to Neuron","local":"exporting-stable-diffusion-to-neuron","sections":[],"depth":3},{"title":"Exporting Stable Diffusion XL to Neuron","local":"exporting-stable-diffusion-xl-to-neuron","sections":[],"depth":3},{"title":"Exporting LLMs to Neuron","local":"exporting-llms-to-neuron","sections":[],"depth":3}],"depth":2},{"title":"Exporting a model to Neuron programmatically via NeuronModel","local":"exporting-a-model-to-neuron-programmatically-via-neuronmodel","sections":[],"depth":2},{"title":"Exporting neuron models using NeuronX TGI","local":"exporting-neuron-models-using-neuronx-tgi","sections":[],"depth":2}],"depth":1}">
	<link href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/entry/start.c3692dcd.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/chunks/scheduler.85c25b89.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/chunks/singletons.bf318e21.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/chunks/paths.4f2bc42b.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/entry/app.40ef12d9.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/chunks/index.c9bcf812.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/nodes/0.8386078c.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/chunks/each.e59479a4.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/nodes/8.d3dd8829.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/chunks/Tip.d8f753fa.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/chunks/CodeBlock.c004bd26.js">
	<link rel="modulepreload" href="/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/chunks/index.9790a2b6.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Export a model to Inferentia","local":"export-a-model-to-inferentia","sections":[{"title":"Summary","local":"summary","sections":[],"depth":2},{"title":"Why compile to Neuron model?","local":"why-compile-to-neuron-model","sections":[],"depth":2},{"title":"Exporting a model to Neuron using the CLI","local":"exporting-a-model-to-neuron-using-the-cli","sections":[{"title":"Exporting standard (non-LLM) models","local":"exporting-standard-non-llm-models","sections":[],"depth":3},{"title":"Exporting Stable Diffusion to Neuron","local":"exporting-stable-diffusion-to-neuron","sections":[],"depth":3},{"title":"Exporting Stable Diffusion XL to Neuron","local":"exporting-stable-diffusion-xl-to-neuron","sections":[],"depth":3},{"title":"Exporting LLMs to Neuron","local":"exporting-llms-to-neuron","sections":[],"depth":3}],"depth":2},{"title":"Exporting a model to Neuron programmatically via NeuronModel","local":"exporting-a-model-to-neuron-programmatically-via-neuronmodel","sections":[],"depth":2},{"title":"Exporting neuron models using NeuronX TGI","local":"exporting-neuron-models-using-neuronx-tgi","sections":[],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="export-a-model-to-inferentia" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#export-a-model-to-inferentia"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Export a model to Inferentia</span></h1> <h2 class="relative group"><a id="summary" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#summary"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Summary</span></h2> <p data-svelte-h="svelte-166ewzk">Exporting a PyTorch model to Neuron model is as simple as</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron \
	--model bert-base-uncased \
	--sequence_length 128 \
	--batch_size 1 \
	bert_neuron/<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-b35baa">Check out the help for more options:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --<span class="hljs-built_in">help</span><!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="why-compile-to-neuron-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#why-compile-to-neuron-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Why compile to Neuron model?</span></h2> <p data-svelte-h="svelte-nznvvy">AWS provides two generations of the Inferentia accelerator built for machine learning inference with higher throughput, lower latency but lower cost: <a href="https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/arch/neuron-hardware/inf2-arch.html" rel="nofollow">inf2 (NeuronCore-v2)</a> and <a href="https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/arch/neuron-hardware/inf1-arch.html#aws-inf1-arch" rel="nofollow">inf1 (NeuronCore-v1)</a>.</p> <p data-svelte-h="svelte-1und41a">In production environments, to deploy 🤗 <a href="https://huggingface.co/docs/transformers/index" rel="nofollow">Transformers</a> models on Neuron devices, you need to compile your models and export them to a serialized format before inference. Through Ahead-Of-Time (AOT) compilation with Neuron Compiler( <a href="https://awsdocs-neuron.readthedocs-hosted.com/en/latest/release-notes/compiler/neuronx-cc/index.html" rel="nofollow">neuronx-cc</a> or <a href="https://awsdocs-neuron.readthedocs-hosted.com/en/latest/release-notes/compiler/neuron-cc/neuron-cc.html" rel="nofollow">neuron-cc</a> ), your models will be converted to serialized and optimized <a href="https://pytorch.org/docs/stable/generated/torch.jit.ScriptModule.html" rel="nofollow">TorchScript modules</a>.</p> <p data-svelte-h="svelte-2ri5fz">Although pre-compilation avoids overhead during the inference, a compiled Neuron model has some limitations:</p> <ul data-svelte-h="svelte-1ln58gr"><li>The input shapes and data types used during the compilation cannot be changed.</li> <li>Neuron models are specialized for each hardware and SDK version, which means:<ul><li>Models compiled with Neuron can no longer be executed in non-Neuron environment.</li> <li>Models compiled for inf1 (NeuronCore-v1) are not compatible with inf2 (NeuronCore-v2), and vice versa.</li> <li>Models compiled for an SDK version are (generally) not compatible with another SDK version.</li></ul></li></ul> <p data-svelte-h="svelte-189k6c5">In this guide, we’ll show you how to export your models to serialized models optimized for Neuron devices.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1u9pown">🤗 Optimum provides support for the Neuron export by leveraging configuration objects.
	These configuration objects come ready made for a number of model architectures, and are designed to be easily extendable to other architectures.</p> <p data-svelte-h="svelte-430w7w"><strong>To check the supported architectures, go to the <a href="../package_reference/configuration">configuration reference page</a>.</strong></p></div> <h2 class="relative group"><a id="exporting-a-model-to-neuron-using-the-cli" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#exporting-a-model-to-neuron-using-the-cli"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Exporting a model to Neuron using the CLI</span></h2> <p data-svelte-h="svelte-jz73et">To export a 🤗 Transformers model to Neuron, you’ll first need to install some extra dependencies:</p> <p data-svelte-h="svelte-1qm5x97"><strong>For Inf2</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install optimum-neuron[neuronx]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-257sak"><strong>For Inf1</strong></p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install optimum-neuron[neuron]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-cb2pjp">The Optimum Neuron export can be used through Optimum command-line:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --<span class="hljs-built_in">help</span>

	usage: optimum-cli <span class="hljs-built_in">export</span> neuron [-h] -m MODEL [--task TASK] [--atol ATOL] [--cache_dir CACHE_DIR] [--trust-remote-code]
	[--compiler_workdir COMPILER_WORKDIR] [--disable-validation] [--auto_cast {none,matmul,all}]
	[--auto_cast_type {bf16,fp16,tf32}] [--dynamic-batch-size] [--num_cores NUM_CORES] [--unet UNET]
	[--output_hidden_states] [--output_attentions] [--batch_size BATCH_SIZE]
	[--sequence_length SEQUENCE_LENGTH] [--num_beams NUM_BEAMS] [--num_choices NUM_CHOICES]
	[--num_channels NUM_CHANNELS] [--width WIDTH] [--height HEIGHT]
	[--num_images_per_prompt NUM_IMAGES_PER_PROMPT] [-O1 \| -O2 \| -O3]
	output

	optional arguments:
	-h, --<span class="hljs-built_in">help</span> show this <span class="hljs-built_in">help</span> message and <span class="hljs-built_in">exit</span>
	-O1 Enables the core performance optimizations <span class="hljs-keyword">in</span> the compiler, <span class="hljs-keyword">while</span> also minimizing compile time.
	-O2 [Default] Provides the best balance between model performance and compile time.
	-O3 May provide additional model execution performance but may incur longer compile <span class="hljs-built_in">times</span> and higher host
	memory usage during model compilation.

	Required arguments:
	-m MODEL, --model MODEL
	Model ID on huggingface.co or path on disk to load model from.
	output Path indicating the directory <span class="hljs-built_in">where</span> to store generated Neuronx compiled TorchScript model.

	Optional arguments:
	--task TASK The task to <span class="hljs-built_in">export</span> the model <span class="hljs-keyword">for</span>. If not specified, the task will be auto-inferred based on the model.
	Available tasks depend on the model, but are among: [<span class="hljs-string">'audio-classification'</span>, <span class="hljs-string">'audio-frame-
	classification'</span>, <span class="hljs-string">'audio-xvector'</span>, <span class="hljs-string">'automatic-speech-recognition'</span>, <span class="hljs-string">'conversational'</span>, <span class="hljs-string">'depth-estimation'</span>,
	<span class="hljs-string">'feature-extraction'</span>, <span class="hljs-string">'fill-mask'</span>, <span class="hljs-string">'image-classification'</span>, <span class="hljs-string">'image-segmentation'</span>, <span class="hljs-string">'image-to-image'</span>,
	<span class="hljs-string">'image-to-text'</span>, <span class="hljs-string">'mask-generation'</span>, <span class="hljs-string">'masked-im'</span>, <span class="hljs-string">'multiple-choice'</span>, <span class="hljs-string">'object-detection'</span>, <span class="hljs-string">'question-
	answering'</span>, <span class="hljs-string">'semantic-segmentation'</span>, <span class="hljs-string">'text-to-audio'</span>, <span class="hljs-string">'text-generation'</span>, <span class="hljs-string">'text2text-generation'</span>, <span class="hljs-string">'text-
	classification'</span>, <span class="hljs-string">'token-classification'</span>, <span class="hljs-string">'zero-shot-image-classification'</span>, <span class="hljs-string">'zero-shot-object-detection'</span>,
	<span class="hljs-string">'stable-diffusion'</span>, <span class="hljs-string">'stable-diffusion-xl'</span>].
	--atol ATOL If specified, the absolute difference tolerance when validating the model. Otherwise, the default atol
	<span class="hljs-keyword">for</span> the model will be used.
	--cache_dir CACHE_DIR
	Path indicating <span class="hljs-built_in">where</span> to store cache.
	--trust-remote-code Allow to use custom code <span class="hljs-keyword">for</span> the modeling hosted <span class="hljs-keyword">in</span> the model repository. This option should only be <span class="hljs-built_in">set</span>
	<span class="hljs-keyword">for</span> repositories you trust and <span class="hljs-keyword">in</span> <span class="hljs-built_in">which</span> you have <span class="hljs-built_in">read</span> the code, as it will execute on your <span class="hljs-built_in">local</span> machine
	arbitrary code present <span class="hljs-keyword">in</span> the model repository.
	--compiler_workdir COMPILER_WORKDIR
	Path indicating the directory <span class="hljs-built_in">where</span> to store intermediary files generated by Neuronx compiler.
	--disable-validation Whether to <span class="hljs-built_in">disable</span> the validation of inference on neuron device compared to the outputs of original
	PyTorch model on CPU.
	--auto_cast {none,matmul,all}
	Whether to cast operations from FP32 to lower precision to speed up the inference. Can be `<span class="hljs-string">"none"</span>`,
	`<span class="hljs-string">"matmul"</span>` or `<span class="hljs-string">"all"</span>`.
	--auto_cast_type {bf16,fp16,tf32}
	The data <span class="hljs-built_in">type</span> to cast FP32 operations to when auto-cast mode is enabled. Can be `<span class="hljs-string">"bf16"</span>`, `<span class="hljs-string">"fp16"</span>` or
	`<span class="hljs-string">"tf32"</span>`.
	--dynamic-batch-size Enable dynamic batch size <span class="hljs-keyword">for</span> neuron compiled model. If this option is enabled, the input batch size can
	be a multiple of the batch size during the compilation, but it comes with a potential tradeoff <span class="hljs-keyword">in</span> terms
	of latency.
	--num_cores NUM_CORES
	The number of cores on <span class="hljs-built_in">which</span> the model should be deployed (text-generation only).
	--unet UNET UNet model ID on huggingface.co or path on disk to load model from. This will replace the unet <span class="hljs-keyword">in</span> the
	original Stable Diffusion pipeline.
	--output_hidden_states
	Whether or not <span class="hljs-keyword">for</span> the traced model to <span class="hljs-built_in">return</span> the hidden states of all layers.
	--output_attentions Whether or not <span class="hljs-keyword">for</span> the traced model to <span class="hljs-built_in">return</span> the attentions tensors of all attention layers.

	Input shapes:
	--batch_size BATCH_SIZE
	Batch size that the Neuronx-cc compiler exported model will be able to take as input.
	--sequence_length SEQUENCE_LENGTH
	Sequence length that the Neuronx-cc compiler exported model will be able to take as input.
	--num_beams NUM_BEAMS
	Number of beams <span class="hljs-keyword">for</span> beam search that the Neuronx-cc compiler exported model will be able to take as
	input.
	--num_choices NUM_CHOICES
	Only <span class="hljs-keyword">for</span> the multiple-choice task. Num choices that the Neuronx-cc compiler exported model will be able
	to take as input.
	--num_channels NUM_CHANNELS
	Image tasks only. Number of channels that the Neuronx-cc compiler exported model will be able to take as
	input.
	--width WIDTH Image tasks only. Width that the Neuronx-cc compiler exported model will be able to take as input.
	--height HEIGHT Image tasks only. Height that the Neuronx-cc compiler exported model will be able to take as input.
	--num_images_per_prompt NUM_IMAGES_PER_PROMPT
	Stable diffusion only. Number of images per prompt that the Neuronx-cc compiler exported model will be
	able to take as input.
	<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="exporting-standard-non-llm-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#exporting-standard-non-llm-models"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Exporting standard (non-LLM) models</span></h3> <p data-svelte-h="svelte-p91rke">Most models present on the Hugging Face hub can be straightforwardly exported using torch trace, then converted to serialized and optimized TorchScript modules.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><img title="Compilation flow" alt="Compilation flow" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/optimum/neuron/inf_compile_flow.png"> <p data-svelte-h="svelte-di565y"><strong>NEFF</strong>: Neuron Executable File Format which is a binary executable on Neuron devices.</p></div> <p data-svelte-h="svelte-9ahyv2">When exporting a model, two sets of export arguments must be passed:</p> <ul data-svelte-h="svelte-uw9cro"><li><code>compiler_args</code> are optional arguments for the compiler, these arguments usually control how the compiler makes tradeoff between the inference performance (latency and throughput) and the accuracy,</li> <li><code>input_shapes</code> are mandatory static shape information that you need to send to the neuron compiler.</li></ul> <p data-svelte-h="svelte-1ldr9ls">Please type the following command to see all export parameters:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron -h<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-eag7fc">Exporting a standard NLP model can be done as follows:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --model distilbert-base-uncased-distilled-squad \
	--batch_size 1 --sequence_length 16 \
	--auto_cast matmul --auto_cast_type fp16 \
	distilbert_base_uncased_squad_neuron/<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1k4xv5r">Here the model was exported with a static input shape of <code>(1, 16)</code>, and with compiler arguments specifying
	that matmul operation must be performed with <code>float16</code> precision for faster inference.</p> <p data-svelte-h="svelte-9jbb9y">After export, you should see the following logs which validate the model on Neuron devices by comparing with PyTorch model on CPU:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->Validating Neuron model...
	-[✓] Neuron model output names match reference model (last_hidden_state)
	- Validating Neuron Model output <span class="hljs-string">"last_hidden_state"</span>:
	-[✓] (1, 16, 32) matches (1, 16, 32)
	-[✓] all values close (atol: 0.0001)
	The Neuronx <span class="hljs-built_in">export</span> succeeded and the exported model was saved at: distilbert_base_uncased_squad_neuron/<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-v1u856">This exports a neuron-compiled TorchScript module of the checkpoint defined by the <code>--model</code> argument.</p> <p data-svelte-h="svelte-vql48n">As you can see, the task was automatically detected. This was possible because the model was on the Hub. For local models, providing the <code>--task</code> argument is needed or it will default to the model architecture without any task specific head:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --model local_path --task question-answering --batch_size 1 --sequence_length 16 --dynamic-batch-size distilbert_base_uncased_squad_neuron/<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1hgclei">Note that providing the <code>--task</code> argument for a model on the Hub will disable the automatic task detection. The resulting <code>model.neuron</code> file, can then be loaded and run on Neuron devices.</p> <p data-svelte-h="svelte-yhg472">For each model architecture, you can find the list of supported tasks via the <code>~exporters.tasks.TasksManager</code>. For example, for DistilBERT, for the Neuron export, we have:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> optimum.exporters.tasks <span class="hljs-keyword">import</span> TasksManager
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> optimum.exporters.neuron.model_configs <span class="hljs-keyword">import</span> * <span class="hljs-comment"># Register neuron specific configs to the TasksManager</span>

	<span class="hljs-meta">>>> </span>distilbert_tasks = <span class="hljs-built_in">list</span>(TasksManager.get_supported_tasks_for_model_type(<span class="hljs-string">"distilbert"</span>, <span class="hljs-string">"neuron"</span>).keys())
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(distilbert_tasks)
	[<span class="hljs-string">'feature-extraction'</span>, <span class="hljs-string">'fill-mask'</span>, <span class="hljs-string">'multiple-choice'</span>, <span class="hljs-string">'question-answering'</span>, <span class="hljs-string">'text-classification'</span>, <span class="hljs-string">'token-classification'</span>]<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1ab1ldn">You can then pass one of these tasks to the <code>--task</code> argument in the <code>optimum-cli export neuron</code> command, as mentioned above.</p> <p data-svelte-h="svelte-3rh7j4">Once exported, the neuron model can be used for inference directly with the <code>NeuronModelForXXX</code> class:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer
	<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronModelForSequenceClassification

	<span class="hljs-meta">>>> </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">"./distilbert-base-uncased-finetuned-sst-2-english_neuron/"</span>)
	<span class="hljs-meta">>>> </span>model = NeuronModelForSequenceClassification.from_pretrained(<span class="hljs-string">"./distilbert-base-uncased-finetuned-sst-2-english_neuron/"</span>)

	<span class="hljs-meta">>>> </span>inputs = tokenizer(<span class="hljs-string">"Hamilton is considered to be the best musical of human history."</span>, return_tensors=<span class="hljs-string">"pt"</span>)
	<span class="hljs-meta">>>> </span>logits = model(**inputs).logits
	<span class="hljs-meta">>>> </span><span class="hljs-built_in">print</span>(model.config.id2label[logits.argmax().item()])
	<span class="hljs-string">'POSITIVE'</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1ugid4e">As you see, there is no need to pass the neuron arguments used during the export as they are
	saved in a <code>config.json</code> file, and will be restored automatically by <code>NeuronModelForXXX</code> class.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">Be careful, inputs are always padded to the shapes used for the compilation, and the padding brings computation overhead.
	Adjust the static shapes to be higher than the shape of the inputs that you will feed into the model during the inference, but not much more.</div> <h3 class="relative group"><a id="exporting-stable-diffusion-to-neuron" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#exporting-stable-diffusion-to-neuron"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Exporting Stable Diffusion to Neuron</span></h3> <p data-svelte-h="svelte-vfmt4g">With the Optimum CLI you can compile components in the Stable Diffusion pipeline to gain acceleration on neuron devices during the inference.</p> <p data-svelte-h="svelte-1lozjup">So far, we support the export of following components in the pipeline:</p> <ul data-svelte-h="svelte-1bxicg6"><li>CLIP text encoder</li> <li>U-Net</li> <li>VAE encoder</li> <li>VAE decoder</li></ul> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-9etaml">“These blocks are chosen because they represent the bulk of the compute in the pipeline, and performance benchmarking has shown that running them on Neuron yields significant performance benefit.”</p> <p data-svelte-h="svelte-1dh7n4s">Besides, don’t hesitate to tweak the compilation configuration to find the best tradeoff between performance v.s accuracy in your use case. By default, we suggest casting FP32 matrix multiplication operations to BF16 which offers good performance with moderate sacrifice of the accuracy. Check out the guide from <a href="https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/appnotes/neuronx-cc/neuronx-cc-training-mixed-precision.html#neuronx-cc-training-mixed-precision" rel="nofollow">AWS Neuron documentation</a> to better understand the options for your compilation.</p></div> <p data-svelte-h="svelte-1s0jvqd">Exporting a stable diffusion checkpoint can be done using the CLI:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --model stabilityai/stable-diffusion-2-1-base \
	--task stable-diffusion \
	--batch_size 1 \
	--height 512 `<span class="hljs-comment"># height in pixels of generated image, eg. 512, 768` \</span>
	--width 512 `<span class="hljs-comment"># width in pixels of generated image, eg. 512, 768` \</span>
	--num_images_per_prompt 4 `<span class="hljs-comment"># number of images to generate per prompt, defaults to 1` \</span>
	--auto_cast matmul `<span class="hljs-comment"># cast only matrix multiplication operations` \</span>
	--auto_cast_type bf16 `<span class="hljs-comment"># cast operations from FP32 to BF16` \</span>
	sd_neuron/<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="exporting-stable-diffusion-xl-to-neuron" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#exporting-stable-diffusion-xl-to-neuron"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Exporting Stable Diffusion XL to Neuron</span></h3> <p data-svelte-h="svelte-12l5vu1">Similar to Stable Diffusion, you will be able to use Optimum CLI to compile components in the SDXL pipeline for inference on neuron devices.</p> <p data-svelte-h="svelte-1cwhii6">We support the export of following components in the pipeline to boost the speed:</p> <ul data-svelte-h="svelte-17e9k9s"><li>Text encoder</li> <li>Second text encoder</li> <li>U-Net (a three times larger UNet than the one in Stable Diffusion pipeline)</li> <li>VAE encoder</li> <li>VAE decoder</li></ul> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-r3q4ut">“Stable Diffusion XL works especially well with images between 768 and 1024.”</p></div> <p data-svelte-h="svelte-112a1oc">Exporting a SDXL checkpoint can be done using the CLI:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --model stabilityai/stable-diffusion-xl-base-1.0 \
	--task stable-diffusion-xl \
	--batch_size 1 \
	--height 1024 `<span class="hljs-comment"># height in pixels of generated image, eg. 768, 1024` \</span>
	--width 1024 `<span class="hljs-comment"># width in pixels of generated image, eg. 768, 1024` \</span>
	--num_images_per_prompt 4 `<span class="hljs-comment"># number of images to generate per prompt, defaults to 1` \</span>
	--auto_cast matmul `<span class="hljs-comment"># cast only matrix multiplication operations` \</span>
	--auto_cast_type bf16 `<span class="hljs-comment"># cast operations from FP32 to BF16` \</span>
	sd_neuron/<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="exporting-llms-to-neuron" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#exporting-llms-to-neuron"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Exporting LLMs to Neuron</span></h3> <p data-svelte-h="svelte-fmy5pe">LLM models are not exported using Torch tracing, but converted directly to Neuron graphs into which the
	transformers checkpoint weights can be loaded.</p> <p data-svelte-h="svelte-1fq0lez">Just like the standard NLP models, you need to specify static parameters when exporting an LLM model:</p> <ul data-svelte-h="svelte-nxtyee"><li><code>batch_size</code> is the number of input sequences that the model will accept. Defaults to 1,</li> <li><code>sequence_length</code> is the maximum number of tokens in an input sequence. Defaults to <code>max_position_embeddings</code> (<code>n_positions</code> for older models).</li> <li><code>auto_cast_type</code> specifies the format to encode the weights. It can be one of <code>fp32</code> (<code>float32</code>), <code>fp16</code> (<code>float16</code>) or <code>bf16</code> (<code>bfloat16</code>). Defaults to <code>fp32</code>.</li> <li><code>num_cores</code> is the number of neuron cores used when instantiating the model. Each neuron core has 16 Gb of memory, which means that
	bigger models need to be split on multiple cores. Defaults to 1,</li></ul> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --model meta-llama/Meta-Llama-3-8B \
	--batch_size 1 \
	--sequence_length 4096 \
	--auto_cast_type fp16 `<span class="hljs-comment"># cast operations from BF16 to FP16` \</span>
	--num_cores 2 \
	llama3_neuron/<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1l8xwdn">An important restriction is that LLM models can only be exported on Neuron platforms, as they are tailored
	to fit on the actual devices during export.</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">The export of LLM models can take much longer than standard models (sometimes more than one hour).</div> <p data-svelte-h="svelte-1jrgmyt">As explained before, the neuron model parameters are static.
	This means in particular that during inference:</p> <ul data-svelte-h="svelte-1y4z1sb"><li>the <code>batch_size</code> of the inputs should be lower to the <code>batch_size</code> used during export,</li> <li>the <code>length</code> of the input sequences should be lower than the <code>sequence_length</code> used during export,</li> <li>the maximum number of tokens (input + generated) cannot exceed the <code>sequence_length</code> used during export.</li></ul> <p data-svelte-h="svelte-cslwz7">Once exported, neuron mmodels can simply be reloaded using the <code>NeuronModelForCausalLM</code> class.
	As with the original transformers models, use <code>generate()</code> instead of <code>forward()</code> to generate text sequences.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->from transformers import AutoTokenizer
	<span class="hljs-deletion">-from transformers import AutoModelForCausalLM</span>
	<span class="hljs-addition">+from optimum.neuron import NeuronModelForCausalLM</span>

	# Instantiate and convert to Neuron a PyTorch checkpoint
	<span class="hljs-deletion">-model = AutoModelForCausalLM.from_pretrained("gpt2")</span>
	<span class="hljs-addition">+model = NeuronModelForCausalLM.from_pretrained("./gpt2-neuron")</span>

	tokenizer = AutoTokenizer.from_pretrained("gpt2")
	tokenizer.pad_token_id = tokenizer.eos_token_id

	tokens = tokenizer("I really wish ", return_tensors="pt")
	with torch.inference_mode():
	sample_output = model.generate(
	**tokens,
	do_sample=True,
	min_length=128,
	max_length=256,
	temperature=0.7,
	)
	outputs = [tokenizer.decode(tok) for tok in sample_output]
	print(outputs)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-1266wn0">The generation is highly configurable. Please refer to <a href="https://huggingface.co/docs/transformers/generation_strategies" rel="nofollow">https://huggingface.co/docs/transformers/generation_strategies</a> for details.</p> <p data-svelte-h="svelte-1g2jw6w">Please be aware that:</p> <ul data-svelte-h="svelte-14ajc9a"><li>for each model architecture, default values are provided for all parameters, but values passed to the <code>generate</code> method will take precedence,</li> <li>the generation parameters can be stored in a <code>generation_config.json</code> file. When such a file is present in model directory,
	it will be parsed to set the default parameters (the values passed to the <code>generate</code> method still take precedence).</li></ul> <h2 class="relative group"><a id="exporting-a-model-to-neuron-programmatically-via-neuronmodel" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#exporting-a-model-to-neuron-programmatically-via-neuronmodel"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Exporting a model to Neuron programmatically via NeuronModel</span></h2> <p data-svelte-h="svelte-11m6qdr">As an alternative to the <code>optimim-cli</code>, you will also be able to export your models to Neuron
	inside your own python script or notebook with <code>optimum.neuron.NeuronModelForXXX</code> model classes.</p> <p data-svelte-h="svelte-1240csr">Here is an example:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronModelForSequenceClassification

	<span class="hljs-meta">>>> </span>input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"sequence_length"</span>: <span class="hljs-number">64</span>} <span class="hljs-comment"># mandatory shapes</span>
	<span class="hljs-meta">>>> </span>model = NeuronModelForSequenceClassification.from_pretrained(
	<span class="hljs-meta">... </span> <span class="hljs-string">"distilbert-base-uncased-finetuned-sst-2-english"</span>, export=<span class="hljs-literal">True</span>, **input_shapes
	<span class="hljs-meta">... </span>)

	<span class="hljs-comment"># Save the model</span>
	<span class="hljs-meta">>>> </span>model.save_pretrained(<span class="hljs-string">"./distilbert-base-uncased-finetuned-sst-2-english_neuron/"</span>)

	<span class="hljs-comment"># Push the neuron model to HF Hub</span>
	<span class="hljs-meta">>>> </span>model.push_to_hub(
	<span class="hljs-meta">... </span> <span class="hljs-string">"a_local_path_for_compiled_neuron_model"</span>, repository_id=<span class="hljs-string">"my-neuron-repo"</span>
	<span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-17uc6vc">This example can be adapted for other model types using the same export parameters as the <code>optimum-cli</code>.</p> <h2 class="relative group"><a id="exporting-neuron-models-using-neuronx-tgi" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#exporting-neuron-models-using-neuronx-tgi"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Exporting neuron models using NeuronX TGI</span></h2> <p data-svelte-h="svelte-1pkpov5">The NeuronX TGI image includes not only NeuronX runtime, but also all packages and tools required to export Neuron models.</p> <p data-svelte-h="svelte-19q682h">Use the following command to export a model to Neuron using a TGI image:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-title">docker</span> run <span class="hljs-comment">--entrypoint optimum-cli \</span>
	-v $(pwd)/<span class="hljs-class"><span class="hljs-keyword">data</span>:/<span class="hljs-keyword">data</span> \</span>
	<span class="hljs-comment">--privileged \</span>
	ghcr.io/huggingface/neuronx-tgi:latest \
	<span class="hljs-keyword">export</span> neuron \
	<span class="hljs-comment">--model <organization>/<model> \</span>
	<span class="hljs-comment">--batch_size 1 \</span>
	<span class="hljs-comment">--sequence_length 4096 \</span>
	<span class="hljs-comment">--auto_cast_type fp16 \</span>
	<span class="hljs-comment">--num_cores 2 \</span>
	/<span class="hljs-class"><span class="hljs-keyword">data</span>/<neuron_model_path></span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-ezhare">The exported model will be saved under <code>./data/<neuron_model_path></code>.</p> <p></p>

	<script>
	{
	__sveltekit_1eimgpm = {
	assets: "/docs/optimum.neuron/v0.2.0.dev2/en",
	base: "/docs/optimum.neuron/v0.2.0.dev2/en",
	env: {}
	};

	const element = document.currentScript.parentElement;

	const data = [null,null];

	Promise.all([
	import("/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/entry/start.c3692dcd.js"),
	import("/docs/optimum.neuron/v0.2.0.dev2/en/_app/immutable/entry/app.40ef12d9.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 8],
	data,
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 69.3 kB
Xet hash:: d40178d03da6620c42b93300b594d08f0d401efca9069d4ddcba5a69b9b00fc4

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.