Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Generate images with Stable Diffusion models on AWS Inferentia","local":"generate-images-with-stable-diffusion-models-on-aws-inferentia","sections":[{"title":"Stable Diffusion","local":"stable-diffusion","sections":[{"title":"Compile Stable Diffusion","local":"compile-stable-diffusion","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3},{"title":"Image-to-Image","local":"image-to-image","sections":[],"depth":3},{"title":"Inpaint","local":"inpaint","sections":[],"depth":3},{"title":"InstructPix2Pix","local":"instructpix2pix","sections":[],"depth":3}],"depth":2},{"title":"Stable Diffusion XL","local":"stable-diffusion-xl","sections":[{"title":"Compile Stable Diffusion XL","local":"compile-stable-diffusion-xl","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3},{"title":"Image-to-Image","local":"image-to-image","sections":[],"depth":3},{"title":"Inpaint","local":"inpaint","sections":[],"depth":3},{"title":"Refine Image Quality","local":"refine-image-quality","sections":[{"title":"Base + refiner model","local":"base--refiner-model","sections":[],"depth":4},{"title":"Base to refiner model","local":"base-to-refiner-model","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Latent Consistency Models","local":"latent-consistency-models","sections":[{"title":"Compile LCM","local":"compile-lcm","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3}],"depth":2},{"title":"Stable Diffusion XL Turbo","local":"stable-diffusion-xl-turbo","sections":[{"title":"Compile SDXL Turbo","local":"compile-sdxl-turbo","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3}],"depth":2},{"title":"Load adapters","local":"load-adapters","sections":[{"title":"LoRA","local":"lora","sections":[],"depth":3}],"depth":2},{"title":"ControlNet","local":"controlnet","sections":[{"title":"Compile ControlNet","local":"compile-controlnet","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3},{"title":"MultiControlNet","local":"multicontrolnet","sections":[],"depth":3}],"depth":2},{"title":"ControlNet with Stable Diffusion XL","local":"controlnet-with-stable-diffusion-xl","sections":[{"title":"Compile","local":"compile","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3}],"depth":2},{"title":"PixArt-α","local":"pixart-α","sections":[{"title":"Compile","local":"compile","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3}],"depth":2}],"depth":1}"> | |
| <link href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/assets/0.e3b0c442.css" rel="modulepreload"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/entry/start.13c1f5a3.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/chunks/scheduler.9039eef2.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/chunks/singletons.a8905f53.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/chunks/paths.f5262881.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/entry/app.a71d5dce.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/chunks/index.cdcc3d35.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/nodes/0.68fa8611.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/chunks/each.e59479a4.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/nodes/18.47e97340.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/chunks/Tip.6f74db41.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/chunks/CodeBlock.e3ac94d9.js"> | |
| <link rel="modulepreload" href="/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/chunks/Heading.96ce3702.js"><!-- HEAD_svelte-u9bgzb_START --><meta name="hf:doc:metadata" content="{"title":"Generate images with Stable Diffusion models on AWS Inferentia","local":"generate-images-with-stable-diffusion-models-on-aws-inferentia","sections":[{"title":"Stable Diffusion","local":"stable-diffusion","sections":[{"title":"Compile Stable Diffusion","local":"compile-stable-diffusion","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3},{"title":"Image-to-Image","local":"image-to-image","sections":[],"depth":3},{"title":"Inpaint","local":"inpaint","sections":[],"depth":3},{"title":"InstructPix2Pix","local":"instructpix2pix","sections":[],"depth":3}],"depth":2},{"title":"Stable Diffusion XL","local":"stable-diffusion-xl","sections":[{"title":"Compile Stable Diffusion XL","local":"compile-stable-diffusion-xl","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3},{"title":"Image-to-Image","local":"image-to-image","sections":[],"depth":3},{"title":"Inpaint","local":"inpaint","sections":[],"depth":3},{"title":"Refine Image Quality","local":"refine-image-quality","sections":[{"title":"Base + refiner model","local":"base--refiner-model","sections":[],"depth":4},{"title":"Base to refiner model","local":"base-to-refiner-model","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Latent Consistency Models","local":"latent-consistency-models","sections":[{"title":"Compile LCM","local":"compile-lcm","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3}],"depth":2},{"title":"Stable Diffusion XL Turbo","local":"stable-diffusion-xl-turbo","sections":[{"title":"Compile SDXL Turbo","local":"compile-sdxl-turbo","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3}],"depth":2},{"title":"Load adapters","local":"load-adapters","sections":[{"title":"LoRA","local":"lora","sections":[],"depth":3}],"depth":2},{"title":"ControlNet","local":"controlnet","sections":[{"title":"Compile ControlNet","local":"compile-controlnet","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3},{"title":"MultiControlNet","local":"multicontrolnet","sections":[],"depth":3}],"depth":2},{"title":"ControlNet with Stable Diffusion XL","local":"controlnet-with-stable-diffusion-xl","sections":[{"title":"Compile","local":"compile","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3}],"depth":2},{"title":"PixArt-α","local":"pixart-α","sections":[{"title":"Compile","local":"compile","sections":[],"depth":3},{"title":"Text-to-Image","local":"text-to-image","sections":[],"depth":3}],"depth":2}],"depth":1}"><!-- HEAD_svelte-u9bgzb_END --> <p></p> <h1 class="relative group"><a id="generate-images-with-stable-diffusion-models-on-aws-inferentia" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#generate-images-with-stable-diffusion-models-on-aws-inferentia"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Generate images with Stable Diffusion models on AWS Inferentia</span></h1> <h2 class="relative group"><a id="stable-diffusion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#stable-diffusion"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Stable Diffusion</span></h2> <p data-svelte-h="svelte-140c11v"><em>There is a notebook version of that tutorial <a href="https://github.com/huggingface/optimum-neuron/blob/main/notebooks/stable-diffusion/stable-diffusion-txt2img.ipynb" rel="nofollow">here</a></em>.</p> <p data-svelte-h="svelte-1g87d8z">🤗 <code>Optimum</code> extends <code>Diffusers</code> to support inference on the second generation of Neuron devices(powering Trainium and Inferentia 2). It aims at inheriting the ease of Diffusers on Neuron.</p> <p data-svelte-h="svelte-1u4t7n7">To get started, make sure you have <a href="../installation">configured your inf2 / trn1 instance</a>, and installed optimum:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->pip install optimum-neuron[neuronx] diffusers<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="compile-stable-diffusion" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compile-stable-diffusion"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Compile Stable Diffusion</span></h3> <p data-svelte-h="svelte-3qllio">To deploy models, you will need to compile them to TorchScript optimized for AWS Neuron. In the case of Stable Diffusion, there are four components which need to be exported to the <code>.neuron</code> format to boost the performance:</p> <ul data-svelte-h="svelte-1c1t3y0"><li>Text encoder</li> <li>U-Net</li> <li>VAE encoder</li> <li>VAE decoder</li></ul> <p data-svelte-h="svelte-15y00oa">You can either compile and export a Stable Diffusion Checkpoint via CLI or <code>NeuronStableDiffusionPipeline</code> class.</p> <p data-svelte-h="svelte-1a2m8iq"><strong><em>Export via CLI</em></strong></p> <p data-svelte-h="svelte-ktaj8q">Here is an example of exporting stable diffusion components with <code>Optimum</code> CLI:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --model stabilityai/stable-diffusion-2-1-base \ | |
| --batch_size 1 \ | |
| --height 512 `<span class="hljs-comment"># height in pixels of generated image, eg. 512, 768` \</span> | |
| --width 512 `<span class="hljs-comment"># width in pixels of generated image, eg. 512, 768` \</span> | |
| --num_images_per_prompt 1 `<span class="hljs-comment"># number of images to generate per prompt, defaults to 1` \</span> | |
| --auto_cast matmul `<span class="hljs-comment"># cast only matrix multiplication operations` \</span> | |
| --auto_cast_type bf16 `<span class="hljs-comment"># cast operations from FP32 to BF16` \</span> | |
| sd_neuron/<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-jz092x">We recommend using a <code>inf2.8xlarge</code> or a larger instance for the model compilation. You will also be able to compile the model with the Optimum CLI on a CPU-only instance (needs ~35 GB memory), and then run the pre-compiled model on <code>inf2.xlarge</code> to reduce the expenses. In this case, don’t forget to disable validation of inference by adding the <code>--disable-validation</code> argument.</p> <p data-svelte-h="svelte-kxhd34"><strong><em>Export via Python API</em></strong></p> <p data-svelte-h="svelte-1bahy14">Here is an example of exporting stable diffusion components with <code>NeuronStableDiffusionPipeline</code>:</p> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-6bhicj">To apply optimized compute of Unet’s attention score, please configure your environment variable with <code>export NEURON_FUSE_SOFTMAX=1</code>.</p> <p data-svelte-h="svelte-1dh7n4s">Besides, don’t hesitate to tweak the compilation configuration to find the best tradeoff between performance v.s accuracy in your use case. By default, we suggest casting FP32 matrix multiplication operations to BF16 which offers good performance with moderate sacrifice of the accuracy. Check out the guide from <a href="https://awsdocs-neuron.readthedocs-hosted.com/en/latest/general/appnotes/neuronx-cc/neuronx-cc-training-mixed-precision.html#neuronx-cc-training-mixed-precision" rel="nofollow">AWS Neuron documentation</a> to better understand the options for your compilation.</p></div> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionPipeline | |
| <span class="hljs-meta">>>> </span>model_id = <span class="hljs-string">"runwayml/stable-diffusion-v1-5"</span> | |
| <span class="hljs-meta">>>> </span>compiler_args = {<span class="hljs-string">"auto_cast"</span>: <span class="hljs-string">"matmul"</span>, <span class="hljs-string">"auto_cast_type"</span>: <span class="hljs-string">"bf16"</span>} | |
| <span class="hljs-meta">>>> </span>input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"height"</span>: <span class="hljs-number">512</span>, <span class="hljs-string">"width"</span>: <span class="hljs-number">512</span>} | |
| <span class="hljs-meta">>>> </span>stable_diffusion = NeuronStableDiffusionPipeline.from_pretrained(model_id, export=<span class="hljs-literal">True</span>, **compiler_args, **input_shapes) | |
| <span class="hljs-comment"># Save locally or upload to the HuggingFace Hub</span> | |
| <span class="hljs-meta">>>> </span>save_directory = <span class="hljs-string">"sd_neuron/"</span> | |
| <span class="hljs-meta">>>> </span>stable_diffusion.save_pretrained(save_directory) | |
| <span class="hljs-meta">>>> </span>stable_diffusion.push_to_hub( | |
| <span class="hljs-meta">... </span> save_directory, repository_id=<span class="hljs-string">"my-neuron-repo"</span>, use_auth_token=<span class="hljs-literal">True</span> | |
| <span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-to-Image</span></h3> <p data-svelte-h="svelte-1r61oa5"><code>NeuronStableDiffusionPipeline</code> class allows you to generate images from a text prompt on neuron devices similar to the experience with <code>Diffusers</code>.</p> <p data-svelte-h="svelte-1q8fgqb">With pre-compiled Stable Diffusion models, now generate an image with a prompt on Neuron:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionPipeline | |
| <span class="hljs-meta">>>> </span>stable_diffusion = NeuronStableDiffusionPipeline.from_pretrained(<span class="hljs-string">"sd_neuron/"</span>) | |
| <span class="hljs-meta">>>> </span>prompt = <span class="hljs-string">"a photo of an astronaut riding a horse on mars"</span> | |
| <span class="hljs-meta">>>> </span>image = stable_diffusion(prompt).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/models/01-sd-image.png" width="256" height="256" alt="stable diffusion generated image"> <h3 class="relative group"><a id="image-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#image-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Image-to-Image</span></h3> <p data-svelte-h="svelte-kqnotf">With the <code>NeuronStableDiffusionImg2ImgPipeline</code> class, you can generate a new image conditioned on a text prompt and an initial image.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> requests | |
| <span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image | |
| <span class="hljs-keyword">from</span> io <span class="hljs-keyword">import</span> BytesIO | |
| <span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionImg2ImgPipeline | |
| <span class="hljs-comment"># compile & save</span> | |
| model_id = <span class="hljs-string">"nitrosocke/Ghibli-Diffusion"</span> | |
| input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"height"</span>: <span class="hljs-number">512</span>, <span class="hljs-string">"width"</span>: <span class="hljs-number">512</span>} | |
| pipeline = NeuronStableDiffusionImg2ImgPipeline.from_pretrained(model_id, export=<span class="hljs-literal">True</span>, **input_shapes) | |
| pipeline.save_pretrained(<span class="hljs-string">"sd_img2img/"</span>) | |
| url = <span class="hljs-string">"https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg"</span> | |
| response = requests.get(url) | |
| init_image = Image.<span class="hljs-built_in">open</span>(BytesIO(response.content)).convert(<span class="hljs-string">"RGB"</span>) | |
| init_image = init_image.resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>)) | |
| prompt = <span class="hljs-string">"ghibli style, a fantasy landscape with snowcapped mountains, trees, lake with detailed reflection. sunlight and cloud in the sky, warm colors, 8K"</span> | |
| image = pipeline(prompt=prompt, image=init_image, strength=<span class="hljs-number">0.75</span>, guidance_scale=<span class="hljs-number">7.5</span>).images[<span class="hljs-number">0</span>] | |
| image.save(<span class="hljs-string">"fantasy_landscape.png"</span>)<!-- HTML_TAG_END --></pre></div> <table data-svelte-h="svelte-i6e2ex"><thead><tr><th align="center"><code>image</code></th> <th align="center"><code>prompt</code></th> <th align="center">output</th> <th align="right"></th></tr></thead> <tbody><tr><td align="center"><img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/03-sd-img2img-init.png" alt="landscape photo" width="256" height="256"></td> <td align="center"><strong><em>ghibli style, a fantasy landscape with snowcapped mountains, trees, lake with detailed reflection. warm colors, 8K</em></strong></td> <td align="center"><img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/04-sd-img2img.png" alt="drawing" width="250"></td> <td align="right"></td></tr></tbody></table> <h3 class="relative group"><a id="inpaint" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inpaint"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inpaint</span></h3> <p data-svelte-h="svelte-vhijyl">With the <code>NeuronStableDiffusionInpaintPipeline</code> class, you can edit specific parts of an image by providing a mask and a text prompt.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> requests | |
| <span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image | |
| <span class="hljs-keyword">from</span> io <span class="hljs-keyword">import</span> BytesIO | |
| <span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionInpaintPipeline | |
| model_id = <span class="hljs-string">"runwayml/stable-diffusion-inpainting"</span> | |
| input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"height"</span>: <span class="hljs-number">512</span>, <span class="hljs-string">"width"</span>: <span class="hljs-number">512</span>} | |
| pipeline = NeuronStableDiffusionInpaintPipeline.from_pretrained(model_id, export=<span class="hljs-literal">True</span>, **input_shapes) | |
| pipeline.save_pretrained(<span class="hljs-string">"sd_inpaint/"</span>) | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">download_image</span>(<span class="hljs-params">url</span>): | |
| response = requests.get(url) | |
| <span class="hljs-keyword">return</span> Image.<span class="hljs-built_in">open</span>(BytesIO(response.content)).convert(<span class="hljs-string">"RGB"</span>) | |
| img_url = <span class="hljs-string">"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png"</span> | |
| mask_url = <span class="hljs-string">"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png"</span> | |
| init_image = download_image(img_url).resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>)) | |
| mask_image = download_image(mask_url).resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>)) | |
| prompt = <span class="hljs-string">"Face of a yellow cat, high resolution, sitting on a park bench"</span> | |
| image = pipeline(prompt=prompt, image=init_image, mask_image=mask_image).images[<span class="hljs-number">0</span>] | |
| image.save(<span class="hljs-string">"cat_on_bench.png"</span>)<!-- HTML_TAG_END --></pre></div> <table data-svelte-h="svelte-v8z5s8"><thead><tr><th align="center"><code>image</code></th> <th align="center"><code>mask_image</code></th> <th align="center"><code>prompt</code></th> <th align="right">output</th></tr></thead> <tbody><tr><td align="center"><img src="https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" alt="drawing" width="250"></td> <td align="center"><img src="https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" alt="drawing" width="250"></td> <td align="center"><strong><em>Face of a yellow cat, high resolution, sitting on a park bench</em></strong></td> <td align="right"><img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/05-sd-inpaint.png" alt="drawing" width="250"></td></tr></tbody></table> <h3 class="relative group"><a id="instructpix2pix" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#instructpix2pix"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>InstructPix2Pix</span></h3> <p data-svelte-h="svelte-1836mxv">With the <code>NeuronStableDiffusionInstructPix2PixPipeline</code> class, you can apply instruction-based image editing using both text guidance and image guidance.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> requests | |
| <span class="hljs-keyword">import</span> PIL | |
| <span class="hljs-keyword">from</span> io <span class="hljs-keyword">import</span> BytesIO | |
| <span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionInstructPix2PixPipeline | |
| <span class="hljs-keyword">def</span> <span class="hljs-title function_">download_image</span>(<span class="hljs-params">url</span>): | |
| response = requests.get(url) | |
| <span class="hljs-keyword">return</span> PIL.Image.<span class="hljs-built_in">open</span>(BytesIO(response.content)).convert(<span class="hljs-string">"RGB"</span>) | |
| model_id = <span class="hljs-string">"timbrooks/instruct-pix2pix"</span> | |
| input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"height"</span>: <span class="hljs-number">512</span>, <span class="hljs-string">"width"</span>: <span class="hljs-number">512</span>} | |
| pipe = NeuronStableDiffusionInstructPix2PixPipeline.from_pretrained( | |
| model_id, export=<span class="hljs-literal">True</span>, dynamic_batch_size=<span class="hljs-literal">True</span>, **input_shapes, | |
| ) | |
| pipe.save_pretrained(<span class="hljs-string">"sd_ip2p/"</span>) | |
| img_url = <span class="hljs-string">"https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/mountain.png"</span> | |
| init_image = download_image(img_url).resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>)) | |
| prompt = <span class="hljs-string">"Add a beautiful sunset"</span> | |
| image = pipe(prompt=prompt, image=init_image).images[<span class="hljs-number">0</span>] | |
| image.save(<span class="hljs-string">"sunset_mountain.png"</span>)<!-- HTML_TAG_END --></pre></div> <table data-svelte-h="svelte-w6yzym"><thead><tr><th align="center"><code>image</code></th> <th align="center"><code>prompt</code></th> <th align="right">output</th></tr></thead> <tbody><tr><td align="center"><img src="https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/mountain.png" alt="drawing" width="250"></td> <td align="center"><strong><em>Add a beautiful sunset</em></strong></td> <td align="right"><img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/11-sd-ip2p.png" alt="drawing" width="250"></td></tr></tbody></table> <h2 class="relative group"><a id="stable-diffusion-xl" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#stable-diffusion-xl"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Stable Diffusion XL</span></h2> <p data-svelte-h="svelte-1q6nuso"><em>There is a notebook version of that tutorial <a href="https://github.com/huggingface/optimum-neuron/blob/main/notebooks/stable-diffusion/stable-diffusion-xl-txt2img.ipynb" rel="nofollow">here</a></em>.</p> <p data-svelte-h="svelte-7xctqf">Stable Diffusion XL (SDXL) is a latent diffusion model for text-to-image. Compared to the previous versions of Stable Diffusion models, it improves the quality of generated images with a times larger UNet.</p> <h3 class="relative group"><a id="compile-stable-diffusion-xl" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compile-stable-diffusion-xl"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Compile Stable Diffusion XL</span></h3> <p data-svelte-h="svelte-u4wwem">To deploy SDXL models, we will also start by compiling the models. We support the export of following components in the pipeline to boost the speed:</p> <ul data-svelte-h="svelte-17e9k9s"><li>Text encoder</li> <li>Second text encoder</li> <li>U-Net (a three times larger UNet than the one in Stable Diffusion pipeline)</li> <li>VAE encoder</li> <li>VAE decoder</li></ul> <p data-svelte-h="svelte-1a2m8iq"><strong><em>Export via CLI</em></strong></p> <p data-svelte-h="svelte-9niqj3">Here is an example of exporting SDXL components with <code>Optimum</code> CLI:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --model stabilityai/stable-diffusion-xl-base-1.0 \ | |
| --batch_size 1 \ | |
| --height 1024 `<span class="hljs-comment"># height in pixels of generated image, eg. 768, 1024` \</span> | |
| --width 1024 `<span class="hljs-comment"># width in pixels of generated image, eg. 768, 1024` \</span> | |
| --num_images_per_prompt 1 `<span class="hljs-comment"># number of images to generate per prompt, defaults to 1` \</span> | |
| --auto_cast matmul `<span class="hljs-comment"># cast only matrix multiplication operations` \</span> | |
| --auto_cast_type bf16 `<span class="hljs-comment"># cast operations from FP32 to BF16` \</span> | |
| sd_neuron_xl/<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-eias7g">We recommend using a <code>inf2.8xlarge</code> or a larger instance for the model compilation. You will also be able to compile the model with Optimum CLI on a CPU-only instance (needs ~92 GB memory), and then run the pre-compiled models on <code>inf2.xlarge</code> to reduce the expenses. In this case, don’t forget to disable validation of inference by adding the <code>--disable-validation</code> argument.</p> <p data-svelte-h="svelte-kxhd34"><strong><em>Export via Python API</em></strong></p> <p data-svelte-h="svelte-17tzr0c">Here is an example of exporting stable diffusion components with <code>NeuronStableDiffusionXLPipeline</code>:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLPipeline | |
| <span class="hljs-meta">>>> </span>model_id = <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span> | |
| <span class="hljs-meta">>>> </span>compiler_args = {<span class="hljs-string">"auto_cast"</span>: <span class="hljs-string">"matmul"</span>, <span class="hljs-string">"auto_cast_type"</span>: <span class="hljs-string">"bf16"</span>} | |
| <span class="hljs-meta">>>> </span>input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"height"</span>: <span class="hljs-number">1024</span>, <span class="hljs-string">"width"</span>: <span class="hljs-number">1024</span>} | |
| <span class="hljs-meta">>>> </span>stable_diffusion_xl = NeuronStableDiffusionXLPipeline.from_pretrained(model_id, export=<span class="hljs-literal">True</span>, **compiler_args, **input_shapes) | |
| <span class="hljs-comment"># Save locally or upload to the HuggingFace Hub</span> | |
| <span class="hljs-meta">>>> </span>save_directory = <span class="hljs-string">"sd_neuron_xl/"</span> | |
| <span class="hljs-meta">>>> </span>stable_diffusion_xl.save_pretrained(save_directory) | |
| <span class="hljs-meta">>>> </span>stable_diffusion_xl.push_to_hub( | |
| <span class="hljs-meta">... </span> save_directory, repository_id=<span class="hljs-string">"my-neuron-repo"</span>, use_auth_token=<span class="hljs-literal">True</span> | |
| <span class="hljs-meta">... </span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-to-Image</span></h3> <p data-svelte-h="svelte-8q3qyn">With pre-compiled SDXL models, now generate an image with a text prompt on Neuron:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLPipeline | |
| <span class="hljs-meta">>>> </span>stable_diffusion_xl = NeuronStableDiffusionXLPipeline.from_pretrained(<span class="hljs-string">"sd_neuron_xl/"</span>) | |
| <span class="hljs-meta">>>> </span>prompt = <span class="hljs-string">"Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"</span> | |
| <span class="hljs-meta">>>> </span>image = stable_diffusion_xl(prompt).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/models/02-sdxl-image.jpeg" width="256" height="256" alt="sdxl generated image"> <h3 class="relative group"><a id="image-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#image-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Image-to-Image</span></h3> <p data-svelte-h="svelte-luub4o">With <code>NeuronStableDiffusionXLImg2ImgPipeline</code>, you can pass an initial image, and a text prompt to condition generated images:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLImg2ImgPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| prompt = <span class="hljs-string">"a dog running, lake, moat"</span> | |
| url = <span class="hljs-string">"https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/sd_xl/castle_friedrich.png"</span> | |
| init_image = load_image(url).convert(<span class="hljs-string">"RGB"</span>) | |
| pipe = NeuronStableDiffusionXLImg2ImgPipeline.from_pretrained(<span class="hljs-string">"sd_neuron_xl/"</span>) | |
| image = pipe(prompt=prompt, image=init_image).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <table data-svelte-h="svelte-1s0d0pv"><thead><tr><th align="center"><code>image</code></th> <th align="center"><code>prompt</code></th> <th align="center">output</th> <th align="right"></th></tr></thead> <tbody><tr><td align="center"><img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/intel/openvino/sd_xl/castle_friedrich.png" alt="castle photo" width="256" height="256"></td> <td align="center"><strong><em>a dog running, lake, moat</em></strong></td> <td align="center"><img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/06-sdxl-img2img.png" alt="castle with dog" width="250"></td> <td align="right"></td></tr></tbody></table> <h3 class="relative group"><a id="inpaint" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inpaint"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Inpaint</span></h3> <p data-svelte-h="svelte-11fbw00">With <code>NeuronStableDiffusionXLInpaintPipeline</code>, pass the original image and a mask of what you want to replace in the original image. Then replace the masked area with content described in a prompt.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLInpaintPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| img_url = <span class="hljs-string">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png"</span> | |
| mask_url = ( | |
| <span class="hljs-string">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-inpaint-mask.png"</span> | |
| ) | |
| init_image = load_image(img_url).convert(<span class="hljs-string">"RGB"</span>) | |
| mask_image = load_image(mask_url).convert(<span class="hljs-string">"RGB"</span>) | |
| prompt = <span class="hljs-string">"A deep sea diver floating"</span> | |
| pipe = NeuronStableDiffusionXLInpaintPipeline.from_pretrained(<span class="hljs-string">"sd_neuron_xl/"</span>) | |
| image = pipe(prompt=prompt, image=init_image, mask_image=mask_image, strength=<span class="hljs-number">0.85</span>, guidance_scale=<span class="hljs-number">12.5</span>).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <table data-svelte-h="svelte-18741xe"><thead><tr><th align="center"><code>image</code></th> <th align="center"><code>mask_image</code></th> <th align="center"><code>prompt</code></th> <th align="right">output</th></tr></thead> <tbody><tr><td align="center"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-text2img.png" alt="drawing" width="250"></td> <td align="center"><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl-inpaint-mask.png" alt="drawing" width="250"></td> <td align="center"><strong><em>A deep sea diver floating</em></strong></td> <td align="right"><img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/07-sdxl-inpaint.png" alt="drawing" width="250"></td></tr></tbody></table> <h3 class="relative group"><a id="refine-image-quality" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#refine-image-quality"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Refine Image Quality</span></h3> <p data-svelte-h="svelte-1usq821">SDXL includes a <a href="https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0" rel="nofollow">refiner model</a> to denoise low-noise stage images generated from the base model. There are two ways to use the refiner:</p> <ol data-svelte-h="svelte-6p4eqi"><li>use the base and refiner model together to produce a refined image.</li> <li>use the base model to produce an image, and subsequently use the refiner model to add more details to the image.</li></ol> <h4 class="relative group"><a id="base--refiner-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#base--refiner-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Base + refiner model</span></h4> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLPipeline, NeuronStableDiffusionXLImg2ImgPipeline | |
| prompt = <span class="hljs-string">"A majestic lion jumping from a big stone at night"</span> | |
| base = NeuronStableDiffusionXLPipeline.from_pretrained(<span class="hljs-string">"sd_neuron_xl/"</span>) | |
| image = base( | |
| prompt=prompt, | |
| num_inference_steps=<span class="hljs-number">40</span>, | |
| denoising_end=<span class="hljs-number">0.8</span>, | |
| output_type=<span class="hljs-string">"latent"</span>, | |
| ).images[<span class="hljs-number">0</span>] | |
| <span class="hljs-keyword">del</span> base <span class="hljs-comment"># To avoid neuron device OOM</span> | |
| refiner = NeuronStableDiffusionXLImg2ImgPipeline.from_pretrained(<span class="hljs-string">"sd_neuron_xl_refiner/"</span>) | |
| image = refiner( | |
| prompt=prompt, | |
| num_inference_steps=<span class="hljs-number">40</span>, | |
| denoising_start=<span class="hljs-number">0.8</span>, | |
| image=image, | |
| ).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/08-sdxl-base-refine.png" width="256" height="256" alt="sdxl base + refiner"> <h4 class="relative group"><a id="base-to-refiner-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#base-to-refiner-model"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Base to refiner model</span></h4> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLPipeline, NeuronStableDiffusionXLImg2ImgPipeline | |
| prompt = <span class="hljs-string">"A majestic lion jumping from a big stone at night"</span> | |
| base = NeuronStableDiffusionXLPipeline.from_pretrained(<span class="hljs-string">"sd_neuron_xl/"</span>) | |
| image = base(prompt=prompt, output_type=<span class="hljs-string">"latent"</span>).images[<span class="hljs-number">0</span>] | |
| <span class="hljs-keyword">del</span> base <span class="hljs-comment"># To avoid neuron device OOM</span> | |
| refiner = NeuronStableDiffusionXLImg2ImgPipeline.from_pretrained(<span class="hljs-string">"sd_neuron_xl_refiner/"</span>) | |
| image = refiner(prompt=prompt, image=image[<span class="hljs-literal">None</span>, :]).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <table data-svelte-h="svelte-1s7g03c"><thead><tr><th align="center"><code>Base Image</code></th> <th align="right">Refined Image</th></tr></thead> <tbody><tr><td align="center"><img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/09-sdxl-base-full.png" alt="drawing" width="250"></td> <td align="right"><img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/010-sdxl-refiner-detailed.png" alt="drawing" width="250"></td></tr></tbody></table> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-1323ut2">To avoid Neuron device out of memory, it’s suggested to finish all base inference and release the device memory before running the refiner.</p></div> <h2 class="relative group"><a id="latent-consistency-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#latent-consistency-models"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Latent Consistency Models</span></h2> <p data-svelte-h="svelte-1ud6a8q">Latent Consistency Models (LCMs) were proposed in <a href="https://huggingface.co/papers/2310.04378" rel="nofollow">Latent Consistency Models: Synthesizing High-Resolution Images with Few-Step Inference by Simian Luo, Yiqin Tan, Longbo Huang, Jian Li, and Hang Zhao</a>. LCMs enable inference with fewer steps on any pre-trained LDMs, including Stable Diffusion and SDXL.</p> <p data-svelte-h="svelte-1ayizu6">In <code>optimum-neuron</code>, you can:</p> <ul data-svelte-h="svelte-r4tdj0"><li>Use the class <code>NeuronLatentConsistencyModelPipeline</code> to compile and run inference of LCMs distilled from Stable Diffusion (SD) models.</li> <li>And continue to use the class <code>NeuronStableDiffusionXLPipeline</code> for LCMs distilled from SDXL models.</li></ul> <p data-svelte-h="svelte-c4lhmw">Here are examples to compile the LCMs of Stable Diffusion ( <a href="https://huggingface.co/SimianLuo/LCM_Dreamshaper_v7" rel="nofollow">SimianLuo/LCM_Dreamshaper_v7</a> ) and Stable Diffusion XL( <a href="https://huggingface.co/latent-consistency/lcm-sdxl" rel="nofollow">latent-consistency/lcm-sdxl</a> ), and then run inference on AWS Inferentia 2 :</p> <h3 class="relative group"><a id="compile-lcm" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compile-lcm"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Compile LCM</span></h3> <p data-svelte-h="svelte-1s0vvjb"><strong><em>LCM of Stable Diffusion</em></strong></p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronLatentConsistencyModelPipeline | |
| model_id = <span class="hljs-string">"SimianLuo/LCM_Dreamshaper_v7"</span> | |
| num_images_per_prompt = <span class="hljs-number">1</span> | |
| input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"height"</span>: <span class="hljs-number">768</span>, <span class="hljs-string">"width"</span>: <span class="hljs-number">768</span>, <span class="hljs-string">"num_images_per_prompt"</span>: num_images_per_prompt} | |
| compiler_args = {<span class="hljs-string">"auto_cast"</span>: <span class="hljs-string">"matmul"</span>, <span class="hljs-string">"auto_cast_type"</span>: <span class="hljs-string">"bf16"</span>} | |
| stable_diffusion = NeuronLatentConsistencyModelPipeline.from_pretrained( | |
| model_id, export=<span class="hljs-literal">True</span>, **compiler_args, **input_shapes | |
| ) | |
| save_directory = <span class="hljs-string">"lcm_sd_neuron/"</span> | |
| stable_diffusion.save_pretrained(save_directory) | |
| <span class="hljs-comment"># Push to hub</span> | |
| stable_diffusion.push_to_hub(save_directory, repository_id=<span class="hljs-string">"my-neuron-repo"</span>, use_auth_token=<span class="hljs-literal">True</span>) <span class="hljs-comment"># Replace with your repo id, eg. "Jingya/LCM_Dreamshaper_v7_neuronx"</span><!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-51zp7x"><strong><em>LCM of Stable Diffusion XL</em></strong></p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLPipeline | |
| model_id = <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span> | |
| unet_id = <span class="hljs-string">"latent-consistency/lcm-sdxl"</span> | |
| num_images_per_prompt = <span class="hljs-number">1</span> | |
| input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"height"</span>: <span class="hljs-number">1024</span>, <span class="hljs-string">"width"</span>: <span class="hljs-number">1024</span>, <span class="hljs-string">"num_images_per_prompt"</span>: num_images_per_prompt} | |
| compiler_args = {<span class="hljs-string">"auto_cast"</span>: <span class="hljs-string">"matmul"</span>, <span class="hljs-string">"auto_cast_type"</span>: <span class="hljs-string">"bf16"</span>} | |
| stable_diffusion = NeuronStableDiffusionXLPipeline.from_pretrained( | |
| model_id, unet_id=unet_id, export=<span class="hljs-literal">True</span>, **compiler_args, **input_shapes | |
| ) | |
| save_directory = <span class="hljs-string">"lcm_sdxl_neuron/"</span> | |
| stable_diffusion.save_pretrained(save_directory) | |
| <span class="hljs-comment"># Push to hub</span> | |
| stable_diffusion.push_to_hub(save_directory, repository_id=<span class="hljs-string">"my-neuron-repo"</span>, use_auth_token=<span class="hljs-literal">True</span>) <span class="hljs-comment"># Replace with your repo id, eg. "Jingya/lcm-sdxl-neuronx"</span><!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-to-Image</span></h3> <p data-svelte-h="svelte-1ebkvay">Now we can generate images from text prompts on Inf2 using the pre-compiled model:</p> <p data-svelte-h="svelte-1s0vvjb"><strong><em>LCM of Stable Diffusion</em></strong></p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronLatentConsistencyModelPipeline | |
| pipe = NeuronLatentConsistencyModelPipeline.from_pretrained(<span class="hljs-string">"Jingya/LCM_Dreamshaper_v7_neuronx"</span>) | |
| prompts = [<span class="hljs-string">"Self-portrait oil painting, a beautiful cyborg with golden hair, 8k"</span>] * <span class="hljs-number">2</span> | |
| images = pipe(prompt=prompts, num_inference_steps=<span class="hljs-number">4</span>, guidance_scale=<span class="hljs-number">8.0</span>).images<!-- HTML_TAG_END --></pre></div> <p data-svelte-h="svelte-51zp7x"><strong><em>LCM of Stable Diffusion XL</em></strong></p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLPipeline | |
| pipe = NeuronStableDiffusionXLPipeline.from_pretrained(<span class="hljs-string">"Jingya/lcm-sdxl-neuronx"</span>) | |
| prompts = [<span class="hljs-string">"a close-up picture of an old man standing in the rain"</span>] * <span class="hljs-number">2</span> | |
| images = pipe(prompt=prompts, num_inference_steps=<span class="hljs-number">4</span>, guidance_scale=<span class="hljs-number">8.0</span>).images<!-- HTML_TAG_END --></pre></div> <h2 class="relative group"><a id="stable-diffusion-xl-turbo" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#stable-diffusion-xl-turbo"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Stable Diffusion XL Turbo</span></h2> <p data-svelte-h="svelte-10h80ju">SDXL Turbo is an adversarial time-distilled Stable Diffusion XL (SDXL) model capable of running inference in as little as 1 step (<a href="https://huggingface.co/docs/diffusers/using-diffusers/sdxl_turbo" rel="nofollow">check <code>🤗diffusers</code> for more details</a>).</p> <p data-svelte-h="svelte-1ayizu6">In <code>optimum-neuron</code>, you can:</p> <ul data-svelte-h="svelte-t7g18i"><li>Use the class <code>NeuronStableDiffusionXLPipeline</code> to compile and run inference.</li></ul> <p data-svelte-h="svelte-ohwj0a">Here we will compile the <a href="https://huggingface.co/stabilityai/sdxl-turbo" rel="nofollow"><code>stabilityai/sdxl-turbo</code></a> model with Optimum CLI.</p> <h3 class="relative group"><a id="compile-sdxl-turbo" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compile-sdxl-turbo"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Compile SDXL Turbo</span></h3> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --model stabilityai/sdxl-turbo --batch_size 1 --height 512 --width 512 --auto_cast matmul --auto_cast_type bf16 sdxl_turbo_neuron/<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-to-Image</span></h3> <p data-svelte-h="svelte-1ebkvay">Now we can generate images from text prompts on Inf2 using the pre-compiled model:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLPipeline | |
| pipe = NeuronStableDiffusionXLPipeline.from_pretrained(<span class="hljs-string">"sdxl_turbo_neuron/"</span>, data_parallel_mode=<span class="hljs-string">"all"</span>) | |
| prompt = [<span class="hljs-string">"Self-portrait oil painting, a beautiful cyborg with golden hair, 8k"</span>] * <span class="hljs-number">2</span> | |
| images = pipe(prompt=prompt, guidance_scale=<span class="hljs-number">0.0</span>, num_inference_steps=<span class="hljs-number">1</span>).images<!-- HTML_TAG_END --></pre></div> <div class="course-tip bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400"><p data-svelte-h="svelte-3zwean">Inf2 instances contain one or more Neuron devices, and each Neuron device includes 2 NeuronCore-v2. By default, we load the whole pipeline of LCM to both Neuron cores. It means that when the batch size is divisible by 2, you can fully leverage the compute power of both cores.</p></div> <h2 class="relative group"><a id="load-adapters" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#load-adapters"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Load adapters</span></h2> <h3 class="relative group"><a id="lora" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#lora"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>LoRA</span></h3> <p data-svelte-h="svelte-k1lsm0">Low-Rank Adaptation is fast way to Stable Diffusion to adapt styles of generated images. In Optimum Neuron, we support using one or multiple LoRA adapters by fusing their parameters into the original parameters of the text encoder(s) and the unet during the compilation. Here below is an example of compiling stable diffusion models with LoRA adapters of your choice and using the compiled artifacts to generate styled images:</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --> | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> LCMScheduler | |
| <span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionPipeline | |
| model_id = <span class="hljs-string">"Lykon/dreamshaper-7"</span> | |
| adapter_id = <span class="hljs-string">"latent-consistency/lcm-lora-sdv1-5"</span> | |
| input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"height"</span>: <span class="hljs-number">512</span>, <span class="hljs-string">"width"</span>: <span class="hljs-number">512</span>, <span class="hljs-string">"num_images_per_prompt"</span>: <span class="hljs-number">1</span>} | |
| compiler_args = {<span class="hljs-string">"auto_cast"</span>: <span class="hljs-string">"matmul"</span>, <span class="hljs-string">"auto_cast_type"</span>: <span class="hljs-string">"bf16"</span>} | |
| <span class="hljs-comment"># Compile</span> | |
| pipe = NeuronStableDiffusionPipeline.from_pretrained( | |
| model_id, | |
| export=<span class="hljs-literal">True</span>, | |
| inline_weights_to_neff=<span class="hljs-literal">True</span>, <span class="hljs-comment"># caveat: performance drop if neff/weights separated, will be improved by a future Neuron sdk release.</span> | |
| lora_model_ids=adapter_id, | |
| lora_weight_names=<span class="hljs-string">"pytorch_lora_weights.safetensors"</span>, | |
| lora_adapter_names=<span class="hljs-string">"lcm"</span>, | |
| **input_shapes, | |
| **compiler_args, | |
| ) | |
| pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) | |
| <span class="hljs-comment"># Save locally or upload to the HuggingFace Hub</span> | |
| pipe.save_pretrained(<span class="hljs-string">"dreamshaper_7_lcm_lora_neuron/"</span>) | |
| <span class="hljs-comment"># Inference</span> | |
| prompt = <span class="hljs-string">"Self-portrait oil painting, a beautiful cyborg with golden hair, 8k"</span> | |
| image = pipe(prompt, num_inference_steps=<span class="hljs-number">4</span>, guidance_scale=<span class="hljs-number">0</span>).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <img src="https://raw.githubusercontent.com/huggingface/optimum-neuron/main/docs/assets/guides/models/03-sd-lora.png" width="256" height="256" alt="stable diffusion generated image with LoRA adapter."> <h2 class="relative group"><a id="controlnet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#controlnet"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>ControlNet</span></h2> <p data-svelte-h="svelte-15r2htw">ControlNet conditions the stable diffusion model with an additional input image. In Optimum Neuron, we support the compilation of one or multiple ControlNet(s) along with the stable diffusion checkpoint. The you can use the compiled artifacts to generate styled images.</p> <h3 class="relative group"><a id="compile-controlnet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compile-controlnet"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Compile ControlNet</span></h3> <p data-svelte-h="svelte-clsbw5">We can either compile one or multiple ControlNet via the Optimum CLI or programatically via the <code>NeuronStableDiffusionControlNetPipeline</code> class by passing the <code>controlnet_ids</code>.</p> <ul data-svelte-h="svelte-127fndl"><li>Export via the Optimum CLI</li></ul> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron -m runwayml/stable-diffusion-v1-5 --batch_size 1 --height 512 --width 512 --controlnet_ids lllyasviel/sd-controlnet-canny --num_images_per_prompt 1 sd_neuron_controlnet/<!-- HTML_TAG_END --></pre></div> <ul data-svelte-h="svelte-zxjvv9"><li>Export via Python API</li></ul> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionControlNetPipeline | |
| model_id = <span class="hljs-string">"runwayml/stable-diffusion-v1-5"</span> | |
| controlnet_id = <span class="hljs-string">"lllyasviel/sd-controlnet-canny"</span> | |
| <span class="hljs-comment"># [Neuron] pipeline</span> | |
| input_shapes = {<span class="hljs-string">"batch_size"</span>: <span class="hljs-number">1</span>, <span class="hljs-string">"height"</span>: <span class="hljs-number">512</span>, <span class="hljs-string">"width"</span>: <span class="hljs-number">512</span>, <span class="hljs-string">"num_images_per_prompt"</span>: <span class="hljs-number">1</span>} | |
| compiler_args = {<span class="hljs-string">"auto_cast"</span>: <span class="hljs-string">"matmul"</span>, <span class="hljs-string">"auto_cast_type"</span>: <span class="hljs-string">"bf16"</span>} | |
| pipe = NeuronStableDiffusionControlNetPipeline.from_pretrained( | |
| model_id, | |
| controlnet_ids=controlnet_id, | |
| export=<span class="hljs-literal">True</span>, | |
| **input_shapes, | |
| **compiler_args, | |
| ) | |
| pipe.save_pretrained(<span class="hljs-string">"sd_neuron_controlnet"</span>)<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-to-Image</span></h3> <p data-svelte-h="svelte-s2jfwm">For text-to-image, we can specify an additional conditioning input.</p> <p data-svelte-h="svelte-e5xot3">Here is an example with a canny image, a white outline of an image on a black background. The ControlNet will use the canny image as a control to guide the model to generate an image with the same outline.</p> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> cv2 | |
| <span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image, make_image_grid | |
| <span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image | |
| <span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionControlNetPipeline | |
| <span class="hljs-comment"># prepare canny image</span> | |
| original_image = load_image( | |
| <span class="hljs-string">"https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png"</span> | |
| ) | |
| image = np.array(original_image) | |
| low_threshold = <span class="hljs-number">100</span> | |
| high_threshold = <span class="hljs-number">200</span> | |
| image = cv2.Canny(image, low_threshold, high_threshold) | |
| image = image[:, :, <span class="hljs-literal">None</span>] | |
| image = np.concatenate([image, image, image], axis=<span class="hljs-number">2</span>) | |
| canny_image = Image.fromarray(image) | |
| <span class="hljs-comment"># load pre-compiled neuron model</span> | |
| pipe = NeuronStableDiffusionControlNetPipeline.from_pretrained(<span class="hljs-string">"sd_neuron_controlnet"</span>) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) | |
| <span class="hljs-comment"># inference</span> | |
| output = pipe(<span class="hljs-string">"the mona lisa"</span>, image=canny_image).images[<span class="hljs-number">0</span>] | |
| compare = make_image_grid([original_image, canny_image, output], rows=<span class="hljs-number">1</span>, cols=<span class="hljs-number">3</span>) | |
| compare.save(<span class="hljs-string">"compare.png"</span>)<!-- HTML_TAG_END --></pre></div> <img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/10-sd-text2img-controlnet.png?download=true" width="768" height="256" alt="stable diffusion 1.5 generated image with controlnet."> <h3 class="relative group"><a id="multicontrolnet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#multicontrolnet"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>MultiControlNet</span></h3> <p data-svelte-h="svelte-ve8s8f">With Optimum Neuron, you can also compose multiple ControlNet conditionings from different image inputs:</p> <ul data-svelte-h="svelte-16kkb80"><li>Compile multiple ControlNet for SD1.5</li></ul> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --inline-weights-neff --model jyoung105/stable-diffusion-v1-5 --task stable-diffusion --auto_cast matmul --auto_cast_type bf16 --batch_size 1 --num_images_per_prompt 1 --controlnet_ids lllyasviel/control_v11p_sd15_openpose lllyasviel/control_v11f1p_sd15_depth --height 512 --width 512 sd15-512x512-bf16-openpose-depth<!-- HTML_TAG_END --></pre></div> <ul data-svelte-h="svelte-1pbmybe"><li>Run SD1.5 with OpenPose and Depth conditionings:</li></ul> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np | |
| <span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image | |
| <span class="hljs-keyword">from</span> controlnet_aux <span class="hljs-keyword">import</span> OpenposeDetector | |
| <span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> UniPCMultistepScheduler | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| <span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionControlNetPipeline | |
| <span class="hljs-comment"># OpenPose+Depth ControlNet</span> | |
| model_id = <span class="hljs-string">"sd15-512x512-bf16-openpose-depth"</span> | |
| <span class="hljs-comment"># Load ControlNet images</span> | |
| <span class="hljs-comment"># 1. openpose</span> | |
| image = load_image(<span class="hljs-string">"https://huggingface.co/lllyasviel/control_v11p_sd15_openpose/resolve/main/images/input.png"</span>) | |
| processor = OpenposeDetector.from_pretrained(<span class="hljs-string">'lllyasviel/ControlNet'</span>) | |
| openpose_image = processor(image) | |
| <span class="hljs-comment"># 2. depth</span> | |
| image = load_image(<span class="hljs-string">"https://huggingface.co/lllyasviel/control_v11p_sd15_depth/resolve/main/images/input.png"</span>) | |
| depth_estimator = pipeline(<span class="hljs-string">'depth-estimation'</span>) | |
| image = depth_estimator(image)[<span class="hljs-string">'depth'</span>] | |
| image = np.array(image) | |
| image = image[:, :, <span class="hljs-literal">None</span>] | |
| image = np.concatenate([image, image, image], axis=<span class="hljs-number">2</span>) | |
| depth_image = Image.fromarray(image) | |
| images = [openpose_image.resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>)), depth_image.resize((<span class="hljs-number">512</span>, <span class="hljs-number">512</span>))] | |
| <span class="hljs-comment"># 3. inference</span> | |
| pipe = NeuronStableDiffusionControlNetPipeline.from_pretrained(model_id) | |
| pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) | |
| prompt = <span class="hljs-string">"a giant in a fantasy landscape, best quality"</span> | |
| negative_prompt = <span class="hljs-string">"monochrome, lowres, bad anatomy, worst quality, low quality"</span> | |
| image = pipe(prompt=prompt, image=images).images[<span class="hljs-number">0</span>] | |
| image.save(<span class="hljs-string">'out.png'</span>)<!-- HTML_TAG_END --></pre></div> <img src="https://huggingface.co/datasets/Jingya/document_images/resolve/main/optimum/neuron/multicontrolnet.png" width="768" height="256" alt="stable diffusion 1.5 generated image with OpenPose and Depth controlnet."> <h2 class="relative group"><a id="controlnet-with-stable-diffusion-xl" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#controlnet-with-stable-diffusion-xl"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>ControlNet with Stable Diffusion XL</span></h2> <h3 class="relative group"><a id="compile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compile"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Compile</span></h3> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron -m stabilityai/stable-diffusion-xl-base-1.0 --task stable-diffusion-xl --batch_size 1 --height 1024 --width 1024 --controlnet_ids diffusers/controlnet-canny-sdxl-1.0-small --num_images_per_prompt 1 sdxl_neuron_controlnet/<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-to-Image</span></h3> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">import</span> cv2 | |
| <span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| <span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image | |
| <span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronStableDiffusionXLControlNetPipeline | |
| <span class="hljs-comment"># Inputs</span> | |
| prompt = <span class="hljs-string">"aerial view, a futuristic research complex in a bright foggy jungle, hard lighting"</span> | |
| negative_prompt = <span class="hljs-string">"low quality, bad quality, sketches"</span> | |
| image = load_image( | |
| <span class="hljs-string">"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png"</span> | |
| ) | |
| image = np.array(image) | |
| image = cv2.Canny(image, <span class="hljs-number">100</span>, <span class="hljs-number">200</span>) | |
| image = image[:, :, <span class="hljs-literal">None</span>] | |
| image = np.concatenate([image, image, image], axis=<span class="hljs-number">2</span>) | |
| image = Image.fromarray(image) | |
| controlnet_conditioning_scale = <span class="hljs-number">0.5</span> <span class="hljs-comment"># recommended for good generalization</span> | |
| pipe = NeuronStableDiffusionXLControlNetPipeline.from_pretrained(<span class="hljs-string">"sdxl_neuron_controlnet"</span>) | |
| images = pipe( | |
| prompt, | |
| negative_prompt=negative_prompt, | |
| image=image, | |
| controlnet_conditioning_scale=controlnet_conditioning_scale, | |
| ).images | |
| images[<span class="hljs-number">0</span>].save(<span class="hljs-string">"hug_lab.png"</span>)<!-- HTML_TAG_END --></pre></div> <img src="https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/12-sdxl-text2img-controlnet.png?download=true" width="768" height="256" alt="stable diffusion xl generated image with controlnet."> <h2 class="relative group"><a id="pixart-α" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#pixart-α"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>PixArt-α</span></h2> <h3 class="relative group"><a id="compile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compile"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Compile</span></h3> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START -->optimum-cli <span class="hljs-built_in">export</span> neuron --model PixArt-alpha/PixArt-XL-2-512x512 --batch_size 1 --height 512 --width 512 --num_images_per_prompt 1 --torch_dtype bfloat16 --sequence_length 120 pixart_alpha_neuron_512/<!-- HTML_TAG_END --></pre></div> <h3 class="relative group"><a id="text-to-image" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#text-to-image"><span><svg class="" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg></span></a> <span>Text-to-Image</span></h3> <div class="code-block relative"><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg class="" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg> <div class="absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0"><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent; "></div> Copied</div></button></div> <pre class=""><!-- HTML_TAG_START --><span class="hljs-keyword">from</span> optimum.neuron <span class="hljs-keyword">import</span> NeuronPixArtAlphaPipeline | |
| neuron_model = NeuronPixArtAlphaPipeline.from_pretrained(<span class="hljs-string">"pixart_alpha_neuron_512/"</span>) | |
| prompt = <span class="hljs-string">"Oppenheimer sits on the beach on a chair, watching a nuclear exposition with a huge mushroom cloud, 120mm."</span> | |
| image = neuron_model(prompt=prompt).images[<span class="hljs-number">0</span>]<!-- HTML_TAG_END --></pre></div> <img src="https://huggingface.co/datasets/Jingya/document_images/resolve/main/optimum/neuron/pixart-alpha-oppenheimer.png" width="256" height="256" alt="PixArt-α generated image."> <p data-svelte-h="svelte-xbi604">Are there any other stable diffusion features that you want us to support in 🤗<code>Optimum-neuron</code>? Please file an issue to <a href="https://github.com/huggingface/optimum-neuron" rel="nofollow"><code>Optimum-neuron</code> Github repo</a> or discuss with us on <a href="https://discuss.huggingface.co/c/optimum/" rel="nofollow">HuggingFace’s community forum</a>, cheers 🤗 !</p> <p></p> | |
| <script> | |
| { | |
| __sveltekit_1shq1jv = { | |
| assets: "/docs/optimum.neuron/v0.0.28.dev2/en", | |
| base: "/docs/optimum.neuron/v0.0.28.dev2/en", | |
| env: {} | |
| }; | |
| const element = document.currentScript.parentElement; | |
| const data = [null,null]; | |
| Promise.all([ | |
| import("/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/entry/start.13c1f5a3.js"), | |
| import("/docs/optimum.neuron/v0.0.28.dev2/en/_app/immutable/entry/app.a71d5dce.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 18], | |
| data, | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 135 kB
- Xet hash:
- dd3b0a1d02975f7c2a16c7c702881004ef3e1ec48bfdd2d660fc4a673588baa0
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.