Buckets:
| <meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"Reduce memory usage","local":"reduce-memory-usage","sections":[{"title":"Multiple GPUs","local":"multiple-gpus","sections":[{"title":"Sharded checkpoints","local":"sharded-checkpoints","sections":[],"depth":3},{"title":"Device placement","local":"device-placement","sections":[],"depth":3}],"depth":2},{"title":"VAE slicing","local":"vae-slicing","sections":[],"depth":2},{"title":"VAE tiling","local":"vae-tiling","sections":[],"depth":2},{"title":"Offloading","local":"offloading","sections":[{"title":"Sequential CPU offloading","local":"sequential-cpu-offloading","sections":[],"depth":3},{"title":"Model offloading","local":"model-offloading","sections":[],"depth":3},{"title":"Group offloading","local":"group-offloading","sections":[{"title":"CUDA stream","local":"cuda-stream","sections":[],"depth":4},{"title":"Offloading to disk","local":"offloading-to-disk","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Layerwise casting","local":"layerwise-casting","sections":[],"depth":2},{"title":"torch.channels_last","local":"torchchannelslast","sections":[],"depth":2},{"title":"Memory-efficient attention","local":"memory-efficient-attention","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}"/> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/entry/start.DH9xfiDx.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/C6UGX4Zx.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/jDjavuwI.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/entry/app.Bgs6fdnu.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/CMF6C6Mt.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/DKNTNR4p.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/i0LL_sTA.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/nodes/0.CAKb-ftv.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/9Uo8d8lw.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/nodes/282.B_zfvEYk.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/BLbO27j4.js" rel="modulepreload"> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/BYVaKgop.js" rel="modulepreload"> | |
| <!--13d35im--><meta name="hf:doc:metadata" content="{"title":"Reduce memory usage","local":"reduce-memory-usage","sections":[{"title":"Multiple GPUs","local":"multiple-gpus","sections":[{"title":"Sharded checkpoints","local":"sharded-checkpoints","sections":[],"depth":3},{"title":"Device placement","local":"device-placement","sections":[],"depth":3}],"depth":2},{"title":"VAE slicing","local":"vae-slicing","sections":[],"depth":2},{"title":"VAE tiling","local":"vae-tiling","sections":[],"depth":2},{"title":"Offloading","local":"offloading","sections":[{"title":"Sequential CPU offloading","local":"sequential-cpu-offloading","sections":[],"depth":3},{"title":"Model offloading","local":"model-offloading","sections":[],"depth":3},{"title":"Group offloading","local":"group-offloading","sections":[{"title":"CUDA stream","local":"cuda-stream","sections":[],"depth":4},{"title":"Offloading to disk","local":"offloading-to-disk","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Layerwise casting","local":"layerwise-casting","sections":[],"depth":2},{"title":"torch.channels_last","local":"torchchannelslast","sections":[],"depth":2},{"title":"Memory-efficient attention","local":"memory-efficient-attention","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}"/><!----> | |
| <link href="/docs/diffusers/pr_14909/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="reduce-memory-usage" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#reduce-memory-usage"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Reduce memory usage</span></h1><!--]--><!----> <p>Modern diffusion models have billions of parameters, which often exceeds the memory available on a consumer GPU. Diffusers provides several techniques to reduce memory usage, such as distributing a model across multiple GPUs, offloading components to the CPU, and storing weights in lower precision.</p> <p>Choose a technique based on your hardware and workload.</p> <table><thead><tr><th>Situation</th><th>Technique</th></tr></thead><tbody><tr><td>You have more than one GPU</td><td><a href="#multiple-gpus">Multiple GPUs</a></td></tr><tr><td>You generate several images per prompt</td><td><a href="#vae-slicing">VAE slicing</a></td></tr><tr><td>You generate high-resolution images</td><td><a href="#vae-tiling">VAE tiling</a></td></tr><tr><td>The model doesn’t fit on a single GPU</td><td><a href="#offloading">Offloading</a></td></tr><tr><td>You want to store weights in lower precision</td><td><a href="#layerwise-casting">Layerwise casting</a></td></tr></tbody></table> <blockquote class="tip"><p>Results vary by model. For example, a transformer-based model may not benefit from some of these techniques as much as a UNet-based model.</p></blockquote> <!--[1--><h2 class="relative group"><a id="multiple-gpus" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#multiple-gpus"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Multiple GPUs</span></h2><!--]--><!----> <p>If you have access to more than one GPU, there are a few options for efficiently loading and distributing a large model across your hardware. These features are supported by the <a href="https://huggingface.co/docs/accelerate/index" rel="nofollow">Accelerate</a> library, so make sure it is installed.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->pip install -U accelerate<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="sharded-checkpoints" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#sharded-checkpoints"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Sharded checkpoints</span></h3><!--]--><!----> <p>A sharded checkpoint splits a large checkpoint into several smaller files that are loaded one at a time. Peak memory only needs to fit the model and the largest shard, instead of the model and the entire checkpoint. Sharding is recommended when the fp32 checkpoint is larger than 5GB. The default shard size is 10GB.</p> <p>Shard a checkpoint with the <code>max_shard_size</code> parameter in <a href="/docs/diffusers/pr_14909/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel | |
| unet = AutoModel.from_pretrained( | |
| <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, subfolder=<span class="hljs-string">"unet"</span> | |
| ) | |
| unet.save_pretrained(<span class="hljs-string">"sdxl-unet-sharded"</span>, max_shard_size=<span class="hljs-string">"5GB"</span>)<!----></pre></div><!----> <p>Load the sharded checkpoint in place of the original to reduce peak memory while loading.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, StableDiffusionXLPipeline | |
| unet = AutoModel.from_pretrained( | |
| <span class="hljs-string">"username/sdxl-unet-sharded"</span>, dtype=torch.float16 | |
| ) | |
| pipeline = StableDiffusionXLPipeline.from_pretrained( | |
| <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| unet=unet, | |
| dtype=torch.float16 | |
| ).to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span><!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="device-placement" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#device-placement"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Device placement</span></h3><!--]--><!----> <p>The <code>device_map</code> parameter controls how the model components in a pipeline or the layers in an individual model are distributed across devices.</p> <blockquote class="warning"><p>Device placement is an experimental feature and the API may change. At the pipeline level, <code>device_map</code> accepts <code>"balanced"</code>, or a single device such as <code>"cuda"</code> or <code>"cpu"</code>.</p></blockquote> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">pipeline level</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">model level</div><!--]--></div> <div class="language-select"><!--[0--><p>The <code>balanced</code> device placement strategy evenly splits the pipeline across all available devices.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLPipeline | |
| pipeline = StableDiffusionXLPipeline.from_pretrained( | |
| <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| dtype=torch.float16, | |
| device_map=<span class="hljs-string">"balanced"</span> | |
| )<!----></pre></div><!----> <p>You can inspect a pipeline’s device map with <code>hf_device_map</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-built_in">print</span>(pipeline.hf_device_map)<!----></pre></div><!----><!----><!--]--><!----> <!--[-1--><!--]--><!----><!----></div><!----> <p>A custom <code>device_map</code> is a dictionary that maps module names to devices, where a device is an integer for a GPU, <code>"cpu"</code>, or <code>"disk"</code>. Print a model’s <code>hf_device_map</code> to see how its layers are distributed, and use it as a starting point to design your own.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-built_in">print</span>(transformer.hf_device_map) | |
| {<span class="hljs-string">'pos_embed'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'time_text_embed'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'context_embedder'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'x_embedder'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'transformer_blocks'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.0'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.1'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.2'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.3'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.4'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.5'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.6'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.7'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.8'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.9'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.10'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.11'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.12'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.13'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.14'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.15'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.16'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.17'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.18'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.19'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.20'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.21'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.22'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.23'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.24'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.25'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.26'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.27'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.28'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.29'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.30'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.31'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.32'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.33'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.34'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.35'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.36'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.37'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'norm_out'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'proj_out'</span>: <span class="hljs-string">'cpu'</span>}<!----></pre></div><!----> <p>For example, the <code>device_map</code> below places <code>single_transformer_blocks.10</code> through <code>single_transformer_blocks.20</code> on a second GPU (<code>1</code>).</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel | |
| device_map = { | |
| <span class="hljs-string">'pos_embed'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'time_text_embed'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'context_embedder'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'x_embedder'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'transformer_blocks'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.0'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.1'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.2'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.3'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.4'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.5'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.6'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.7'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.8'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.9'</span>: <span class="hljs-number">0</span>, <span class="hljs-string">'single_transformer_blocks.10'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.11'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.12'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.13'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.14'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.15'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.16'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.17'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.18'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.19'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.20'</span>: <span class="hljs-number">1</span>, <span class="hljs-string">'single_transformer_blocks.21'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.22'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.23'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.24'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.25'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.26'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.27'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.28'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.29'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.30'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.31'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.32'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.33'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.34'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.35'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.36'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'single_transformer_blocks.37'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'norm_out'</span>: <span class="hljs-string">'cpu'</span>, <span class="hljs-string">'proj_out'</span>: <span class="hljs-string">'cpu'</span> | |
| } | |
| transformer = AutoModel.from_pretrained( | |
| <span class="hljs-string">"black-forest-labs/FLUX.1-dev"</span>, | |
| subfolder=<span class="hljs-string">"transformer"</span>, | |
| device_map=device_map, | |
| dtype=torch.bfloat16 | |
| )<!----></pre></div><!----> <p>To cap how much memory each GPU uses, pass <code>max_memory</code>, a dictionary that maps each device to a limit. Components aren’t placed on GPUs you leave out, and a component that doesn’t fit under a GPU’s limit is placed on the CPU instead.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLPipeline | |
| max_memory = {<span class="hljs-number">0</span>: <span class="hljs-string">"16GB"</span>, <span class="hljs-number">1</span>: <span class="hljs-string">"16GB"</span>} | |
| pipeline = StableDiffusionXLPipeline.from_pretrained( | |
| <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| dtype=torch.float16, | |
| device_map=<span class="hljs-string">"balanced"</span>, | |
| max_memory=max_memory | |
| )<!----></pre></div><!----> <p>By default, Diffusers uses all available memory on each GPU. Components that don’t fit on a GPU are placed on the CPU. If most of the pipeline ends up on the CPU, try a single GPU with one of the offloading methods below instead.</p> <ul><li><a href="/docs/diffusers/pr_14909/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> moves one whole model to the GPU at a time. It’s faster, but each model must fit on a single GPU.</li> <li><a href="/docs/diffusers/pr_14909/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_sequential_cpu_offload">enable_sequential_cpu_offload()</a> moves one submodule to the GPU at a time. It uses the least GPU memory, but it’s very slow.</li></ul> <p>Before calling <code>.to()</code>, <code>enable_sequential_cpu_offload</code>, or <code>enable_model_cpu_offload</code> on a device-mapped pipeline, reset its device map with <a href="/docs/diffusers/pr_14909/en/api/pipelines/overview#diffusers.DiffusionPipeline.reset_device_map">reset_device_map()</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.reset_device_map()<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="vae-slicing" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#vae-slicing"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>VAE slicing</span></h2><!--]--><!----> <p>VAE slicing splits a batch of latents into single latents and decodes them one at a time. The decoded images are concatenated back into a batch at the end. Peak decoding memory stays close to the cost of decoding one image, which makes slicing useful when generating several images at once. It has no effect on single-image batches.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-text "><!---->Without slicing: one decode for the whole batch | |
| [ z1 | z2 | z3 | z4 ] --> VAE decode --> [ img1 | img2 | img3 | img4 ] | |
| (peak memory: 4 images) | |
| With slicing: decode one latent at a time, then concatenate | |
| [ z1 ] --> VAE decode --> [ img1 ] --+ | |
| [ z2 ] --> VAE decode --> [ img2 ] --+ | |
| [ z3 ] --> VAE decode --> [ img3 ] --+ | |
| [ z4 ] --> VAE decode --> [ img4 ] --+ | |
| (peak memory: 1 image) | | |
| v | |
| [ img1 | img2 | img3 | img4 ]<!----></pre></div><!----> <p>Call <code>enable_slicing()</code> to enable VAE slicing.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLPipeline | |
| pipeline = StableDiffusionXLPipeline.from_pretrained( | |
| <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, | |
| dtype=torch.float16, | |
| ).to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| pipeline.vae.enable_slicing() | |
| pipeline([<span class="hljs-string">"An astronaut riding a horse on Mars"</span>]*<span class="hljs-number">32</span>).images[<span class="hljs-number">0</span>] | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Max memory allocated: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB"</span>)<!----></pre></div><!----> <blockquote><p>[!WARNING][AsymmetricAutoencoderKL](/docs/diffusers/pr_14909/en/api/models/asymmetricautoencoderkl#diffusers.AsymmetricAutoencoderKL) doesn’t support slicing.</p></blockquote> <!--[1--><h2 class="relative group"><a id="vae-tiling" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#vae-tiling"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>VAE tiling</span></h2><!--]--><!----> <p>VAE tiling splits a latent into overlapping tiles and decodes each tile separately. The overlapping edges are blended together to stitch the tiles into the final image. Peak decoding memory depends mostly on the tile size instead of the full image size, which makes tiling useful for generating high-resolution images.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-text "><!---->1. Split the latent into tiles 2. Decode each tile 3. Blend the overlaps | |
| that overlap by 25% separately and stitch | |
| +--------+----+--------+ | |
| | tile 1 |####| tile 2 | tile 1 --> decode +-------------------+ | |
| | |####| | tile 2 --> decode | | | |
| +--------+----+--------+ --> tile 3 --> decode --> | full-resolution | | |
| |########|####|########| tile 4 --> decode | image | | |
| +--------+----+--------+ | | | |
| | tile 3 |####| tile 4 | (peak memory: 1 tile) +-------------------+ | |
| | |####| | | |
| +--------+----+--------+ | |
| #### = region shared by neighboring tiles<!----></pre></div><!----> <p>Tiles are decoded separately, so tone may vary slightly from tile to tile, but there shouldn’t be any obvious seams. Tiling also applies when encoding an image into latents, and only activates when the input is larger than the VAE’s <code>sample_size</code>.</p> <p>Call <code>enable_tiling()</code> to enable VAE tiling.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoPipelineForImage2Image | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image | |
| pipeline = AutoPipelineForImage2Image.from_pretrained( | |
| <span class="hljs-string">"stabilityai/stable-diffusion-xl-base-1.0"</span>, dtype=torch.float16 | |
| ).to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| pipeline.vae.enable_tiling() | |
| init_image = load_image(<span class="hljs-string">"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/img2img-sdxl-init.png"</span>) | |
| prompt = <span class="hljs-string">"Astronaut in a jungle, cold color palette, muted colors, detailed, 8k"</span> | |
| pipeline(prompt, image=init_image, strength=<span class="hljs-number">0.5</span>).images[<span class="hljs-number">0</span>] | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Max memory allocated: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB"</span>)<!----></pre></div><!----> <blockquote><p>[!WARNING][AsymmetricAutoencoderKL](/docs/diffusers/pr_14909/en/api/models/asymmetricautoencoderkl#diffusers.AsymmetricAutoencoderKL) doesn’t support tiling.</p></blockquote> <!--[1--><h2 class="relative group"><a id="offloading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#offloading"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Offloading</span></h2><!--]--><!----> <p>Offloading keeps inactive layers or models on the CPU and moves them to the GPU only when they’re needed. You can combine offloading with quantization and torch.compile to balance inference speed and memory usage.</p> <p>Refer to the <a href="./speed-memory-optims">Compiling and offloading quantized models</a> guide for more details.</p> <!--[2--><h3 class="relative group"><a id="sequential-cpu-offloading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#sequential-cpu-offloading"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Sequential CPU offloading</span></h3><!--]--><!----> <p>Sequential CPU offloading keeps weights on the CPU and moves each submodule to the GPU only when it runs. The entire model is never on the GPU at once, so sequential offloading uses the least GPU memory of the offloading methods. It’s also the slowest because submodules are transferred between devices many times during inference, which often makes it impractical.</p> <blockquote class="warning"><p>Don’t move the pipeline to CUDA before calling <code>enable_sequential_cpu_offload</code>, otherwise the memory savings are minimal. Refer to <a href="https://github.com/huggingface/diffusers/issues/1934" rel="nofollow">issue #1934</a> for more details. Sequential offloading is stateful and installs hooks on the model.</p></blockquote> <p>Call <a href="/docs/diffusers/pr_14909/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_sequential_cpu_offload">enable_sequential_cpu_offload()</a> to enable it on a pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline | |
| pipeline = DiffusionPipeline.from_pretrained( | |
| <span class="hljs-string">"black-forest-labs/FLUX.1-schnell"</span>, dtype=torch.bfloat16 | |
| ) | |
| pipeline.enable_sequential_cpu_offload() | |
| pipeline( | |
| prompt=<span class="hljs-string">"An astronaut riding a horse on Mars"</span>, | |
| guidance_scale=<span class="hljs-number">0.</span>, | |
| height=<span class="hljs-number">768</span>, | |
| width=<span class="hljs-number">1360</span>, | |
| num_inference_steps=<span class="hljs-number">4</span>, | |
| max_sequence_length=<span class="hljs-number">256</span>, | |
| ).images[<span class="hljs-number">0</span>] | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Max memory allocated: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB"</span>)<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="model-offloading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#model-offloading"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Model offloading</span></h3><!--]--><!----> <p>Model offloading moves whole models to the GPU instead of individual submodules. Only one model, such as the text encoder, denoiser (UNet or transformer), or VAE, is on the GPU at a time while the other models stay on the CPU. A model that runs multiple times, like the denoiser, stays on the GPU until it finishes. Model offloading avoids the transfer overhead of <a href="#sequential-cpu-offloading">sequential CPU offloading</a>, which makes it faster, but the memory savings are smaller.</p> <blockquote class="warning"><p>Model offloading is stateful and installs hooks on each model. If you call a model outside the pipeline, run the models in the pipeline’s order so they’re offloaded correctly, or <a href="https://huggingface.co/docs/accelerate/en/package_reference/big_modeling#accelerate.hooks.remove_hook_from_module" rel="nofollow">remove the hooks</a> first.</p></blockquote> <p>Call <a href="/docs/diffusers/pr_14909/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> to enable it on a pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline | |
| pipeline = DiffusionPipeline.from_pretrained( | |
| <span class="hljs-string">"black-forest-labs/FLUX.1-schnell"</span>, dtype=torch.bfloat16 | |
| ) | |
| pipeline.enable_model_cpu_offload() | |
| pipeline( | |
| prompt=<span class="hljs-string">"An astronaut riding a horse on Mars"</span>, | |
| guidance_scale=<span class="hljs-number">0.</span>, | |
| height=<span class="hljs-number">768</span>, | |
| width=<span class="hljs-number">1360</span>, | |
| num_inference_steps=<span class="hljs-number">4</span>, | |
| max_sequence_length=<span class="hljs-number">256</span>, | |
| ).images[<span class="hljs-number">0</span>] | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Max memory allocated: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB"</span>)<!----></pre></div><!----> <p>Model offloading also helps when you call <a href="/docs/diffusers/pr_14909/en/api/pipelines/stable_diffusion/stable_diffusion_xl#diffusers.StableDiffusionXLPipeline.encode_prompt">encode_prompt()</a> on its own, because only the text encoders are moved to the GPU.</p> <!--[2--><h3 class="relative group"><a id="group-offloading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#group-offloading"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Group offloading</span></h3><!--]--><!----> <p>Group offloading moves groups of internal layers (<a href="https://pytorch.org/docs/stable/generated/torch.nn.ModuleList.html" rel="nofollow">torch.nn.ModuleList</a> or <a href="https://pytorch.org/docs/stable/generated/torch.nn.Sequential.html" rel="nofollow">torch.nn.Sequential</a>) to the CPU. It usually uses less memory than <a href="#model-offloading">model offloading</a> and runs faster than <a href="#sequential-cpu-offloading">sequential CPU offloading</a> because it reduces communication overhead.</p> <blockquote class="warning"><p>Group offloading may not work with models whose forward pass moves inputs to the weights’ device, because that conflicts with how group offloading moves tensors.</p></blockquote> <p>Set <code>offload_type</code> to <code>block_level</code> or <code>leaf_level</code> to choose how layers are grouped.</p> <ul><li><code>block_level</code> offloads groups of layers, and <code>num_blocks_per_group</code> sets the size of each group. For example, <code>num_blocks_per_group=2</code> on a model with 40 layers creates 20 groups and moves 2 layers to the GPU at a time.</li> <li><code>leaf_level</code> offloads each individual layer, similar to <a href="#sequential-cpu-offloading">sequential CPU offloading</a>, but it can be much faster with <a href="#cuda-stream">CUDA streams</a>.</li></ul> <p>Group offloading is supported for entire pipelines or individual models. Apply it to the whole pipeline for the simplest setup, or to individual models to mix offloading techniques.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">pipeline</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">model</div><!--]--></div> <div class="language-select"><!--[0--><p>Call <a href="/docs/diffusers/pr_14909/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_group_offload">enable_group_offload()</a> on a pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video | |
| onload_device = torch.device(<span class="hljs-string">"cuda"</span>) | |
| offload_device = torch.device(<span class="hljs-string">"cpu"</span>) | |
| pipeline = CogVideoXPipeline.from_pretrained(<span class="hljs-string">"THUDM/CogVideoX-5b"</span>, dtype=torch.bfloat16) | |
| pipeline.enable_group_offload( | |
| onload_device=onload_device, | |
| offload_device=offload_device, | |
| offload_type=<span class="hljs-string">"leaf_level"</span>, | |
| use_stream=<span class="hljs-literal">True</span> | |
| ) | |
| prompt = <span class="hljs-string">"A panda playing a tiny acoustic guitar in a bamboo forest"</span> | |
| video = pipeline(prompt=prompt, guidance_scale=<span class="hljs-number">6</span>, num_inference_steps=<span class="hljs-number">50</span>).frames[<span class="hljs-number">0</span>] | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Max memory allocated: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB"</span>) | |
| export_to_video(video, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">8</span>)<!----></pre></div><!----><!----><!--]--><!----> <!--[-1--><!--]--><!----><!----></div><!----> <!--[3--><h4 class="relative group"><a id="cuda-stream" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#cuda-stream"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>CUDA stream</span></h4><!--]--><!----> <p>Set <code>use_stream=True</code> on CUDA devices to prefetch the next layer onto the GPU while the current layer is still running. Overlapping data transfer and computation can make group offloading much faster than <a href="#sequential-cpu-offloading">sequential CPU offloading</a>. Streams create a pinned copy of each weight in CPU memory, so system RAM usage can reach about twice the model size.</p> <p>Set <code>record_stream=True</code> for more of a speedup at the cost of slightly increased memory usage. Refer to the <a href="https://pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html" rel="nofollow">torch.Tensor.record_stream</a> docs to learn more.</p> <blockquote class="tip"><p>If a VAE has tiling enabled and <code>use_stream=True</code>, run a forward pass with dummy inputs before inference to avoid device mismatch errors. Open an <a href="https://github.com/huggingface/diffusers/issues" rel="nofollow">issue</a> if this doesn’t work for your model.</p></blockquote> <p>Streams require <code>num_blocks_per_group=1</code> with <code>block_level</code> offloading. Other values log a warning and are reset to <code>1</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type=<span class="hljs-string">"leaf_level"</span>, use_stream=<span class="hljs-literal">True</span>, record_stream=<span class="hljs-literal">True</span>)<!----></pre></div><!----> <p>Set <code>low_cpu_mem_usage=True</code> to reduce CPU memory usage with streams. Tensors are pinned on the fly instead of all at once up front, which saves CPU memory but can increase inference time. It works best with <code>leaf_level</code> offloading when CPU memory is the bottleneck.</p> <!--[3--><h4 class="relative group"><a id="offloading-to-disk" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#offloading-to-disk"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Offloading to disk</span></h4><!--]--><!----> <p>Group offloading can use a lot of system memory depending on the model size. On systems with limited RAM, offload to disk instead.</p> <p>Set the <code>offload_to_disk_path</code> argument in either <a href="/docs/diffusers/pr_14909/en/api/models/overview#diffusers.ModelMixin.enable_group_offload">enable_group_offload()</a> or <a href="/docs/diffusers/pr_14909/en/api/utilities#diffusers.hooks.apply_group_offloading">apply_group_offloading()</a> to offload the model to the disk.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type=<span class="hljs-string">"leaf_level"</span>, offload_to_disk_path=<span class="hljs-string">"path/to/disk"</span>) | |
| apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type=<span class="hljs-string">"block_level"</span>, num_blocks_per_group=<span class="hljs-number">2</span>, offload_to_disk_path=<span class="hljs-string">"path/to/disk"</span>)<!----></pre></div><!----> <p>Compare the speed and memory trade-offs in the <a href="https://github.com/huggingface/diffusers/pull/11682#issue-3129365363" rel="nofollow">disk offloading benchmark</a> and the <a href="https://github.com/huggingface/diffusers/pull/11682#issuecomment-2955715126" rel="nofollow">follow-up benchmark</a>.</p> <!--[1--><h2 class="relative group"><a id="layerwise-casting" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#layerwise-casting"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Layerwise casting</span></h2><!--]--><!----> <p>Layerwise casting stores weights in a smaller data format (for example, <code>torch.float8_e4m3fn</code> and <code>torch.float8_e5m2</code>) to use less memory and upcasts those weights to a higher precision like <code>torch.float16</code> or <code>torch.bfloat16</code> for computation. By default, positional embeddings, patch embeddings, normalization layers, and the input and output projections are skipped because storing them in fp8 can degrade generation quality.</p> <p>Combine layerwise casting with <a href="#group-offloading">group offloading</a> for even more memory savings.</p> <blockquote class="warning"><p>Layerwise casting may not work with all models if the forward implementation contains internal typecasting of weights. The current implementation of layerwise casting assumes the forward pass is independent of the weight precision and the input datatypes are always specified in <code>compute_dtype</code> (see the <a href="https://github.com/huggingface/transformers/blob/7f5077e53682ca855afc826162b204ebf809f1f9/src/transformers/models/t5/modeling_t5.py#L294-L299" rel="nofollow">T5 implementation in Transformers</a> for an incompatible example).</p> <p>Layerwise casting may also fail on custom modeling implementations with <a href="https://huggingface.co/docs/peft/index" rel="nofollow">PEFT</a> layers. Diffusers includes some checks for this, but they aren’t extensively tested and may not catch every case.</p></blockquote> <p>Call <a href="/docs/diffusers/pr_14909/en/api/models/overview#diffusers.ModelMixin.enable_layerwise_casting">enable_layerwise_casting()</a> to set the storage and computation datatypes.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline, CogVideoXTransformer3DModel | |
| <span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video | |
| transformer = CogVideoXTransformer3DModel.from_pretrained( | |
| <span class="hljs-string">"THUDM/CogVideoX-5b"</span>, | |
| subfolder=<span class="hljs-string">"transformer"</span>, | |
| dtype=torch.bfloat16 | |
| ) | |
| transformer.enable_layerwise_casting(storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16) | |
| pipeline = CogVideoXPipeline.from_pretrained(<span class="hljs-string">"THUDM/CogVideoX-5b"</span>, | |
| transformer=transformer, | |
| dtype=torch.bfloat16 | |
| ).to(<span class="hljs-string">"cuda"</span>) <span class="hljs-comment"># or "mps", "xpu", "cpu"</span> | |
| prompt = <span class="hljs-string">"A panda playing a tiny acoustic guitar in a bamboo forest"</span> | |
| video = pipeline(prompt=prompt, guidance_scale=<span class="hljs-number">6</span>, num_inference_steps=<span class="hljs-number">50</span>).frames[<span class="hljs-number">0</span>] | |
| <span class="hljs-built_in">print</span>(<span class="hljs-string">f"Max memory allocated: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB"</span>) | |
| export_to_video(video, <span class="hljs-string">"output.mp4"</span>, fps=<span class="hljs-number">8</span>)<!----></pre></div><!----> <p>For more control, use <a href="/docs/diffusers/pr_14909/en/api/utilities#diffusers.hooks.apply_layerwise_casting">apply_layerwise_casting()</a>. Call it on specific internal modules to apply layerwise casting to only part of a model, and use <code>skip_modules_pattern</code> or <code>skip_modules_classes</code> to exclude modules such as normalization layers.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXTransformer3DModel | |
| <span class="hljs-keyword">from</span> diffusers.hooks <span class="hljs-keyword">import</span> apply_layerwise_casting | |
| transformer = CogVideoXTransformer3DModel.from_pretrained( | |
| <span class="hljs-string">"THUDM/CogVideoX-5b"</span>, | |
| subfolder=<span class="hljs-string">"transformer"</span>, | |
| dtype=torch.bfloat16 | |
| ) | |
| <span class="hljs-comment"># skip the normalization layer</span> | |
| apply_layerwise_casting( | |
| transformer, | |
| storage_dtype=torch.float8_e4m3fn, | |
| compute_dtype=torch.bfloat16, | |
| skip_modules_pattern=[<span class="hljs-string">"norm"</span>], | |
| non_blocking=<span class="hljs-literal">True</span>, | |
| )<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="torchchannelslast" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#torchchannelslast"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>torch.channels_last</span></h2><!--]--><!----> <p><a href="https://pytorch.org/tutorials/intermediate/memory_format_tutorial.html" rel="nofollow">torch.channels_last</a> changes how tensors are stored in memory from <code>(batch size, channels, height, width)</code> to <code>(batch size, height, width, channels)</code>. Storing each pixel’s channels next to each other matches how many GPU kernels read memory, which mainly speeds up inference rather than reducing memory.</p> <p>channels_last only affects 4D tensors, so it can benefit convolution-based models like UNets and VAEs. Not all operators support the channels-last format, and some models may run slower with it, so benchmark it on your model first.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline | |
| pipeline = StableDiffusionPipeline.from_pretrained( | |
| <span class="hljs-string">"stable-diffusion-v1-5/stable-diffusion-v1-5"</span>, dtype=torch.float16 | |
| ).to(<span class="hljs-string">"cuda"</span>) | |
| <span class="hljs-built_in">print</span>(pipeline.unet.conv_out.state_dict()[<span class="hljs-string">"weight"</span>].stride()) <span class="hljs-comment"># (2880, 9, 3, 1)</span> | |
| pipeline.unet.to(memory_format=torch.channels_last) <span class="hljs-comment"># in-place operation</span> | |
| <span class="hljs-built_in">print</span>( | |
| pipeline.unet.conv_out.state_dict()[<span class="hljs-string">"weight"</span>].stride() | |
| ) <span class="hljs-comment"># (2880, 1, 960, 320) having a stride of 1 for the 2nd dimension proves that it works</span><!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="memory-efficient-attention" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#memory-efficient-attention"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Memory-efficient attention</span></h2><!--]--><!----> <p>Diffusers supports multiple memory-efficient attention backends (FlashAttention, xFormers, SageAttention, and more) through <a href="/docs/diffusers/pr_14909/en/api/models/overview#diffusers.ModelMixin.set_attention_backend">set_attention_backend()</a>. Refer to the <a href="./attention_backends">Attention backends</a> guide to learn how to switch between them.</p> <!--[1--><h2 class="relative group"><a id="next-steps" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#next-steps"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Next steps</span></h2><!--]--><!----> <ul><li>Combine offloading with quantization and torch.compile in the <a href="./speed-memory-optims">Compiling and offloading quantized models</a> guide.</li> <li>Reduce memory further with <a href="../quantization/overview">quantization</a>.</li> <li>Switch attention implementations in the <a href="./attention_backends">Attention backends</a> guide.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/memory.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]--> | |
| <script> | |
| { | |
| __sveltekit_1abqrln = { | |
| base: "/docs/diffusers/pr_14909/en", | |
| assets: "/docs/diffusers/pr_14909/en" | |
| }; | |
| const element = document.currentScript.parentElement; | |
| Promise.all([ | |
| import("/docs/diffusers/pr_14909/en/_app/immutable/entry/start.DH9xfiDx.js"), | |
| import("/docs/diffusers/pr_14909/en/_app/immutable/entry/app.Bgs6fdnu.js") | |
| ]).then(([kit, app]) => { | |
| kit.start(app, element, { | |
| node_ids: [0, 282], | |
| data: [null,null], | |
| form: null, | |
| error: null | |
| }); | |
| }); | |
| } | |
| </script> | |
Xet Storage Details
- Size:
- 94.6 kB
- Xet hash:
- 4e42766181740609c2ce1aec46ef82d5d4abbe0ab3d0238d32aa59635e00e112
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.