Buckets:

hf-doc-build
/

doc-dev

Files

xet

hf-doc-build/doc-dev / diffusers /pr_14304 /en /modular_diffusers /components_manager.html

HuggingFaceDocBuilder

11 days ago

download

raw

54.7 kB

	<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{"title":"ComponentsManager","local":"componentsmanager","sections":[{"title":"Connect to a pipeline","local":"connect-to-a-pipeline","sections":[],"depth":2},{"title":"Inspect components","local":"inspect-components","sections":[],"depth":2},{"title":"Offloading","local":"offloading","sections":[{"title":"Inspecting what the offloader did","local":"inspecting-what-the-offloader-did","sections":[],"depth":3},{"title":"Writing a custom offload strategy","local":"writing-a-custom-offload-strategy","sections":[],"depth":3},{"title":"Finding the bottleneck with the peak column","local":"finding-the-bottleneck-with-the-peak-column","sections":[],"depth":3},{"title":"When a forward pass still runs out of memory","local":"when-a-forward-pass-still-runs-out-of-memory","sections":[],"depth":3}],"depth":2}],"depth":1}"/>
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/entry/start.Bf_ocnxn.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/chunks/9K87XrA5.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/chunks/jDjavuwI.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/entry/app.BmvUxn26.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/chunks/CMF6C6Mt.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/chunks/CUPhp92f.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/chunks/i0LL_sTA.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/nodes/0.DTOisooP.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/chunks/9Uo8d8lw.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/nodes/267.CuhL1vl1.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/chunks/BtE7mKSK.js" rel="modulepreload">
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/chunks/BoRAdXUU.js" rel="modulepreload">
	<!--1ngi0d6--><meta name="hf:doc:metadata" content="{"title":"ComponentsManager","local":"componentsmanager","sections":[{"title":"Connect to a pipeline","local":"connect-to-a-pipeline","sections":[],"depth":2},{"title":"Inspect components","local":"inspect-components","sections":[],"depth":2},{"title":"Offloading","local":"offloading","sections":[{"title":"Inspecting what the offloader did","local":"inspecting-what-the-offloader-did","sections":[],"depth":3},{"title":"Writing a custom offload strategy","local":"writing-a-custom-offload-strategy","sections":[],"depth":3},{"title":"Finding the bottleneck with the peak column","local":"finding-the-bottleneck-with-the-peak-column","sections":[],"depth":3},{"title":"When a forward pass still runs out of memory","local":"when-a-forward-pass-still-runs-out-of-memory","sections":[],"depth":3}],"depth":2}],"depth":1}"/><!---->
	<link href="/docs/diffusers/pr_14304/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="componentsmanager" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#componentsmanager"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>ComponentsManager</span></h1><!--]--><!----> <p>The <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_components#diffusers.ComponentsManager">ComponentsManager</a> is a model registry and management system for Modular Diffusers. It adds and tracks models, stores useful metadata (model size, device placement, adapters), and supports offloading.</p> <p>This guide will show you how to use <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_components#diffusers.ComponentsManager">ComponentsManager</a> to manage components and device memory.</p> <!--[1--><h2 class="relative group"><a id="connect-to-a-pipeline" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#connect-to-a-pipeline"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Connect to a pipeline</span></h2><!--]--><!----> <p>Create a <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_components#diffusers.ComponentsManager">ComponentsManager</a> and pass it to a <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline#diffusers.ModularPipeline">ModularPipeline</a> with either <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline#diffusers.ModularPipeline.from_pretrained">from_pretrained()</a> or <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_blocks#diffusers.ModularPipelineBlocks.init_pipeline">init_pipeline()</a>.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">from_pretrained</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">init_pipeline</div><!--]--></div> <div class="language-select"><!--[0--><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ModularPipeline, ComponentsManager
	<span class="hljs-keyword">import</span> torch

	manager = ComponentsManager()
	pipe = ModularPipeline.from_pretrained(<span class="hljs-string">"Tongyi-MAI/Z-Image-Turbo"</span>, components_manager=manager)
	pipe.load_components(dtype=torch.bfloat16)<!----></pre></div><!----><!--]--><!----> <!--[-1--><!--]--><!----><!----></div><!----> <p>Components loaded by the pipeline are automatically registered in the manager. You can inspect them right away.</p> <!--[1--><h2 class="relative group"><a id="inspect-components" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inspect-components"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Inspect components</span></h2><!--]--><!----> <p>Print the <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_components#diffusers.ComponentsManager">ComponentsManager</a> to see all registered components, including their class, device placement, dtype, memory size, and load ID.</p> <p>The output below corresponds to the <code>from_pretrained</code> example above.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->Components:
	=======================================================================================================================================================================
	Models:
	-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
	Name_ID \| Class \| Device: act(<span class="hljs-built_in">exec</span>) \| Dtype \| Size \| Load ID \| Collection
	-----------------------------------------------------------------------------------------------------------------------------------------------------------------------
	vae_140458257515376 \| AutoencoderKL \| cpu \| torch.bfloat16 \| <span class="hljs-number">159.87</span> MB \| Tongyi-MAI/Z-Image-Turbo\|vae\|null\|null \| N/A
	text_encoder_140458257514752 \| Qwen3Model \| cpu \| torch.bfloat16 \| <span class="hljs-number">7.49</span> GB \| Tongyi-MAI/Z-Image-Turbo\|text_encoder\|null\|null \| N/A
	transformer_140458257515616 \| ZImageTransformer2DModel \| cpu \| torch.bfloat16 \| <span class="hljs-number">11.46</span> GB \| Tongyi-MAI/Z-Image-Turbo\|transformer\|null\|null \| N/A
	-----------------------------------------------------------------------------------------------------------------------------------------------------------------------

	Other Components:
	------------------------------------------------------------------------
	ID \| Class \| Collection
	------------------------------------------------------------------------
	scheduler_140461023555264 \| FlowMatchEulerDiscreteScheduler \| N/A
	tokenizer_140458256346432 \| Qwen2Tokenizer \| N/A
	------------------------------------------------------------------------<!----></pre></div><!----> <p>The table shows models (with device, dtype, and memory info) separately from other components like schedulers and tokenizers. If any models have LoRA adapters, IP-Adapters, or quantization applied, that information is displayed in an additional section at the bottom.</p> <!--[1--><h2 class="relative group"><a id="offloading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#offloading"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Offloading</span></h2><!--]--><!----> <p>The <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_components#diffusers.ComponentsManager.enable_auto_cpu_offload">enable_auto_cpu_offload()</a> method is a global offloading strategy that works across all models regardless of which pipeline is using them. Once enabled, you don’t need to worry about device placement if you add or remove components.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->manager.enable_auto_cpu_offload(device=<span class="hljs-string">"cuda"</span>)<!----></pre></div><!----> <p>All models begin on the CPU and <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_components#diffusers.ComponentsManager">ComponentsManager</a> moves them to the appropriate device right before they’re needed, and moves other models back to the CPU when the incoming model wouldn’t fit. Each offloading decision checks the memory actually available on the device at that moment and keeps <code>memory_reserve</code> (3GB unless you change it) of it free as headroom for activations, which scale with resolution, batch size, and sequence length rather than with the number of models — image models around 1024px are comfortable with a few GB, while video models need more.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->manager.enable_auto_cpu_offload(device=<span class="hljs-string">"cuda"</span>, memory_reserve=<span class="hljs-string">"1GB"</span>)<!----></pre></div><!----> <p>The reserve is an estimate, so a forward pass can still run out of memory. By default the manager recovers on its own: it offloads the smallest model on the device and retries the forward pass, escalating one model at a time until it fits — pass <code>retry_on_oom=False</code> to raise the error instead. A model that still doesn’t fit once everything else is offloaded is too large for the device on its own; use <a href="../optimization/memory#group-offloading">group offloading</a> for it.</p> <p>Components added while offloading is enabled join the managed set without disturbing it: the new component starts on CPU and everything already resident stays where it is; removing a component likewise detaches only that component.</p> <!--[2--><h3 class="relative group"><a id="inspecting-what-the-offloader-did" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#inspecting-what-the-offloader-did"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Inspecting what the offloader did</span></h3><!--]--><!----> <p>Every move is recorded. Print <code>ComponentsManager.offload_record</code> after a run to see the sequence, what each move cost, and where a forward pass ran out of memory.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->manager.enable_auto_cpu_offload(device=<span class="hljs-string">"cuda"</span>)
	pipe(prompt=<span class="hljs-string">"a cat"</span>)
	<span class="hljs-built_in">print</span>(manager.offload_record)<!----></pre></div><!----> <p>On a 20GB card, the Z-Image pipeline from the example above records this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># \| Onload \| Offloaded \| Available \| Peak \| Reason</span>
	-------------------------------------------------------------------------------------------------------------------
	<span class="hljs-number">1</span> \| text_encoder_129824856400544 (<span class="hljs-number">7.49</span> GB) \| - \| <span class="hljs-number">18.02</span> GB \| <span class="hljs-number">8.50</span> KB \|
	<span class="hljs-number">2</span> \| transformer_129824854851072 (<span class="hljs-number">11.46</span> GB) \| text_encoder_129824856400544 (<span class="hljs-number">7.49</span> GB) \| <span class="hljs-number">10.38</span> GB \| <span class="hljs-number">7.77</span> GB \|
	<span class="hljs-number">3</span> \| vae_129824905851968 (<span class="hljs-number">159.87</span> MB) \| - \| <span class="hljs-number">6.36</span> GB \| <span class="hljs-number">12.07</span> GB \|
	-------------------------------------------------------------------------------------------------------------------<!----></pre></div><!----> <p>Each row is one decision: the model that loaded, what was offloaded to make room for it, and the memory picture (<code>Available</code> and <code>Peak</code>, read just before the decision’s first move) it was based on. Here the transformer found 10.38 GB free — not enough for its 11.46 GB of weights while keeping the 3GB <code>memory_reserve</code> — so the text encoder was offloaded first. The VAE then fit into the 6.36 GB left next to the transformer, so it loaded without pushing anything off. The <code>Reason</code> column stays empty for these planned moves; it is filled when a model is offloaded for any other reason: <code>oom_retry:<model></code> when a forward pass ran out of memory and offloading was the rescue (covered below), or <code>offloading_disabled</code> when <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_components#diffusers.ComponentsManager.disable_auto_cpu_offload">disable_auto_cpu_offload()</a> moves everything back to CPU. To watch the moves live as they happen instead, enable info logging with <code>diffusers.logging.set_verbosity_info()</code>.</p> <p>A model appearing repeatedly in this table is thrashing — it is being offloaded and re-loaded every step, which costs a PCIe transfer each way. That usually means <code>memory_reserve</code> is too large (models are pushed off that would have fit) or too small (each step ends in an OOM retry). It can also mean the default strategy is making the wrong call for your workload: a model that runs again inside the same denoise loop should stay resident even when evicting it looks fine on memory alone. That is a case for a custom strategy, which the next section covers.</p> <!--[2--><h3 class="relative group"><a id="writing-a-custom-offload-strategy" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#writing-a-custom-offload-strategy"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Writing a custom offload strategy</span></h3><!--]--><!----> <p>Every decision in the record above was made by an <em>offload strategy</em>: a callable consulted each time a model needs to move onto the device. It receives the hooks of the models currently on the device (<code>hooks</code>), the model about to load (<code>model_id</code>, <code>model</code>), and the <code>execution_device</code>, and returns which of the resident hooks to offload first.</p> <p>The default strategy, <code>AutoOffloadStrategy</code>, is what produced the records so far: it works out how much room the incoming model needs — its weights plus <code>memory_reserve</code>, measured against the memory currently free — and frees the smallest combination of resident models that covers it, or nothing if the model already fits.</p> <p>To make different decisions, write your own callable with the same signature and set it with <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_components#diffusers.ComponentsManager.set_offload_strategy">set_offload_strategy()</a>. For example, we can create a sequential strategy with one line:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">class</span> <span class="hljs-title class_">OffloadEverything</span>:
	<span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, hooks, model_id, model, execution_device</span>):
	<span class="hljs-keyword">return</span> hooks <span class="hljs-comment"># offload every other resident model before each load</span><!----></pre></div><!----> <p>This strategy offloads and onloads models in the sequence they are called (pretty much what <a href="/docs/diffusers/pr_14304/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> does for standard pipelines). Use it like this:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->manager.enable_auto_cpu_offload(device=<span class="hljs-string">"cuda"</span>)
	manager.set_offload_strategy(OffloadEverything())

	pipe(prompt=<span class="hljs-string">"a cat"</span>)
	<span class="hljs-built_in">print</span>(manager.offload_record)<!----></pre></div><!----> <p>The record verifies it did what it promised — every load now evicts the previous resident, even on a large card with plenty of free memory:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># \| Onload \| Offloaded \| Available \| Peak \| Reason</span>
	-------------------------------------------------------------------------------------------------------------------
	<span class="hljs-number">1</span> \| text_encoder_136361472535648 (<span class="hljs-number">7.49</span> GB) \| - \| <span class="hljs-number">78.57</span> GB \| <span class="hljs-number">8.50</span> KB \|
	<span class="hljs-number">2</span> \| transformer_136361472167088 (<span class="hljs-number">11.46</span> GB) \| text_encoder_136361472535648 (<span class="hljs-number">7.49</span> GB) \| <span class="hljs-number">70.93</span> GB \| <span class="hljs-number">7.77</span> GB \|
	<span class="hljs-number">3</span> \| vae_136361468822832 (<span class="hljs-number">159.87</span> MB) \| transformer_136361472167088 (<span class="hljs-number">11.46</span> GB) \| <span class="hljs-number">66.91</span> GB \| <span class="hljs-number">12.07</span> GB \|
	-------------------------------------------------------------------------------------------------------------------<!----></pre></div><!----> <p>The strategy is plain Python over the resident hooks — each carries its <code>model_id</code> (the component name plus a unique suffix) and its <code>model</code> — so it can encode policy the memory numbers can’t express. Say your pipeline has two DiTs that alternate every denoising step: evicting one to make room for the other is exactly the thrashing described above, so keep them resident together by never offloading a <code>dit</code> for a <code>dit</code>, whatever the memory picture says:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">class</span> <span class="hljs-title class_">KeepDitsTogether</span>:
	<span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, hooks, model_id, model, execution_device</span>):
	<span class="hljs-keyword">if</span> model_id.startswith(<span class="hljs-string">"dit"</span>):
	<span class="hljs-keyword">return</span> [hook <span class="hljs-keyword">for</span> hook <span class="hljs-keyword">in</span> hooks <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> hook.model_id.startswith(<span class="hljs-string">"dit"</span>)]
	<span class="hljs-keyword">return</span> hooks<!----></pre></div><!----> <!--[2--><h3 class="relative group"><a id="finding-the-bottleneck-with-the-peak-column" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#finding-the-bottleneck-with-the-peak-column"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Finding the bottleneck with the peak column</span></h3><!--]--><!----> <p><code>Peak</code> is the device’s peak allocated memory as of that moment, so reading down the column tells you <em>when</em> the run’s heaviest moments happened. In the first record above: nothing had run yet when the text encoder loaded (8.50 KB); by the transformer’s turn the peak was the text encoder’s weights plus its activations (7.77 GB); by the VAE’s turn the transformer and its denoising steps had pushed it to 12.07 GB. The end of the run isn’t a row (nothing loads after the VAE), so read the final peak directly — it’s the same counter:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->torch.cuda.max_memory_allocated() <span class="hljs-comment"># 14.15 GB — the peak of the whole run</span><!----></pre></div><!----> <p>That last jump, from 12.07 GB to 14.15 GB with the transformer (11.46 GB) and VAE (159.87 MB) resident, is the decode stage: the ~2.4 GB it needed on top of the weights is exactly the headroom <code>memory_reserve</code> exists to protect, so the default 3GB covers this workload. That is the general recipe for sizing the reserve: run your workflow once at the resolution, batch size, and sequence length you intend to use (activations scale with all three), find the largest jump in the peak column, subtract the weights that loaded in that stretch, and round up generously.</p> <!--[2--><h3 class="relative group"><a id="when-a-forward-pass-still-runs-out-of-memory" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#when-a-forward-pass-still-runs-out-of-memory"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>When a forward pass still runs out of memory</span></h3><!--]--><!----> <p>The strategy plans with weights. It frees enough room for the incoming model’s weights plus <code>memory_reserve</code> — but it cannot see how much the forward pass it just made room for is about to allocate, so the reserve <em>is</em> the activation budget. When the reserve under-covers the real need, the decisions all succeed and the forward pass itself runs out of memory; that is when the OOM retry from earlier steps in, offloading the smallest resident model and rerunning the forward pass until it fits. In other words: an <code>oom_retry</code> row in the record means <code>memory_reserve</code> was set smaller than what the activations actually needed.</p> <p>Here is the same Z-Image workflow on a 13.5GB card with <code>memory_reserve=0</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># \| Onload \| Offloaded \| Available \| Peak \| Reason</span>
	-------------------------------------------------------------------------------------------------------------------
	<span class="hljs-number">1</span> \| text_encoder_133045385135840 (<span class="hljs-number">7.49</span> GB) \| - \| <span class="hljs-number">12.90</span> GB \| <span class="hljs-number">8.50</span> KB \|
	<span class="hljs-number">2</span> \| transformer_133045391586896 (<span class="hljs-number">11.46</span> GB) \| text_encoder_133045385135840 (<span class="hljs-number">7.49</span> GB) \| <span class="hljs-number">5.25</span> GB \| <span class="hljs-number">7.77</span> GB \|
	<span class="hljs-number">3</span> \| vae_133044657178144 (<span class="hljs-number">159.87</span> MB) \| - \| <span class="hljs-number">1.29</span> GB \| <span class="hljs-number">12.07</span> GB \|
	<span class="hljs-number">4</span> \| - \| transformer_133045391586896 (<span class="hljs-number">11.46</span> GB) \| <span class="hljs-number">693.77</span> MB \| <span class="hljs-number">12.65</span> GB \| oom_retry:vae_133044657178144
	-------------------------------------------------------------------------------------------------------------------<!----></pre></div><!----> <p>The first three rows are ordinary: the text encoder yields to the transformer, and with a reserve of 0 the VAE is allowed to squeeze into the 1.29 GB next to it. But decode needs ~2.4 GB of activations (the jump we measured earlier), so the forward pass dies, and row 4 is the rescue: the transformer — the smallest (and only) other resident model — is pushed off mid-run and decode reruns with the whole card to itself. The run completes, but it paid for the bad plan at the worst moment: an unplanned 11.46 GB transfer in the middle of decode, and a re-load next time the transformer runs.</p> <p>The record already told us the right value: decode’s ~2.4 GB of activations need covering, so the default <code>"3GB"</code> reserve fits this workload. Same card, <code>memory_reserve="3GB"</code>:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># \| Onload \| Offloaded \| Available \| Peak \| Reason</span>
	-------------------------------------------------------------------------------------------------------------------
	<span class="hljs-number">1</span> \| text_encoder_129850895513456 (<span class="hljs-number">7.49</span> GB) \| - \| <span class="hljs-number">12.90</span> GB \| <span class="hljs-number">8.50</span> KB \|
	<span class="hljs-number">2</span> \| transformer_129850972639344 (<span class="hljs-number">11.46</span> GB) \| text_encoder_129850895513456 (<span class="hljs-number">7.49</span> GB) \| <span class="hljs-number">5.25</span> GB \| <span class="hljs-number">7.77</span> GB \|
	<span class="hljs-number">3</span> \| vae_129850924782688 (<span class="hljs-number">159.87</span> MB) \| transformer_129850972639344 (<span class="hljs-number">11.46</span> GB) \| <span class="hljs-number">1.29</span> GB \| <span class="hljs-number">12.07</span> GB \|
	-------------------------------------------------------------------------------------------------------------------<!----></pre></div><!----> <p>Now the eviction happens <em>before</em> decode instead of in the middle of it — the VAE’s row offloads the transformer up front because 1.29 GB free minus the reserve leaves no room for it, and there is no OOM row. The result is the same set of transfers, planned instead of forced. If a model OOMs with nothing left to offload, it does not fit on the device by itself — that terminal case and the group-offloading escape are covered <a href="#offloading">above</a>.</p> <p>Call <a href="/docs/diffusers/pr_14304/en/api/modular_diffusers/pipeline_components#diffusers.ComponentsManager.disable_auto_cpu_offload">disable_auto_cpu_offload()</a> to disable offloading.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!---->manager.disable_auto_cpu_offload()<!----></pre></div><!----> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/modular_diffusers/components_manager.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->

	<script>
	{
	__sveltekit_dwtdqu = {
	base: "/docs/diffusers/pr_14304/en",
	assets: "/docs/diffusers/pr_14304/en"
	};

	const element = document.currentScript.parentElement;

	Promise.all([
	import("/docs/diffusers/pr_14304/en/_app/immutable/entry/start.Bf_ocnxn.js"),
	import("/docs/diffusers/pr_14304/en/_app/immutable/entry/app.BmvUxn26.js")
	]).then(([kit, app]) => {
	kit.start(app, element, {
	node_ids: [0, 267],
	data: [null,null],
	form: null,
	error: null
	});
	});
	}
	</script>

Xet Storage Details

Size:: 54.7 kB
Xet hash:: d04d236517f0c73d0a526be01b908686bb469a507f22ccf353ccf767ce8f2e68

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.