Buckets:

hf-doc-build
/

doc-dev

Files
xet
hf-doc-build/doc-dev / diffusers /pr_14909 /en /optimization /speed-memory-optims.html
HuggingFaceDocBuilder
23 days ago
download
raw
22.8 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Compiling and offloading quantized models&quot;,&quot;local&quot;:&quot;compiling-and-offloading-quantized-models&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Quantization and torch.compile&quot;,&quot;local&quot;:&quot;quantization-and-torchcompile&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Quantization, torch.compile, and offloading&quot;,&quot;local&quot;:&quot;quantization-torchcompile-and-offloading&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Next steps&quot;,&quot;local&quot;:&quot;next-steps&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/entry/start.DH9xfiDx.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/C6UGX4Zx.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/jDjavuwI.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/entry/app.Bgs6fdnu.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/CMF6C6Mt.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/DKNTNR4p.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/i0LL_sTA.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/nodes/0.CAKb-ftv.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/9Uo8d8lw.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/nodes/283.DaI4y3VP.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/BLbO27j4.js" rel="modulepreload">
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/chunks/BYVaKgop.js" rel="modulepreload">
		<!--qpa0fj--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Compiling and offloading quantized models&quot;,&quot;local&quot;:&quot;compiling-and-offloading-quantized-models&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Quantization and torch.compile&quot;,&quot;local&quot;:&quot;quantization-and-torchcompile&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Quantization, torch.compile, and offloading&quot;,&quot;local&quot;:&quot;quantization-torchcompile-and-offloading&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Next steps&quot;,&quot;local&quot;:&quot;next-steps&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/diffusers/pr_14909/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="compiling-and-offloading-quantized-models" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#compiling-and-offloading-quantized-models"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Compiling and offloading quantized models</span></h1><!--]--><!----> <p>Quantization, <a href="./fp16#torchcompile">torch.compile</a>, and <a href="./memory#offloading">offloading</a> can be combined to balance <a href="./fp16">inference speed</a> and <a href="./memory">memory usage</a>. Quantization reduces the memory needed to store weights, torch.compile speeds up inference, and offloading keeps inactive layers or models on the CPU until they’re needed. Other techniques trade one for the other. For example, <a href="./cache">caching</a> speeds up inference but increases memory usage because it stores intermediate outputs.</p> <blockquote class="tip"><p>Refer to the <a href="./fp16#torchcompile">torch.compile</a> guide to learn more about compilation. For example, <a href="./fp16#regional-compilation">regional compilation</a> can significantly reduce compilation time while keeping a comparable speedup.</p></blockquote> <p>The offloading method to combine with quantization depends on the workload.</p> <ul><li>For image generation, <a href="./memory#model-offloading">model offloading</a> usually works best. Image models do less compute per layer, so with group offloading, the current layer often finishes before the next layer has transferred and the GPU waits on the CPU.</li> <li>For video generation, <a href="./memory#group-offloading">group offloading</a> usually works better. Video models are more compute-bound, so data transfer can overlap with computation.</li></ul> <p>The table below shows the latency and memory usage of each combination on Flux.</p> <table><thead><tr><th>Combination</th><th>Latency (s)</th><th>Memory usage (GB)</th></tr></thead><tbody><tr><td>quantization</td><td>32.602</td><td>14.9453</td></tr><tr><td>quantization, torch.compile</td><td>25.847</td><td>14.9448</td></tr><tr><td>quantization, torch.compile, model CPU offloading</td><td>32.312</td><td>12.2369</td></tr></tbody></table> <small>Benchmarked on Flux with an RTX 4090, with the `transformer` and `text_encoder_2` (T5) components quantized. Use the <a href="https://gist.github.com/sayakpaul/0db9d8eeeb3d2a0e5ed7cf0d9ca19b7d">benchmarking script</a> to evaluate your own model.</small> <p>The examples below use <a href="../quantization/bitsandbytes#torchcompile">bitsandbytes</a>, but other quantization backends, such as <a href="../quantization/torchao">TorchAO</a>, also support compilation and offloading. Install the latest version of bitsandbytes. <a href="https://pytorch.org/get-started/locally/" rel="nofollow">PyTorch nightly</a> is also recommended.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->pip install -U bitsandbytes<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="quantization-and-torchcompile" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quantization-and-torchcompile"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Quantization and torch.compile</span></h2><!--]--><!----> <p><a href="../quantization/overview">Quantize</a> a model to reduce the memory needed to store its weights, then <a href="./fp16#torchcompile">compile</a> it to speed up inference.</p> <p>Set <code>torch._dynamo.config.capture_dynamic_output_shape_ops = True</code> so <a href="https://docs.pytorch.org/docs/stable/torch.compiler_dynamo_overview.html" rel="nofollow">Dynamo</a> can compile bitsandbytes ops whose output shapes depend on the input.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig

torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

<span class="hljs-comment"># quantize</span>
pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder_2&quot;</span>],
)
pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    quantization_config=pipeline_quant_config,
    dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

<span class="hljs-comment"># compile</span>
pipeline.transformer.to(memory_format=torch.channels_last)
pipeline.transformer.<span class="hljs-built_in">compile</span>(mode=<span class="hljs-string">&quot;max-autotune&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)
pipeline(
    <span class="hljs-string">&quot;cinematic film still of a cat sipping a margarita in a pool in Palm Springs, California, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain&quot;</span>
).images[<span class="hljs-number">0</span>]<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="quantization-torchcompile-and-offloading" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#quantization-torchcompile-and-offloading"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Quantization, torch.compile, and offloading</span></h2><!--]--><!----> <p>Add offloading to quantization and torch.compile to reduce memory usage further. Offloading keeps layers or model components on the CPU and moves them to the GPU only when they’re needed.</p> <p>Raise the <a href="https://docs.pytorch.org/docs/stable/torch.compiler_dynamo_overview.html" rel="nofollow">Dynamo</a> <code>cache_size_limit</code> to avoid excessive recompilation with offloading, and set <code>capture_dynamic_output_shape_ops = True</code> to compile bitsandbytes ops whose output shapes depend on the input.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">model CPU offloading</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">group offloading</div><!--]--></div> <div class="language-select"><!--[0--><p><a href="./memory#model-offloading">Model offloading</a> moves a whole pipeline component, like the transformer, to the GPU only when it’s needed for computation. Otherwise, the component stays on the CPU.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">from</span> diffusers.quantizers <span class="hljs-keyword">import</span> PipelineQuantizationConfig

torch._dynamo.config.cache_size_limit = <span class="hljs-number">1000</span>
torch._dynamo.config.capture_dynamic_output_shape_ops = <span class="hljs-literal">True</span>

<span class="hljs-comment"># quantize</span>
pipeline_quant_config = PipelineQuantizationConfig(
    quant_backend=<span class="hljs-string">&quot;bitsandbytes_4bit&quot;</span>,
    quant_kwargs={<span class="hljs-string">&quot;load_in_4bit&quot;</span>: <span class="hljs-literal">True</span>, <span class="hljs-string">&quot;bnb_4bit_quant_type&quot;</span>: <span class="hljs-string">&quot;nf4&quot;</span>, <span class="hljs-string">&quot;bnb_4bit_compute_dtype&quot;</span>: torch.bfloat16},
    components_to_quantize=[<span class="hljs-string">&quot;transformer&quot;</span>, <span class="hljs-string">&quot;text_encoder_2&quot;</span>],
)
pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    quantization_config=pipeline_quant_config,
    dtype=torch.bfloat16,
)

<span class="hljs-comment"># model CPU offloading</span>
pipeline.enable_model_cpu_offload()

<span class="hljs-comment"># compile</span>
pipeline.transformer.<span class="hljs-built_in">compile</span>()
pipeline(
    <span class="hljs-string">&quot;cinematic film still of a cat sipping a margarita in a pool in Palm Springs, California, highly detailed, high budget hollywood movie, cinemascope, moody, epic, gorgeous, film grain&quot;</span>
).images[<span class="hljs-number">0</span>]<!----></pre></div><!----><!----><!--]--><!----> <!--[-1--><!--]--><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="next-steps" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#next-steps"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>Next steps</span></h2><!--]--><!----> <ul><li>Learn more about each offloading method in the <a href="./memory">Reduce memory usage</a> guide.</li> <li>Speed up inference further with the <a href="./fp16#torchcompile">torch.compile</a> guide.</li> <li>Compare quantization backends in the <a href="../quantization/overview">quantization overview</a>.</li></ul> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/speed-memory-optims.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1abqrln = {
						base: "/docs/diffusers/pr_14909/en",
						assets: "/docs/diffusers/pr_14909/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/diffusers/pr_14909/en/_app/immutable/entry/start.DH9xfiDx.js"),
						import("/docs/diffusers/pr_14909/en/_app/immutable/entry/app.Bgs6fdnu.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 283],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 22.8 kB
Xet hash:: 051291d434ccd7d987104852a106eac5c7b847845f2db9531d4dd6f7ceac86a4
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.