diffusers/pr_14371/en/_app/immutable/nodes/276.BjzVxS5W.js

import"../chunks/DsnmJJEf.js";import{i as T,h as q,C as b,H as s,a as t,E as C,s as I}from"../chunks/BtE7mKSK.js";import{p as k,o as _,s as a,f as A,a as U,b as N,d as w,n as z}from"../chunks/jDjavuwI.js";const Q='{"title":"Nunchaku Lite","local":"nunchaku-lite","sections":[{"title":"Install the CUDA kernels","local":"install-the-cuda-kernels","sections":[],"depth":2},{"title":"Load a quantized pipeline","local":"load-a-quantized-pipeline","sections":[],"depth":2},{"title":"Supported quantization types","local":"supported-quantization-types","sections":[],"depth":2},{"title":"NunchakuLiteQuantizationConfig","local":"nunchakulitequantizationconfig","sections":[],"depth":2},{"title":"torch.compile","local":"torchcompile","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';var B=w('<meta name="hf:doc:metadata"/>'),v=w(`<p></p> <!> <!> <p>Nunchaku Lite is a quantization backend for loading prequantized checkpoints in Diffusers. Create compatible checkpoints with <a href="https://github.com/rootonchair/diffuse-compressor" rel="nofollow">diffuse-compressor</a>. It quantizes and exports a transformer, then packages it as a Diffusers pipeline.</p> <p>Nunchaku Lite builds on the original <a href="https://github.com/nunchaku-ai/nunchaku" rel="nofollow">Nunchaku</a> inference engine, <a href="https://github.com/nunchaku-ai/deepcompressor" rel="nofollow">DeepCompressor</a> quantization library, and <a href="https://arxiv.org/abs/2411.05007" rel="nofollow">SVDQuant paper</a>.</p> <!> <p>The kernels package supplies the optimized CUDA kernels, which load automatically. Install it first.</p> <!> <!> <p>Load the prequantized pipeline with <a href="/docs/diffusers/pr_14371/en/api/pipelines/overview#diffusers.DiffusionPipeline.from_pretrained">from_pretrained()</a>, which reads the quantization
config from <code>config.json</code>.</p> <!> <blockquote class="note"><p>The exported state dict must match the target Diffusers model architecture exactly. For example, a checkpoint
quantized with fused QKV projections won’t load into a model config that expects separate Q, K, and V projection
modules.</p></blockquote> <!> <p>Nunchaku Lite supports the following quantized linear layer formats.</p> <blockquote class="tip"><p>Use <code>nvfp4</code> on Blackwell GPUs. Running <code>int4</code> checkpoints on Blackwell can be slower than <code>nvfp4</code>.</p></blockquote> <p>The CUDA kernels currently support the following NVIDIA GPU architectures:</p> <ul><li><code>sm_75</code> (Turing, for example RTX 2080)</li> <li><code>sm_80</code> (Ampere, for example A100)</li> <li><code>sm_86</code> (Ampere, for example RTX 3090 and RTX A6000)</li> <li><code>sm_89</code> (Ada, for example RTX 4090)</li> <li><code>sm_120</code> (Blackwell, for example RTX 5090)</li></ul> <blockquote class="note"><p>Hopper GPUs, such as <code>sm_90</code> H100 and H200, are not currently supported.</p></blockquote> <p><code>nvfp4</code> checkpoints require a Blackwell or newer NVIDIA GPU. On Blackwell GPUs, use PyTorch >= 2.7 with CUDA >= 12.8. <code>int4</code> checkpoints require a Turing or newer NVIDIA GPU.</p> <table><thead><tr><th>Method</th><th align="right">Precision</th><th align="right">Group size</th><th>Notes</th></tr></thead><tbody><tr><td><code>svdq_w4a4</code></td><td align="right"><code>nvfp4</code></td><td align="right">16</td><td>Uses NVFP4 runtime kernels with SVDQ low-rank correction.</td></tr><tr><td><code>svdq_w4a4</code></td><td align="right"><code>int4</code></td><td align="right">64</td><td>Uses INT4 W4A4 kernels with SVDQ low-rank correction.</td></tr><tr><td><code>awq_w4a16</code></td><td align="right"><code>int4</code></td><td align="right">64</td><td>Uses INT4 weight-only AWQ-style kernels.</td></tr></tbody></table> <!> <p>The <code>config.json</code> file must include a <a href="/docs/diffusers/pr_14371/en/api/quantization#diffusers.NunchakuLiteQuantizationConfig">NunchakuLiteQuantizationConfig</a>. It defines the runtime <code>compute_dtype</code> and the target modules for each Nunchaku Lite quantization method.</p> <ul><li><code>compute_dtype</code>: runtime dtype for floating-point buffers in quantized modules, typically <code>torch.bfloat16</code>.</li> <li><code>svdq_w4a4</code>: SVDQ W4A4 target config with <code>precision</code>, <code>group_size</code>, <code>rank</code>, and <code>targets</code>.</li> <li><code>awq_w4a16</code>: AWQ W4A16 target config with <code>precision</code>, <code>group_size</code>, and <code>targets</code>.</li></ul> <p>Each entry in <code>targets</code> must point to a linear layer. Diffusers swaps each <code>svdq_w4a4</code> target for an SVDQ W4A4 layer and each <code>awq_w4a16</code> target for an AWQ W4A16 layer. The example below shows the
expected shape with shortened target lists.</p> <p>List each module you want to quantize under <code>svdq_w4a4</code> or <code>awq_w4a16</code>. A module can only use one method, so don’t list the same target under both.</p> <!> <!> <p>Nunchaku Lite kernels and quantized linear layers are compatible with <a href="../optimization/fp16#torchcompile"><code>torch.compile</code></a>.
Compile the quantized transformer after loading the pipeline for faster inference.</p> <!> <p>The compiled Nunchaku Lite NVFP4 pipeline runs 1.8x faster than the original BF16 pipeline (2.271s → 1.675s on an RTX PRO 6000).</p> <!> <ul><li><a href="https://github.com/rootonchair/diffuse-compressor" rel="nofollow">diffuse-compressor</a></li> <li><a href="https://nunchaku.tech/docs/nunchaku/installation/installation.html" rel="nofollow">Nunchaku installation requirements</a></li></ul> <!> <p></p>`,1);function X(f,g){k(g,!1),_(()=>{new URLSearchParams(window.location.search).get("fw")}),T();var n=v();q("p4df5g",y=>{var j=B();I(j,"content",Q),U(y,j)});var e=a(A(n),2);b(e,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var l=a(e,2);s(l,{title:"Nunchaku Lite",local:"nunchaku-lite",headingTag:"h1"});var o=a(l,6);s(o,{title:"Install the CUDA kernels",local:"install-the-cuda-kernels",headingTag:"h2"});var i=a(o,4);t(i,{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwa2VybmVscw==",highlighted:"pip install -U kernels",lang:"bash",wrap:!1});var p=a(i,2);s(p,{title:"Load a quantized pipeline",local:"load-a-quantized-pipeline",headingTag:"h2"});var c=a(p,4);t(c,{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMnJvb3RvbmNoYWlyJTJGRVJOSUUtSW1hZ2UtVHVyYm8tbnVuY2hha3UtbGl0ZS1udmZwNCUyMiUwQSUwQXBpcGUlMjAlM0QlMjBEaWZmdXNpb25QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwbW9kZWxfaWQlMkMlMjBkdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKS50byglMjJjdWRhJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMkElMjBtb2Rlcm4lMjByZWQlMjBhcm1jaGFpciUyMGluJTIwYSUyMHF1aWV0JTIwc3R1ZGlvJTJDJTIwc29mdCUyMHdpbmRvdyUyMGxpZ2h0JTJDJTIwcmVhbGlzdGljJTIwcHJvZHVjdCUyMHBob3RvZ3JhcGh5JTIyJTBBaW1hZ2UlMjAlM0QlMjBwaXBlKCUwQSUyMCUyMCUyMCUyMHByb21wdCUzRHByb21wdCUyQyUwQSUyMCUyMCUyMCUyMGhlaWdodCUzRDEwMjQlMkMlMEElMjAlMjAlMjAlMjB3aWR0aCUzRDEwMjQlMkMlMEElMjAlMjAlMjAlMjBudW1faW5mZXJlbmNlX3N0ZXBzJTNEOCUyQyUwQSUyMCUyMCUyMCUyMGd1aWRhbmNlX3NjYWxlJTNEMS4wJTJDJTBBKS5pbWFnZXMlNUIwJTVEJTBBaW1hZ2Uuc2F2ZSglMjJlcm5pZS1pbWFnZS10dXJiby1udW5jaGFrdS1saXRlLnBuZyUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline

model_id = <span class="hljs-string">&quot;rootonchair/ERNIE-Image-Turbo-nunchaku-lite-nvfp4&quot;</span>

pipe = DiffusionPipeline.from_pretrained(
    model_id, dtype=torch.bfloat16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;A modern red armchair in a quiet studio, soft window light, realistic product photography&quot;</span>
image = pipe(
    prompt=prompt,
    height=<span class="hljs-number">1024</span>,
    width=<span class="hljs-number">1024</span>,
    num_inference_steps=<span class="hljs-number">8</span>,
    guidance_scale=<span class="hljs-number">1.0</span>,
).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;ernie-image-turbo-nunchaku-lite.png&quot;</span>)`,lang:"python",wrap:!1});var u=a(c,4);s(u,{title:"Supported quantization types",local:"supported-quantization-types",headingTag:"h2"});var r=a(u,16);s(r,{title:"NunchakuLiteQuantizationConfig",local:"nunchakulitequantizationconfig",headingTag:"h2"});var d=a(r,10);t(d,{code:"JTdCJTBBJTIwJTIwJTIyX2NsYXNzX25hbWUlMjIlM0ElMjAlMjJFcm5pZUltYWdlVHJhbnNmb3JtZXIyRE1vZGVsJTIyJTJDJTBBJTIwJTIwJTIycXVhbnRpemF0aW9uX2NvbmZpZyUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMnF1YW50X21ldGhvZCUyMiUzQSUyMCUyMm51bmNoYWt1X2xpdGUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJjb21wdXRlX2R0eXBlJTIyJTNBJTIwJTIyYmZsb2F0MTYlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJzdmRxX3c0YTQlMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjJwcmVjaXNpb24lMjIlM0ElMjAlMjJudmZwNCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMmdyb3VwX3NpemUlMjIlM0ElMjAxNiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMnJhbmslMjIlM0ElMjAzMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMnRhcmdldHMlMjIlM0ElMjAlNUIlMjJsYXllcnMuMC5zZWxmX2F0dGVudGlvbi50b19xJTIyJTVEJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyYXdxX3c0YTE2JTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIycHJlY2lzaW9uJTIyJTNBJTIwJTIyaW50NCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMmdyb3VwX3NpemUlMjIlM0ElMjA2NCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMnRhcmdldHMlMjIlM0ElMjAlNUIlMjJmaW5hbF9saW5lYXIlMjIlNUQlMEElMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlN0QlMEElN0Q=",highlighted:`<span class="hljs-punctuation">{</span>
  <span class="hljs-attr">&quot;_class_name&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;ErnieImageTransformer2DModel&quot;</span><span class="hljs-punctuation">,</span>
  <span class="hljs-attr">&quot;quantization_config&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;quant_method&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;nunchaku_lite&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;compute_dtype&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;bfloat16&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;svdq_w4a4&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
      <span class="hljs-attr">&quot;precision&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;nvfp4&quot;</span><span class="hljs-punctuation">,</span>
      <span class="hljs-attr">&quot;group_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">16</span><span class="hljs-punctuation">,</span>
      <span class="hljs-attr">&quot;rank&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">32</span><span class="hljs-punctuation">,</span>
      <span class="hljs-attr">&quot;targets&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-string">&quot;layers.0.self_attention.to_q&quot;</span><span class="hljs-punctuation">]</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;awq_w4a16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
      <span class="hljs-attr">&quot;precision&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;int4&quot;</span><span class="hljs-punctuation">,</span>
      <span class="hljs-attr">&quot;group_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">64</span><span class="hljs-punctuation">,</span>
      <span class="hljs-attr">&quot;targets&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span><span class="hljs-string">&quot;final_linear&quot;</span><span class="hljs-punctuation">]</span>
    <span class="hljs-punctuation">}</span>
  <span class="hljs-punctuation">}</span>
<span class="hljs-punctuation">}</span>`,lang:"json",wrap:!1});var h=a(d,2);s(h,{title:"torch.compile",local:"torchcompile",headingTag:"h2"});var M=a(h,4);t(M,{code:"cGlwZS50cmFuc2Zvcm1lciUyMCUzRCUyMHRvcmNoLmNvbXBpbGUocGlwZS50cmFuc2Zvcm1lciUyQyUyMG1vZGUlM0QlMjJkZWZhdWx0JTIyJTJDJTIwZnVsbGdyYXBoJTNEVHJ1ZSk=",highlighted:'pipe.transformer = torch.<span class="hljs-built_in">compile</span>(pipe.transformer, mode=<span class="hljs-string">&quot;default&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)',lang:"python",wrap:!1});var m=a(M,4);s(m,{title:"Resources",local:"resources",headingTag:"h2"});var J=a(m,4);C(J,{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/nunchaku.md"}),z(2),U(f,n),N()}export{X as component};