diffusers/pr_9791/en/_app/immutable/nodes/186.2b8cb683.js

import{s as Zt,o as gt,n as Me}from"../chunks/scheduler.8c3d61f6.js";import{S as Ct,i as _t,g as h,s as a,r as p,A as Wt,h as J,f as l,c as o,j as Tt,u as d,x as $,k as wt,y as vt,a as s,v as u,d as y,t as c,w as b}from"../chunks/index.da70eac4.js";import{T as lt}from"../chunks/Tip.1d9b8c37.js";import{C as R}from"../chunks/CodeBlock.00a903b3.js";import{H as z,E as Rt}from"../chunks/EditOnGithub.1e64e623.js";import{H as Bt,a as Ut}from"../chunks/HfOption.c1483eb1.js";function zt(v){let i,T="Quantizing a model in 8-bit halves the memory-usage:",r,f,m,M,_="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",W,U,g,Z,B='Once a model is quantized, you can push the model to the Hub with the <a href="/docs/diffusers/pr_9791/en/api/pipelines/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization <code>config.json</code> file is pushed first, followed by the quantized model weights. You can also save the serialized 4-bit models locally with <a href="/docs/diffusers/pr_9791/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.',C;return f=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),U=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIlMEEpJTBBbW9kZWxfOGJpdC50cmFuc2Zvcm1lcl9ibG9ja3MubGF5ZXJzJTVCLTElNUQubm9ybTIud2VpZ2h0LmR0eXBl",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
    torch_dtype=torch.float32
)
model_8bit.transformer_blocks.layers[-<span class="hljs-number">1</span>].norm2.weight.dtype`,wrap:!1}}),{c(){i=h("p"),i.textContent=T,r=a(),p(f.$$.fragment),m=a(),M=h("p"),M.innerHTML=_,W=a(),p(U.$$.fragment),g=a(),Z=h("p"),Z.innerHTML=B},l(n){i=J(n,"P",{"data-svelte-h":!0}),$(i)!=="svelte-4djpqq"&&(i.textContent=T),r=o(n),d(f.$$.fragment,n),m=o(n),M=J(n,"P",{"data-svelte-h":!0}),$(M)!=="svelte-kfzkum"&&(M.innerHTML=_),W=o(n),d(U.$$.fragment,n),g=o(n),Z=J(n,"P",{"data-svelte-h":!0}),$(Z)!=="svelte-18vdqhc"&&(Z.innerHTML=B)},m(n,w){s(n,i,w),s(n,r,w),u(f,n,w),s(n,m,w),s(n,M,w),s(n,W,w),u(U,n,w),s(n,g,w),s(n,Z,w),C=!0},p:Me,i(n){C||(y(f.$$.fragment,n),y(U.$$.fragment,n),C=!0)},o(n){c(f.$$.fragment,n),c(U.$$.fragment,n),C=!1},d(n){n&&(l(i),l(r),l(m),l(M),l(W),l(g),l(Z)),b(f,n),b(U,n)}}}function kt(v){let i,T="Quantizing a model in 4-bit reduces your memory-usage by 4x:",r,f,m,M,_="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",W,U,g,Z,B='Call <a href="/docs/diffusers/pr_9791/en/api/pipelines/overview#diffusers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> after loading it in 4-bit precision. You can also save the serialized 4-bit models locally with <a href="/docs/diffusers/pr_9791/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>.',C;return f=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),U=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIlMEEpJTBBbW9kZWxfNGJpdC50cmFuc2Zvcm1lcl9ibG9ja3MubGF5ZXJzJTVCLTElNUQubm9ybTIud2VpZ2h0LmR0eXBl",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
    torch_dtype=torch.float32
)
model_4bit.transformer_blocks.layers[-<span class="hljs-number">1</span>].norm2.weight.dtype`,wrap:!1}}),{c(){i=h("p"),i.textContent=T,r=a(),p(f.$$.fragment),m=a(),M=h("p"),M.innerHTML=_,W=a(),p(U.$$.fragment),g=a(),Z=h("p"),Z.innerHTML=B},l(n){i=J(n,"P",{"data-svelte-h":!0}),$(i)!=="svelte-i07c71"&&(i.textContent=T),r=o(n),d(f.$$.fragment,n),m=o(n),M=J(n,"P",{"data-svelte-h":!0}),$(M)!=="svelte-kfzkum"&&(M.innerHTML=_),W=o(n),d(U.$$.fragment,n),g=o(n),Z=J(n,"P",{"data-svelte-h":!0}),$(Z)!=="svelte-i1mk7z"&&(Z.innerHTML=B)},m(n,w){s(n,i,w),s(n,r,w),u(f,n,w),s(n,m,w),s(n,M,w),s(n,W,w),u(U,n,w),s(n,g,w),s(n,Z,w),C=!0},p:Me,i(n){C||(y(f.$$.fragment,n),y(U.$$.fragment,n),C=!0)},o(n){c(f.$$.fragment,n),c(U.$$.fragment,n),C=!1},d(n){n&&(l(i),l(r),l(m),l(M),l(W),l(g),l(Z)),b(f,n),b(U,n)}}}function jt(v){let i,T,r,f;return i=new Ut({props:{id:"bnb",option:"8-bit",$$slots:{default:[zt]},$$scope:{ctx:v}}}),r=new Ut({props:{id:"bnb",option:"4-bit",$$slots:{default:[kt]},$$scope:{ctx:v}}}),{c(){p(i.$$.fragment),T=a(),p(r.$$.fragment)},l(m){d(i.$$.fragment,m),T=o(m),d(r.$$.fragment,m)},m(m,M){u(i,m,M),s(m,T,M),u(r,m,M),f=!0},p(m,M){const _={};M&2&&(_.$$scope={dirty:M,ctx:m}),i.$set(_);const W={};M&2&&(W.$$scope={dirty:M,ctx:m}),r.$set(W)},i(m){f||(y(i.$$.fragment,m),y(r.$$.fragment,m),f=!0)},o(m){c(i.$$.fragment,m),c(r.$$.fragment,m),f=!1},d(m){m&&l(T),b(i,m),b(r,m)}}}function Ft(v){let i,T="Training with 8-bit and 4-bit weights are only supported for training <em>extra</em> parameters.";return{c(){i=h("p"),i.innerHTML=T},l(r){i=J(r,"P",{"data-svelte-h":!0}),$(i)!=="svelte-of9sym"&&(i.innerHTML=T)},m(r,f){s(r,i,f)},p:Me,d(r){r&&l(i)}}}function Gt(v){let i,T='Learn more about the details of 8-bit quantization in this <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">blog post</a>!';return{c(){i=h("p"),i.innerHTML=T},l(r){i=J(r,"P",{"data-svelte-h":!0}),$(i)!=="svelte-1bb05fp"&&(i.innerHTML=T)},m(r,f){s(r,i,f)},p:Me,d(r){r&&l(i)}}}function qt(v){let i,T='Learn more about its details in this <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">blog post</a>.';return{c(){i=h("p"),i.innerHTML=T},l(r){i=J(r,"P",{"data-svelte-h":!0}),$(i)!=="svelte-kpdzjq"&&(i.innerHTML=T)},m(r,f){s(r,i,f)},p:Me,d(r){r&&l(i)}}}function Vt(v){let i,T,r,f,m,M,_,W='<a href="https://huggingface.co/docs/bitsandbytes/index" rel="nofollow">bitsandbytes</a> is the easiest option for quantizing a model to 8 and 4-bit. 8-bit quantization multiplies outliers in fp16 with non-outliers in int8, converts the non-outlier values back to fp16, and then adds them together to return the weights in fp16. This reduces the degradative effect outlier values have on a model’s performance.',U,g,Z='4-bit quantization compresses a model even further, and it is commonly used with <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> to finetune quantized LLMs.',B,C,n="To use bitsandbytes, make sure you have the following libraries installed:",w,q,he,V,st='Now you can quantize a model by passing a <a href="/docs/diffusers/pr_9791/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a> to <a href="/docs/diffusers/pr_9791/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>. This works for any model in any modality, as long as it supports loading with <a href="https://hf.co/docs/accelerate/index" rel="nofollow">Accelerate</a> and contains <code>torch.nn.Linear</code> layers.',Je,k,$e,j,Te,Q,nt="Check your memory footprint with the <code>get_memory_footprint</code> method:",we,x,Ue,X,it='Quantized models can be loaded from the <a href="/docs/diffusers/pr_9791/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a> method without needing to specify the <code>quantization_config</code> parameters:',Ze,E,ge,I,Ce,F,_e,N,at="This section explores some of the specific features of 8-bit models, such as outlier thresholds and skipping module conversion.",We,Y,ve,L,ot="An “outlier” is a hidden state value greater than a certain threshold, and these values are computed in fp16. While the values are usually normally distributed ([-3.5, 3.5]), this distribution can be very different for large models ([-60, 6] or [6, 60]). 8-bit quantization works well for values ~5, but beyond that, there is a significant performance penalty. A good default threshold value is 6, but a lower threshold may be needed for more unstable models (small models or finetuning).",Re,H,rt='To find the best threshold for your model, we recommend experimenting with the <code>llm_int8_threshold</code> parameter in <a href="/docs/diffusers/pr_9791/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Be,A,ze,S,ke,D,mt='For some models, you don’t need to quantize every module to 8-bit which can actually cause instability. For example, for diffusion models like <a href="../api/pipelines/stable_diffusion/stable_diffusion_3">Stable Diffusion 3</a>, the <code>proj_out</code> module can be skipped using the <code>llm_int8_skip_modules</code> parameter in <a href="/docs/diffusers/pr_9791/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',je,P,Fe,K,Ge,G,qe,O,ft="This section explores some of the specific features of 4-bit models, such as changing the compute data type, using the Normal Float 4 (NF4) data type, and using nested quantization.",Ve,ee,Qe,te,pt='To speedup computation, you can change the data type from float32 (the default value) to bf16 using the <code>bnb_4bit_compute_dtype</code> parameter in <a href="/docs/diffusers/pr_9791/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',xe,le,Xe,se,Ee,ne,dt='NF4 is a 4-bit data type from the <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> paper, adapted for weights initialized from a normal distribution. You should use NF4 for training 4-bit base models. This can be configured with the <code>bnb_4bit_quant_type</code> parameter in the <a href="/docs/diffusers/pr_9791/en/api/quantization#diffusers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Ie,ie,Ne,ae,ut="For inference, the <code>bnb_4bit_quant_type</code> does not have a huge impact on performance. However, to remain consistent with the model weights, you should use the <code>bnb_4bit_compute_dtype</code> and <code>torch_dtype</code> values.",Ye,oe,Le,re,yt="Nested quantization is a technique that can save additional memory at no additional performance cost. This feature performs a second quantization of the already quantized weights to save an additional 0.4 bits/parameter.",He,me,Ae,fe,Se,pe,ct="Once quantized, you can dequantize the model to the original precision but this might result in a small quality loss of the model. Make sure you have enough GPU RAM to fit the dequantized model.",De,de,Pe,ue,Ke,ye,bt='<li><a href="https://gist.github.com/sayakpaul/c76bd845b48759e11687ac550b99d8b4" rel="nofollow">End-to-end notebook showing Flux.1 Dev inference in a free-tier Colab</a></li> <li><a href="https://gist.github.com/sayakpaul/05afd428bc089b47af7c016e42004527" rel="nofollow">Training</a></li>',Oe,ce,et,be,tt;return m=new z({props:{title:"bitsandbytes",local:"bitsandbytes",headingTag:"h1"}}),q=new R({props:{code:"cGlwJTIwaW5zdGFsbCUyMGRpZmZ1c2VycyUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlMjAtVQ==",highlighted:"pip install diffusers transformers accelerate bitsandbytes -U",wrap:!1}}),k=new Bt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[jt]},$$scope:{ctx:v}}}),j=new lt({props:{warning:!0,$$slots:{default:[Ft]},$$scope:{ctx:v}}}),x=new R({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())',wrap:!1}}),E=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmhmLWludGVybmFsLXRlc3RpbmclMkZmbHV4LjEtZGV2LW5mNC1wa2clMjIlMkMlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUwQSk=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;hf-internal-testing/flux.1-dev-nf4-pkg&quot;</span>, subfolder=<span class="hljs-string">&quot;transformer&quot;</span>
)`,wrap:!1}}),I=new z({props:{title:"8-bit (LLM.int8() algorithm)",local:"8-bit-llmint8-algorithm",headingTag:"h2"}}),F=new lt({props:{$$slots:{default:[Gt]},$$scope:{ctx:v}}}),Y=new z({props:{title:"Outlier threshold",local:"outlier-threshold",headingTag:"h3"}}),A=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwlMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGxvYWRfaW5fOGJpdCUzRFRydWUlMkMlMjBsbG1faW50OF90aHJlc2hvbGQlM0QxMCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwRmx1eFRyYW5zZm9ybWVyMkRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxTransformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = FluxTransformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),S=new z({props:{title:"Skip module conversion",local:"skip-module-conversion",headingTag:"h3"}}),P=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFNEM1RyYW5zZm9ybWVyMkRNb2RlbCUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl84Yml0JTNEVHJ1ZSUyQyUyMGxsbV9pbnQ4X3NraXBfbW9kdWxlcyUzRCU1QiUyMnByb2pfb3V0JTIyJTVEJTJDJTBBKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBTRDNUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJpbGl0eWFpJTJGc3RhYmxlLWRpZmZ1c2lvbi0zLW1lZGl1bS1kaWZmdXNlcnMlMjIlMkMlMEElMjAlMjAlMjAlMjBzdWJmb2xkZXIlM0QlMjJ0cmFuc2Zvcm1lciUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> SD3Transformer2DModel, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_8bit=<span class="hljs-literal">True</span>, llm_int8_skip_modules=[<span class="hljs-string">&quot;proj_out&quot;</span>],
)

model_8bit = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),K=new z({props:{title:"4-bit (QLoRA algorithm)",local:"4-bit-qlora-algorithm",headingTag:"h2"}}),G=new lt({props:{$$slots:{default:[qt]},$$scope:{ctx:v}}}),ee=new z({props:{title:"Compute data type",local:"compute-data-type",headingTag:"h3"}}),le=new R({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`,wrap:!1}}),se=new z({props:{title:"Normal Float 4 (NF4)",local:"normal-float-4-nf4",headingTag:"h3"}}),ie=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW5mNF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBbW9kZWxfbmY0JTIwJTNEJTIwU0QzVHJhbnNmb3JtZXIyRE1vZGVsLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJzdGFiaWxpdHlhaSUyRnN0YWJsZS1kaWZmdXNpb24tMy1tZWRpdW0tZGlmZnVzZXJzJTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEbmY0X2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

nf4_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

model_nf4 = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=nf4_config,
)`,wrap:!1}}),oe=new z({props:{title:"Nested quantization",local:"nested-quantization",headingTag:"h3"}}),me=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQWRvdWJsZV9xdWFudF9tb2RlbCUyMCUzRCUyMFNEM1RyYW5zZm9ybWVyMkRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyc3RhYmlsaXR5YWklMkZzdGFibGUtZGlmZnVzaW9uLTMtbWVkaXVtLWRpZmZ1c2VycyUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGRvdWJsZV9xdWFudF9jb25maWclMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

double_quant_model = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=double_quant_config,
)`,wrap:!1}}),fe=new z({props:{title:"Dequantizing bitsandbytes models",local:"dequantizing-bitsandbytes-models",headingTag:"h2"}}),de=new R({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQWRvdWJsZV9xdWFudF9tb2RlbCUyMCUzRCUyMFNEM1RyYW5zZm9ybWVyMkRNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyc3RhYmlsaXR5YWklMkZzdGFibGUtZGlmZnVzaW9uLTMtbWVkaXVtLWRpZmZ1c2VycyUyMiUyQyUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGRvdWJsZV9xdWFudF9jb25maWclMkMlMEEpJTBBbW9kZWwuZGVxdWFudGl6ZSgp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

double_quant_model = SD3Transformer2DModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-3-medium-diffusers&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    quantization_config=double_quant_config,
)
model.dequantize()`,wrap:!1}}),ue=new z({props:{title:"Resources",local:"resources",headingTag:"h2"}}),ce=new Rt({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/bitsandbytes.md"}}),{c(){i=h("meta"),T=a(),r=h("p"),f=a(),p(m.$$.fragment),M=a(),_=h("p"),_.innerHTML=W,U=a(),g=h("p"),g.innerHTML=Z,B=a(),C=h("p"),C.textContent=n,w=a(),p(q.$$.fragment),he=a(),V=h("p"),V.innerHTML=st,Je=a(),p(k.$$.fragment),$e=a(),p(j.$$.fragment),Te=a(),Q=h("p"),Q.innerHTML=nt,we=a(),p(x.$$.fragment),Ue=a(),X=h("p"),X.innerHTML=it,Ze=a(),p(E.$$.fragment),ge=a(),p(I.$$.fragment),Ce=a(),p(F.$$.fragment),_e=a(),N=h("p"),N.textContent=at,We=a(),p(Y.$$.fragment),ve=a(),L=h("p"),L.textContent=ot,Re=a(),H=h("p"),H.innerHTML=rt,Be=a(),p(A.$$.fragment),ze=a(),p(S.$$.fragment),ke=a(),D=h("p"),D.innerHTML=mt,je=a(),p(P.$$.fragment),Fe=a(),p(K.$$.fragment),Ge=a(),p(G.$$.fragment),qe=a(),O=h("p"),O.textContent=ft,Ve=a(),p(ee.$$.fragment),Qe=a(),te=h("p"),te.innerHTML=pt,xe=a(),p(le.$$.fragment),Xe=a(),p(se.$$.fragment),Ee=a(),ne=h("p"),ne.innerHTML=dt,Ie=a(),p(ie.$$.fragment),Ne=a(),ae=h("p"),ae.innerHTML=ut,Ye=a(),p(oe.$$.fragment),Le=a(),re=h("p"),re.textContent=yt,He=a(),p(me.$$.fragment),Ae=a(),p(fe.$$.fragment),Se=a(),pe=h("p"),pe.textContent=ct,De=a(),p(de.$$.fragment),Pe=a(),p(ue.$$.fragment),Ke=a(),ye=h("ul"),ye.innerHTML=bt,Oe=a(),p(ce.$$.fragment),et=a(),be=h("p"),this.h()},l(e){const t=Wt("svelte-u9bgzb",document.head);i=J(t,"META",{name:!0,content:!0}),t.forEach(l),T=o(e),r=J(e,"P",{}),Tt(r).forEach(l),f=o(e),d(m.$$.fragment,e),M=o(e),_=J(e,"P",{"data-svelte-h":!0}),$(_)!=="svelte-1m50ob7"&&(_.innerHTML=W),U=o(e),g=J(e,"P",{"data-svelte-h":!0}),$(g)!=="svelte-11pyf03"&&(g.innerHTML=Z),B=o(e),C=J(e,"P",{"data-svelte-h":!0}),$(C)!=="svelte-gf36q7"&&(C.textContent=n),w=o(e),d(q.$$.fragment,e),he=o(e),V=J(e,"P",{"data-svelte-h":!0}),$(V)!=="svelte-mjq7o1"&&(V.innerHTML=st),Je=o(e),d(k.$$.fragment,e),$e=o(e),d(j.$$.fragment,e),Te=o(e),Q=J(e,"P",{"data-svelte-h":!0}),$(Q)!=="svelte-139tok6"&&(Q.innerHTML=nt),we=o(e),d(x.$$.fragment,e),Ue=o(e),X=J(e,"P",{"data-svelte-h":!0}),$(X)!=="svelte-1qtuvpr"&&(X.innerHTML=it),Ze=o(e),d(E.$$.fragment,e),ge=o(e),d(I.$$.fragment,e),Ce=o(e),d(F.$$.fragment,e),_e=o(e),N=J(e,"P",{"data-svelte-h":!0}),$(N)!=="svelte-1myadau"&&(N.textContent=at),We=o(e),d(Y.$$.fragment,e),ve=o(e),L=J(e,"P",{"data-svelte-h":!0}),$(L)!=="svelte-ur5rgd"&&(L.textContent=ot),Re=o(e),H=J(e,"P",{"data-svelte-h":!0}),$(H)!=="svelte-16ffh27"&&(H.innerHTML=rt),Be=o(e),d(A.$$.fragment,e),ze=o(e),d(S.$$.fragment,e),ke=o(e),D=J(e,"P",{"data-svelte-h":!0}),$(D)!=="svelte-y4i7ya"&&(D.innerHTML=mt),je=o(e),d(P.$$.fragment,e),Fe=o(e),d(K.$$.fragment,e),Ge=o(e),d(G.$$.fragment,e),qe=o(e),O=J(e,"P",{"data-svelte-h":!0}),$(O)!=="svelte-7ob7j"&&(O.textContent=ft),Ve=o(e),d(ee.$$.fragment,e),Qe=o(e),te=J(e,"P",{"data-svelte-h":!0}),$(te)!=="svelte-1mzcevu"&&(te.innerHTML=pt),xe=o(e),d(le.$$.fragment,e),Xe=o(e),d(se.$$.fragment,e),Ee=o(e),ne=J(e,"P",{"data-svelte-h":!0}),$(ne)!=="svelte-glehta"&&(ne.innerHTML=dt),Ie=o(e),d(ie.$$.fragment,e),Ne=o(e),ae=J(e,"P",{"data-svelte-h":!0}),$(ae)!=="svelte-1qoc2ct"&&(ae.innerHTML=ut),Ye=o(e),d(oe.$$.fragment,e),Le=o(e),re=J(e,"P",{"data-svelte-h":!0}),$(re)!=="svelte-ep1hhf"&&(re.textContent=yt),He=o(e),d(me.$$.fragment,e),Ae=o(e),d(fe.$$.fragment,e),Se=o(e),pe=J(e,"P",{"data-svelte-h":!0}),$(pe)!=="svelte-1gygypd"&&(pe.textContent=ct),De=o(e),d(de.$$.fragment,e),Pe=o(e),d(ue.$$.fragment,e),Ke=o(e),ye=J(e,"UL",{"data-svelte-h":!0}),$(ye)!=="svelte-vejlmz"&&(ye.innerHTML=bt),Oe=o(e),d(ce.$$.fragment,e),et=o(e),be=J(e,"P",{}),Tt(be).forEach(l),this.h()},h(){wt(i,"name","hf:doc:metadata"),wt(i,"content",Qt)},m(e,t){vt(document.head,i),s(e,T,t),s(e,r,t),s(e,f,t),u(m,e,t),s(e,M,t),s(e,_,t),s(e,U,t),s(e,g,t),s(e,B,t),s(e,C,t),s(e,w,t),u(q,e,t),s(e,he,t),s(e,V,t),s(e,Je,t),u(k,e,t),s(e,$e,t),u(j,e,t),s(e,Te,t),s(e,Q,t),s(e,we,t),u(x,e,t),s(e,Ue,t),s(e,X,t),s(e,Ze,t),u(E,e,t),s(e,ge,t),u(I,e,t),s(e,Ce,t),u(F,e,t),s(e,_e,t),s(e,N,t),s(e,We,t),u(Y,e,t),s(e,ve,t),s(e,L,t),s(e,Re,t),s(e,H,t),s(e,Be,t),u(A,e,t),s(e,ze,t),u(S,e,t),s(e,ke,t),s(e,D,t),s(e,je,t),u(P,e,t),s(e,Fe,t),u(K,e,t),s(e,Ge,t),u(G,e,t),s(e,qe,t),s(e,O,t),s(e,Ve,t),u(ee,e,t),s(e,Qe,t),s(e,te,t),s(e,xe,t),u(le,e,t),s(e,Xe,t),u(se,e,t),s(e,Ee,t),s(e,ne,t),s(e,Ie,t),u(ie,e,t),s(e,Ne,t),s(e,ae,t),s(e,Ye,t),u(oe,e,t),s(e,Le,t),s(e,re,t),s(e,He,t),u(me,e,t),s(e,Ae,t),u(fe,e,t),s(e,Se,t),s(e,pe,t),s(e,De,t),u(de,e,t),s(e,Pe,t),u(ue,e,t),s(e,Ke,t),s(e,ye,t),s(e,Oe,t),u(ce,e,t),s(e,et,t),s(e,be,t),tt=!0},p(e,[t]){const Mt={};t&2&&(Mt.$$scope={dirty:t,ctx:e}),k.$set(Mt);const ht={};t&2&&(ht.$$scope={dirty:t,ctx:e}),j.$set(ht);const Jt={};t&2&&(Jt.$$scope={dirty:t,ctx:e}),F.$set(Jt);const $t={};t&2&&($t.$$scope={dirty:t,ctx:e}),G.$set($t)},i(e){tt||(y(m.$$.fragment,e),y(q.$$.fragment,e),y(k.$$.fragment,e),y(j.$$.fragment,e),y(x.$$.fragment,e),y(E.$$.fragment,e),y(I.$$.fragment,e),y(F.$$.fragment,e),y(Y.$$.fragment,e),y(A.$$.fragment,e),y(S.$$.fragment,e),y(P.$$.fragment,e),y(K.$$.fragment,e),y(G.$$.fragment,e),y(ee.$$.fragment,e),y(le.$$.fragment,e),y(se.$$.fragment,e),y(ie.$$.fragment,e),y(oe.$$.fragment,e),y(me.$$.fragment,e),y(fe.$$.fragment,e),y(de.$$.fragment,e),y(ue.$$.fragment,e),y(ce.$$.fragment,e),tt=!0)},o(e){c(m.$$.fragment,e),c(q.$$.fragment,e),c(k.$$.fragment,e),c(j.$$.fragment,e),c(x.$$.fragment,e),c(E.$$.fragment,e),c(I.$$.fragment,e),c(F.$$.fragment,e),c(Y.$$.fragment,e),c(A.$$.fragment,e),c(S.$$.fragment,e),c(P.$$.fragment,e),c(K.$$.fragment,e),c(G.$$.fragment,e),c(ee.$$.fragment,e),c(le.$$.fragment,e),c(se.$$.fragment,e),c(ie.$$.fragment,e),c(oe.$$.fragment,e),c(me.$$.fragment,e),c(fe.$$.fragment,e),c(de.$$.fragment,e),c(ue.$$.fragment,e),c(ce.$$.fragment,e),tt=!1},d(e){e&&(l(T),l(r),l(f),l(M),l(_),l(U),l(g),l(B),l(C),l(w),l(he),l(V),l(Je),l($e),l(Te),l(Q),l(we),l(Ue),l(X),l(Ze),l(ge),l(Ce),l(_e),l(N),l(We),l(ve),l(L),l(Re),l(H),l(Be),l(ze),l(ke),l(D),l(je),l(Fe),l(Ge),l(qe),l(O),l(Ve),l(Qe),l(te),l(xe),l(Xe),l(Ee),l(ne),l(Ie),l(Ne),l(ae),l(Ye),l(Le),l(re),l(He),l(Ae),l(Se),l(pe),l(De),l(Pe),l(Ke),l(ye),l(Oe),l(et),l(be)),l(i),b(m,e),b(q,e),b(k,e),b(j,e),b(x,e),b(E,e),b(I,e),b(F,e),b(Y,e),b(A,e),b(S,e),b(P,e),b(K,e),b(G,e),b(ee,e),b(le,e),b(se,e),b(ie,e),b(oe,e),b(me,e),b(fe,e),b(de,e),b(ue,e),b(ce,e)}}}const Qt='{"title":"bitsandbytes","local":"bitsandbytes","sections":[{"title":"8-bit (LLM.int8() algorithm)","local":"8-bit-llmint8-algorithm","sections":[{"title":"Outlier threshold","local":"outlier-threshold","sections":[],"depth":3},{"title":"Skip module conversion","local":"skip-module-conversion","sections":[],"depth":3}],"depth":2},{"title":"4-bit (QLoRA algorithm)","local":"4-bit-qlora-algorithm","sections":[{"title":"Compute data type","local":"compute-data-type","sections":[],"depth":3},{"title":"Normal Float 4 (NF4)","local":"normal-float-4-nf4","sections":[],"depth":3},{"title":"Nested quantization","local":"nested-quantization","sections":[],"depth":3}],"depth":2},{"title":"Dequantizing bitsandbytes models","local":"dequantizing-bitsandbytes-models","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function xt(v){return gt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ht extends Ct{constructor(i){super(),_t(this,i,xt,Vt,Zt,{})}}export{Ht as component};