diffusers/main/en/_app/pages/optimization/memory.mdx-hf-doc-builder.js

import{S as ep,i as tp,s as lp,e as s,k as p,w as b,t as o,M as sp,c as a,d as l,m as f,a as n,x as U,h as r,b as h,G as t,g as m,y as _,q as v,o as j,B as E,v as ap}from"../../chunks/vendor-hf-doc-builder.js";import{T as Yt}from"../../chunks/Tip-hf-doc-builder.js";import{I as ke}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Q}from"../../chunks/CodeBlock-hf-doc-builder.js";function np(W){let d,w,c,u,Z;return{c(){d=s("p"),w=o("In many cases, optimizing for memory or speed leads to improved performance in the other, so you should try to optimize for both whenever you can. This guide focuses on minimizing memory usage, but you can also learn more about how to "),c=s("a"),u=o("Speed up inference"),Z=o("."),this.h()},l(y){d=a(y,"P",{});var M=n(d);w=r(M,"In many cases, optimizing for memory or speed leads to improved performance in the other, so you should try to optimize for both whenever you can. This guide focuses on minimizing memory usage, but you can also learn more about how to "),c=a(M,"A",{href:!0});var I=n(c);u=r(I,"Speed up inference"),I.forEach(l),Z=r(M,"."),M.forEach(l),this.h()},h(){h(c,"href","fp16")},m(y,M){m(y,d,M),t(d,w),t(d,c),t(c,u),t(d,Z)},d(y){y&&l(d)}}}function op(W){let d,w,c,u,Z;return{c(){d=s("p"),w=o("Consider using "),c=s("a"),u=o("model offloading"),Z=o(" if you want to optimize for speed because it is much faster. The tradeoff is your memory savings won\u2019t be as large."),this.h()},l(y){d=a(y,"P",{});var M=n(d);w=r(M,"Consider using "),c=a(M,"A",{href:!0});var I=n(c);u=r(I,"model offloading"),I.forEach(l),Z=r(M," if you want to optimize for speed because it is much faster. The tradeoff is your memory savings won\u2019t be as large."),M.forEach(l),this.h()},h(){h(c,"href","#model-offloading")},m(y,M){m(y,d,M),t(d,w),t(d,c),t(c,u),t(d,Z)},d(y){y&&l(d)}}}function rp(W){let d,w,c,u,Z,y,M,I,k,B,J,G,R;return{c(){d=s("p"),w=o("When using "),c=s("a"),u=o("enable_sequential_cpu_offload()"),Z=o(", don\u2019t move the pipeline to CUDA beforehand or else the gain in memory consumption will only be minimal (see this "),y=s("a"),M=o("issue"),I=o(" for more information)."),k=p(),B=s("p"),J=s("a"),G=o("enable_sequential_cpu_offload()"),R=o(" is a stateful operation that installs hooks on the models."),this.h()},l(T){d=a(T,"P",{});var g=n(d);w=r(g,"When using "),c=a(g,"A",{href:!0});var z=n(c);u=r(z,"enable_sequential_cpu_offload()"),z.forEach(l),Z=r(g,", don\u2019t move the pipeline to CUDA beforehand or else the gain in memory consumption will only be minimal (see this "),y=a(g,"A",{href:!0,rel:!0});var at=n(y);M=r(at,"issue"),at.forEach(l),I=r(g," for more information)."),g.forEach(l),k=f(T),B=a(T,"P",{});var de=n(B);J=a(de,"A",{href:!0});var S=n(J);G=r(S,"enable_sequential_cpu_offload()"),S.forEach(l),R=r(de," is a stateful operation that installs hooks on the models."),de.forEach(l),this.h()},h(){h(c,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/latent_upscale#diffusers.StableDiffusionLatentUpscalePipeline.enable_sequential_cpu_offload"),h(y,"href","https://github.com/huggingface/diffusers/issues/1934"),h(y,"rel","nofollow"),h(J,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/latent_upscale#diffusers.StableDiffusionLatentUpscalePipeline.enable_sequential_cpu_offload")},m(T,g){m(T,d,g),t(d,w),t(d,c),t(c,u),t(d,Z),t(d,y),t(y,M),t(d,I),m(T,k,g),m(T,B,g),t(B,J),t(J,G),t(B,R)},d(T){T&&l(d),T&&l(k),T&&l(B)}}}function ip(W){let d,w;return{c(){d=s("p"),w=o("Model offloading requires \u{1F917} Accelerate version 0.17.0 or higher.")},l(c){d=a(c,"P",{});var u=n(d);w=r(u,"Model offloading requires \u{1F917} Accelerate version 0.17.0 or higher."),u.forEach(l)},m(c,u){m(c,d,u),t(d,w)},d(c){c&&l(d)}}}function pp(W){let d,w,c,u,Z,y,M,I,k,B;return{c(){d=s("p"),w=o("In order to properly offload models after they\u2019re called, it is required to run the entire pipeline and models are called in the pipeline\u2019s expected order. Exercise caution if models are reused outside the context of the pipeline after hooks have been installed. See "),c=s("a"),u=o("Removing Hooks"),Z=o(`
for more information.`),y=p(),M=s("p"),I=s("a"),k=o("enable_model_cpu_offload()"),B=o(" is a stateful operation that installs hooks on the models and state on the pipeline."),this.h()},l(J){d=a(J,"P",{});var G=n(d);w=r(G,"In order to properly offload models after they\u2019re called, it is required to run the entire pipeline and models are called in the pipeline\u2019s expected order. Exercise caution if models are reused outside the context of the pipeline after hooks have been installed. See "),c=a(G,"A",{href:!0,rel:!0});var R=n(c);u=r(R,"Removing Hooks"),R.forEach(l),Z=r(G,`
for more information.`),G.forEach(l),y=f(J),M=a(J,"P",{});var T=n(M);I=a(T,"A",{href:!0});var g=n(I);k=r(g,"enable_model_cpu_offload()"),g.forEach(l),B=r(T," is a stateful operation that installs hooks on the models and state on the pipeline."),T.forEach(l),this.h()},h(){h(c,"href","https://huggingface.co/docs/accelerate/en/package_reference/big_modeling#accelerate.hooks.remove_hook_from_module"),h(c,"rel","nofollow"),h(I,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/gligen#diffusers.StableDiffusionGLIGENTextImagePipeline.enable_model_cpu_offload")},m(J,G){m(J,d,G),t(d,w),t(d,c),t(c,u),t(d,Z),m(J,y,G),m(J,M,G),t(M,I),t(I,k),t(M,B)},d(J){J&&l(d),J&&l(y),J&&l(M)}}}function fp(W){let d,w;return{c(){d=s("p"),w=o("If you have PyTorch 2.0 installed, you shouldn\u2019t use xFormers!")},l(c){d=a(c,"P",{});var u=n(d);w=r(u,"If you have PyTorch 2.0 installed, you shouldn\u2019t use xFormers!"),u.forEach(l)},m(c,u){m(c,d,u),t(d,w)},d(c){c&&l(d)}}}function mp(W){let d,w,c,u,Z,y,M,I,k,B,J,G,R,T,g,z,at,de,S,Qt,x,ns,Ta,zt,ba,Ua,xt,_a,va,$,H,Ht,ja,Ea,Dt,Ia,Za,Ft,ga,Ca,D,Pt,Ba,Ga,qt,Wa,Va,Lt,$a,ka,F,Kt,Ra,Sa,Ot,Na,Aa,el,Xa,Ya,P,tl,Qa,za,ll,xa,Ha,sl,Da,Fa,q,al,Pa,qa,nl,La,Ka,ol,Oa,os,L,he,rl,Re,en,il,tn,rs,ce,ln,nt,sn,an,is,ue,nn,ot,on,rn,ps,Se,fs,rt,pn,ms,K,ye,pl,Ne,fn,fl,mn,ds,Me,dn,it,hn,cn,hs,we,un,pt,yn,Mn,cs,Ae,us,ft,wn,ys,O,Je,ml,Xe,Jn,dl,Tn,Ms,mt,bn,ws,Te,Un,dt,_n,vn,Js,Ye,Ts,be,jn,hl,En,In,bs,Ue,Us,ht,Zn,_s,Qe,vs,_e,js,ee,ve,cl,ze,gn,ul,Cn,Es,je,Is,xe,ct,Bn,Gn,Zs,N,Wn,yl,Vn,$n,Ml,kn,Rn,gs,ut,Sn,Cs,Ee,Nn,yt,An,Xn,Bs,He,Gs,Mt,Yn,Ws,De,Vs,Ie,$s,te,Ze,wl,Fe,Qn,Jl,zn,ks,wt,xn,Rs,Jt,Hn,Ss,Pe,Ns,le,ge,Tl,qe,Dn,bl,Fn,As,Ce,Pn,Ul,qn,Ln,Xs,Tt,Kn,Ys,Le,Qs,Be,On,_l,eo,to,zs,Ke,xs,se,Ge,vl,Oe,lo,jl,so,Hs,A,ao,et,no,oo,tt,ro,io,Ds,bt,po,Fs,We,El,ae,Il,fo,mo,Zl,ho,co,gl,uo,yo,C,ne,Cl,Mo,wo,Bl,Jo,To,Gl,bo,Uo,oe,Wl,_o,vo,Vl,jo,Eo,$l,Io,Zo,re,kl,go,Co,Rl,Bo,Go,Sl,Wo,Vo,ie,Nl,$o,ko,Al,Ro,So,Xl,No,Ao,pe,Yl,Xo,Yo,Ql,Qo,zo,zl,xo,Ho,fe,xl,Do,Fo,Hl,Po,qo,Dl,Lo,Ko,me,Fl,Oo,er,Pl,tr,lr,ql,sr,Ps,Ve,qs,Ut,ar,Ls,X,Ll,nr,or,Kl,rr,ir,Ol,_t,pr,Ks,$e,fr,vt,mr,dr,Os,lt,ea;return y=new ke({}),T=new Yt({props:{$$slots:{default:[np]},$$scope:{ctx:W}}}),Re=new ke({}),Se=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEFwaXBlJTIwJTNEJTIwcGlwZS50byglMjJjdWRhJTIyKSUwQSUwQXByb21wdCUyMCUzRCUyMCUyMmElMjBwaG90byUyMG9mJTIwYW4lMjBhc3Ryb25hdXQlMjByaWRpbmclMjBhJTIwaG9yc2UlMjBvbiUyMG1hcnMlMjIlMEFwaXBlLmVuYWJsZV92YWVfc2xpY2luZygpJTBBaW1hZ2VzJTIwJTNEJTIwcGlwZSglNUJwcm9tcHQlNUQlMjAqJTIwMzIpLmltYWdlcw==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_vae_slicing()
images = pipe([prompt] * <span class="hljs-number">32</span>).images`}}),Ne=new ke({}),Ae=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMkMlMjBVbmlQQ011bHRpc3RlcFNjaGVkdWxlciUwQSUwQXBpcGUlMjAlM0QlMjBTdGFibGVEaWZmdXNpb25QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIycnVud2F5bWwlMkZzdGFibGUtZGlmZnVzaW9uLXYxLTUlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMkMlMEElMjAlMjAlMjAlMjB1c2Vfc2FmZXRlbnNvcnMlM0RUcnVlJTJDJTBBKSUwQXBpcGUuc2NoZWR1bGVyJTIwJTNEJTIwVW5pUENNdWx0aXN0ZXBTY2hlZHVsZXIuZnJvbV9jb25maWcocGlwZS5zY2hlZHVsZXIuY29uZmlnKSUwQXBpcGUlMjAlM0QlMjBwaXBlLnRvKCUyMmN1ZGElMjIpJTBBcHJvbXB0JTIwJTNEJTIwJTIyYSUyMGJlYXV0aWZ1bCUyMGxhbmRzY2FwZSUyMHBob3RvZ3JhcGglMjIlMEFwaXBlLmVuYWJsZV92YWVfdGlsaW5nKCklMEFwaXBlLmVuYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbigpJTBBJTBBaW1hZ2UlMjAlM0QlMjBwaXBlKCU1QnByb21wdCU1RCUyQyUyMHdpZHRoJTNEMzg0MCUyQyUyMGhlaWdodCUzRDIyMjQlMkMlMjBudW1faW5mZXJlbmNlX3N0ZXBzJTNEMjApLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline, UniPCMultistepScheduler

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)
prompt = <span class="hljs-string">&quot;a beautiful landscape photograph&quot;</span>
pipe.enable_vae_tiling()
pipe.enable_xformers_memory_efficient_attention()

image = pipe([prompt], width=<span class="hljs-number">3840</span>, height=<span class="hljs-number">2224</span>, num_inference_steps=<span class="hljs-number">20</span>).images[<span class="hljs-number">0</span>]`}}),Xe=new ke({}),Ye=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJhJTIwcGhvdG8lMjBvZiUyMGFuJTIwYXN0cm9uYXV0JTIwcmlkaW5nJTIwYSUyMGhvcnNlJTIwb24lMjBtYXJzJTIyJTBBcGlwZS5lbmFibGVfc2VxdWVudGlhbF9jcHVfb2ZmbG9hZCgpJTBBaW1hZ2UlMjAlM0QlMjBwaXBlKHByb21wdCkuaW1hZ2VzJTVCMCU1RA==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_sequential_cpu_offload()
image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),Ue=new Yt({props:{$$slots:{default:[op]},$$scope:{ctx:W}}}),Qe=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJhJTIwcGhvdG8lMjBvZiUyMGFuJTIwYXN0cm9uYXV0JTIwcmlkaW5nJTIwYSUyMGhvcnNlJTIwb24lMjBtYXJzJTIyJTBBcGlwZS5lbmFibGVfc2VxdWVudGlhbF9jcHVfb2ZmbG9hZCgpJTBBJTBBaW1hZ2UlMjAlM0QlMjBwaXBlKHByb21wdCkuaW1hZ2VzJTVCMCU1RA==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_sequential_cpu_offload()

image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),_e=new Yt({props:{warning:!0,$$slots:{default:[rp]},$$scope:{ctx:W}}}),ze=new ke({}),je=new Yt({props:{$$slots:{default:[ip]},$$scope:{ctx:W}}}),He=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTIwJTIwJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJhJTIwcGhvdG8lMjBvZiUyMGFuJTIwYXN0cm9uYXV0JTIwcmlkaW5nJTIwYSUyMGhvcnNlJTIwb24lMjBtYXJzJTIyJTBBcGlwZS5lbmFibGVfbW9kZWxfY3B1X29mZmxvYWQoKSUwQWltYWdlJTIwJTNEJTIwcGlwZShwcm9tcHQpLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,  
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_model_cpu_offload()
image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),De=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjJhJTIwcGhvdG8lMjBvZiUyMGFuJTIwYXN0cm9uYXV0JTIwcmlkaW5nJTIwYSUyMGhvcnNlJTIwb24lMjBtYXJzJTIyJTBBcGlwZS5lbmFibGVfbW9kZWxfY3B1X29mZmxvYWQoKSUwQSUwQWltYWdlJTIwJTNEJTIwcGlwZShwcm9tcHQpLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline

pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
)

prompt = <span class="hljs-string">&quot;a photo of an astronaut riding a horse on mars&quot;</span>
pipe.enable_model_cpu_offload()

image = pipe(prompt).images[<span class="hljs-number">0</span>]`}}),Ie=new Yt({props:{warning:!0,$$slots:{default:[pp]},$$scope:{ctx:W}}}),Fe=new ke({}),Pe=new Q({props:{code:"cHJpbnQocGlwZS51bmV0LmNvbnZfb3V0LnN0YXRlX2RpY3QoKSU1QiUyMndlaWdodCUyMiU1RC5zdHJpZGUoKSklMjAlMjAlMjMlMjAoMjg4MCUyQyUyMDklMkMlMjAzJTJDJTIwMSklMEFwaXBlLnVuZXQudG8obWVtb3J5X2Zvcm1hdCUzRHRvcmNoLmNoYW5uZWxzX2xhc3QpJTIwJTIwJTIzJTIwaW4tcGxhY2UlMjBvcGVyYXRpb24lMEFwcmludCglMEElMjAlMjAlMjAlMjBwaXBlLnVuZXQuY29udl9vdXQuc3RhdGVfZGljdCgpJTVCJTIyd2VpZ2h0JTIyJTVELnN0cmlkZSgpJTBBKSUyMCUyMCUyMyUyMCgyODgwJTJDJTIwMSUyQyUyMDk2MCUyQyUyMDMyMCklMjBoYXZpbmclMjBhJTIwc3RyaWRlJTIwb2YlMjAxJTIwZm9yJTIwdGhlJTIwMm5kJTIwZGltZW5zaW9uJTIwcHJvdmVzJTIwdGhhdCUyMGl0JTIwd29ya3M=",highlighted:`<span class="hljs-built_in">print</span>(pipe.unet.conv_out.state_dict()[<span class="hljs-string">&quot;weight&quot;</span>].stride())  <span class="hljs-comment"># (2880, 9, 3, 1)</span>
pipe.unet.to(memory_format=torch.channels_last)  <span class="hljs-comment"># in-place operation</span>
<span class="hljs-built_in">print</span>(
    pipe.unet.conv_out.state_dict()[<span class="hljs-string">&quot;weight&quot;</span>].stride()
)  <span class="hljs-comment"># (2880, 1, 960, 320) having a stride of 1 for the 2nd dimension proves that it works</span>`}}),qe=new ke({}),Le=new Q({props:{code:"aW1wb3J0JTIwdGltZSUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBaW1wb3J0JTIwZnVuY3Rvb2xzJTBBJTBBJTIzJTIwdG9yY2glMjBkaXNhYmxlJTIwZ3JhZCUwQXRvcmNoLnNldF9ncmFkX2VuYWJsZWQoRmFsc2UpJTBBJTBBJTIzJTIwc2V0JTIwdmFyaWFibGVzJTBBbl9leHBlcmltZW50cyUyMCUzRCUyMDIlMEF1bmV0X3J1bnNfcGVyX2V4cGVyaW1lbnQlMjAlM0QlMjA1MCUwQSUwQSUwQSUyMyUyMGxvYWQlMjBpbnB1dHMlMEFkZWYlMjBnZW5lcmF0ZV9pbnB1dHMoKSUzQSUwQSUyMCUyMCUyMCUyMHNhbXBsZSUyMCUzRCUyMHRvcmNoLnJhbmRuKDIlMkMlMjA0JTJDJTIwNjQlMkMlMjA2NCkuaGFsZigpLmN1ZGEoKSUwQSUyMCUyMCUyMCUyMHRpbWVzdGVwJTIwJTNEJTIwdG9yY2gucmFuZCgxKS5oYWxmKCkuY3VkYSgpJTIwKiUyMDk5OSUwQSUyMCUyMCUyMCUyMGVuY29kZXJfaGlkZGVuX3N0YXRlcyUyMCUzRCUyMHRvcmNoLnJhbmRuKDIlMkMlMjA3NyUyQyUyMDc2OCkuaGFsZigpLmN1ZGEoKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHNhbXBsZSUyQyUyMHRpbWVzdGVwJTJDJTIwZW5jb2Rlcl9oaWRkZW5fc3RhdGVzJTBBJTBBJTBBcGlwZSUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJydW53YXltbCUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMHVzZV9zYWZldGVuc29ycyUzRFRydWUlMkMlMEEpLnRvKCUyMmN1ZGElMjIpJTBBdW5ldCUyMCUzRCUyMHBpcGUudW5ldCUwQXVuZXQuZXZhbCgpJTBBdW5ldC50byhtZW1vcnlfZm9ybWF0JTNEdG9yY2guY2hhbm5lbHNfbGFzdCklMjAlMjAlMjMlMjB1c2UlMjBjaGFubmVsc19sYXN0JTIwbWVtb3J5JTIwZm9ybWF0JTBBdW5ldC5mb3J3YXJkJTIwJTNEJTIwZnVuY3Rvb2xzLnBhcnRpYWwodW5ldC5mb3J3YXJkJTJDJTIwcmV0dXJuX2RpY3QlM0RGYWxzZSklMjAlMjAlMjMlMjBzZXQlMjByZXR1cm5fZGljdCUzREZhbHNlJTIwYXMlMjBkZWZhdWx0JTBBJTBBJTIzJTIwd2FybXVwJTBBZm9yJTIwXyUyMGluJTIwcmFuZ2UoMyklM0ElMEElMjAlMjAlMjAlMjB3aXRoJTIwdG9yY2guaW5mZXJlbmNlX21vZGUoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMGdlbmVyYXRlX2lucHV0cygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3JpZ19vdXRwdXQlMjAlM0QlMjB1bmV0KCppbnB1dHMpJTBBJTBBJTIzJTIwdHJhY2UlMEFwcmludCglMjJ0cmFjaW5nLi4lMjIpJTBBdW5ldF90cmFjZWQlMjAlM0QlMjB0b3JjaC5qaXQudHJhY2UodW5ldCUyQyUyMGlucHV0cyklMEF1bmV0X3RyYWNlZC5ldmFsKCklMEFwcmludCglMjJkb25lJTIwdHJhY2luZyUyMiklMEElMEElMEElMjMlMjB3YXJtdXAlMjBhbmQlMjBvcHRpbWl6ZSUyMGdyYXBoJTBBZm9yJTIwXyUyMGluJTIwcmFuZ2UoNSklM0ElMEElMjAlMjAlMjAlMjB3aXRoJTIwdG9yY2guaW5mZXJlbmNlX21vZGUoKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMGdlbmVyYXRlX2lucHV0cygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3JpZ19vdXRwdXQlMjAlM0QlMjB1bmV0X3RyYWNlZCgqaW5wdXRzKSUwQSUwQSUwQSUyMyUyMGJlbmNobWFya2luZyUwQXdpdGglMjB0b3JjaC5pbmZlcmVuY2VfbW9kZSgpJTNBJTBBJTIwJTIwJTIwJTIwZm9yJTIwXyUyMGluJTIwcmFuZ2Uobl9leHBlcmltZW50cyklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3JjaC5jdWRhLnN5bmNocm9uaXplKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdGFydF90aW1lJTIwJTNEJTIwdGltZS50aW1lKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBfJTIwaW4lMjByYW5nZSh1bmV0X3J1bnNfcGVyX2V4cGVyaW1lbnQpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3JpZ19vdXRwdXQlMjAlM0QlMjB1bmV0X3RyYWNlZCgqaW5wdXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcmNoLmN1ZGEuc3luY2hyb25pemUoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGYlMjJ1bmV0JTIwdHJhY2VkJTIwaW5mZXJlbmNlJTIwdG9vayUyMCU3QnRpbWUudGltZSgpJTIwLSUyMHN0YXJ0X3RpbWUlM0EuMmYlN0QlMjBzZWNvbmRzJTIyKSUwQSUyMCUyMCUyMCUyMGZvciUyMF8lMjBpbiUyMHJhbmdlKG5fZXhwZXJpbWVudHMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdG9yY2guY3VkYS5zeW5jaHJvbml6ZSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RhcnRfdGltZSUyMCUzRCUyMHRpbWUudGltZSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwXyUyMGluJTIwcmFuZ2UodW5ldF9ydW5zX3Blcl9leHBlcmltZW50KSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9yaWdfb3V0cHV0JTIwJTNEJTIwdW5ldCgqaW5wdXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcmNoLmN1ZGEuc3luY2hyb25pemUoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGYlMjJ1bmV0JTIwaW5mZXJlbmNlJTIwdG9vayUyMCU3QnRpbWUudGltZSgpJTIwLSUyMHN0YXJ0X3RpbWUlM0EuMmYlN0QlMjBzZWNvbmRzJTIyKSUwQSUwQSUyMyUyMHNhdmUlMjB0aGUlMjBtb2RlbCUwQXVuZXRfdHJhY2VkLnNhdmUoJTIydW5ldF90cmFjZWQucHQlMjIp",highlighted:`<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> functools

<span class="hljs-comment"># torch disable grad</span>
torch.set_grad_enabled(<span class="hljs-literal">False</span>)

<span class="hljs-comment"># set variables</span>
n_experiments = <span class="hljs-number">2</span>
unet_runs_per_experiment = <span class="hljs-number">50</span>


<span class="hljs-comment"># load inputs</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_inputs</span>():
    sample = torch.randn(<span class="hljs-number">2</span>, <span class="hljs-number">4</span>, <span class="hljs-number">64</span>, <span class="hljs-number">64</span>).half().cuda()
    timestep = torch.rand(<span class="hljs-number">1</span>).half().cuda() * <span class="hljs-number">999</span>
    encoder_hidden_states = torch.randn(<span class="hljs-number">2</span>, <span class="hljs-number">77</span>, <span class="hljs-number">768</span>).half().cuda()
    <span class="hljs-keyword">return</span> sample, timestep, encoder_hidden_states


pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
unet = pipe.unet
unet.<span class="hljs-built_in">eval</span>()
unet.to(memory_format=torch.channels_last)  <span class="hljs-comment"># use channels_last memory format</span>
unet.forward = functools.partial(unet.forward, return_dict=<span class="hljs-literal">False</span>)  <span class="hljs-comment"># set return_dict=False as default</span>

<span class="hljs-comment"># warmup</span>
<span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>):
    <span class="hljs-keyword">with</span> torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet(*inputs)

<span class="hljs-comment"># trace</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;tracing..&quot;</span>)
unet_traced = torch.jit.trace(unet, inputs)
unet_traced.<span class="hljs-built_in">eval</span>()
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;done tracing&quot;</span>)


<span class="hljs-comment"># warmup and optimize graph</span>
<span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>):
    <span class="hljs-keyword">with</span> torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet_traced(*inputs)


<span class="hljs-comment"># benchmarking</span>
<span class="hljs-keyword">with</span> torch.inference_mode():
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(unet_runs_per_experiment):
            orig_output = unet_traced(*inputs)
        torch.cuda.synchronize()
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;unet traced inference took <span class="hljs-subst">{time.time() - start_time:<span class="hljs-number">.2</span>f}</span> seconds&quot;</span>)
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(unet_runs_per_experiment):
            orig_output = unet(*inputs)
        torch.cuda.synchronize()
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;unet inference took <span class="hljs-subst">{time.time() - start_time:<span class="hljs-number">.2</span>f}</span> seconds&quot;</span>)

<span class="hljs-comment"># save the model</span>
unet_traced.save(<span class="hljs-string">&quot;unet_traced.pt&quot;</span>)`}}),Ke=new Q({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGF0YWNsYXNzZXMlMjBpbXBvcnQlMjBkYXRhY2xhc3MlMEElMEElMEElNDBkYXRhY2xhc3MlMEFjbGFzcyUyMFVOZXQyRENvbmRpdGlvbk91dHB1dCUzQSUwQSUyMCUyMCUyMCUyMHNhbXBsZSUzQSUyMHRvcmNoLkZsb2F0VGVuc29yJTBBJTBBJTBBcGlwZSUyMCUzRCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJydW53YXltbCUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMHVzZV9zYWZldGVuc29ycyUzRFRydWUlMkMlMEEpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTIzJTIwdXNlJTIwaml0dGVkJTIwdW5ldCUwQXVuZXRfdHJhY2VkJTIwJTNEJTIwdG9yY2guaml0LmxvYWQoJTIydW5ldF90cmFjZWQucHQlMjIpJTBBJTBBJTBBJTIzJTIwZGVsJTIwcGlwZS51bmV0JTBBY2xhc3MlMjBUcmFjZWRVTmV0KHRvcmNoLm5uLk1vZHVsZSklM0ElMEElMjAlMjAlMjAlMjBkZWYlMjBfX2luaXRfXyhzZWxmKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN1cGVyKCkuX19pbml0X18oKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuaW5fY2hhbm5lbHMlMjAlM0QlMjBwaXBlLnVuZXQuaW5fY2hhbm5lbHMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmRldmljZSUyMCUzRCUyMHBpcGUudW5ldC5kZXZpY2UlMEElMEElMjAlMjAlMjAlMjBkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBsYXRlbnRfbW9kZWxfaW5wdXQlMkMlMjB0JTJDJTIwZW5jb2Rlcl9oaWRkZW5fc3RhdGVzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNhbXBsZSUyMCUzRCUyMHVuZXRfdHJhY2VkKGxhdGVudF9tb2RlbF9pbnB1dCUyQyUyMHQlMkMlMjBlbmNvZGVyX2hpZGRlbl9zdGF0ZXMpJTVCMCU1RCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMFVOZXQyRENvbmRpdGlvbk91dHB1dChzYW1wbGUlM0RzYW1wbGUpJTBBJTBBJTBBcGlwZS51bmV0JTIwJTNEJTIwVHJhY2VkVU5ldCgpJTBBJTBBd2l0aCUyMHRvcmNoLmluZmVyZW5jZV9tb2RlKCklM0ElMEElMjAlMjAlMjAlMjBpbWFnZSUyMCUzRCUyMHBpcGUoJTVCcHJvbXB0JTVEJTIwKiUyMDElMkMlMjBudW1faW5mZXJlbmNlX3N0ZXBzJTNENTApLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> dataclasses <span class="hljs-keyword">import</span> dataclass


<span class="hljs-meta">@dataclass</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">UNet2DConditionOutput</span>:
    sample: torch.FloatTensor


pipe = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># use jitted unet</span>
unet_traced = torch.jit.load(<span class="hljs-string">&quot;unet_traced.pt&quot;</span>)


<span class="hljs-comment"># del pipe.unet</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">TracedUNet</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.in_channels = pipe.unet.in_channels
        self.device = pipe.unet.device

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, latent_model_input, t, encoder_hidden_states</span>):
        sample = unet_traced(latent_model_input, t, encoder_hidden_states)[<span class="hljs-number">0</span>]
        <span class="hljs-keyword">return</span> UNet2DConditionOutput(sample=sample)


pipe.unet = TracedUNet()

<span class="hljs-keyword">with</span> torch.inference_mode():
    image = pipe([prompt] * <span class="hljs-number">1</span>, num_inference_steps=<span class="hljs-number">50</span>).images[<span class="hljs-number">0</span>]`}}),Oe=new ke({}),Ve=new Yt({props:{warning:!0,$$slots:{default:[fp]},$$scope:{ctx:W}}}),lt=new Q({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMERpZmZ1c2lvblBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEElMEFwaXBlJTIwJTNEJTIwRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnJ1bndheW1sJTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwdXNlX3NhZmV0ZW5zb3JzJTNEVHJ1ZSUyQyUwQSkudG8oJTIyY3VkYSUyMiklMEElMEFwaXBlLmVuYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbigpJTBBJTBBd2l0aCUyMHRvcmNoLmluZmVyZW5jZV9tb2RlKCklM0ElMEElMjAlMjAlMjAlMjBzYW1wbGUlMjAlM0QlMjBwaXBlKCUyMmElMjBzbWFsbCUyMGNhdCUyMiklMEElMEElMjMlMjBvcHRpb25hbCUzQSUyMFlvdSUyMGNhbiUyMGRpc2FibGUlMjBpdCUyMHZpYSUwQSUyMyUyMHBpcGUuZGlzYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbigp",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline
<span class="hljs-keyword">import</span> torch

pipe = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;runwayml/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

pipe.enable_xformers_memory_efficient_attention()

<span class="hljs-keyword">with</span> torch.inference_mode():
    sample = pipe(<span class="hljs-string">&quot;a small cat&quot;</span>)

<span class="hljs-comment"># optional: You can disable it via</span>
<span class="hljs-comment"># pipe.disable_xformers_memory_efficient_attention()</span>`}}),{c(){d=s("meta"),w=p(),c=s("h1"),u=s("a"),Z=s("span"),b(y.$$.fragment),M=p(),I=s("span"),k=o("Reduce memory usage"),B=p(),J=s("p"),G=o("A barrier to using diffusion models is the large amount of memory required. To overcome this challenge, there are several memory-reducing techniques you can use to run even some of the largest models on free-tier or consumer GPUs. Some of these techniques can even be combined to further reduce memory usage."),R=p(),b(T.$$.fragment),g=p(),z=s("p"),at=o("The results below are obtained from generating a single 512x512 image from the prompt a photo of an astronaut riding a horse on mars with 50 DDIM steps on a Nvidia Titan RTX, demonstrating the speed-up you can expect as a result of reduced memory consumption."),de=p(),S=s("table"),Qt=s("thead"),x=s("tr"),ns=s("th"),Ta=p(),zt=s("th"),ba=o("latency"),Ua=p(),xt=s("th"),_a=o("speed-up"),va=p(),$=s("tbody"),H=s("tr"),Ht=s("td"),ja=o("original"),Ea=p(),Dt=s("td"),Ia=o("9.50s"),Za=p(),Ft=s("td"),ga=o("x1"),Ca=p(),D=s("tr"),Pt=s("td"),Ba=o("fp16"),Ga=p(),qt=s("td"),Wa=o("3.61s"),Va=p(),Lt=s("td"),$a=o("x2.63"),ka=p(),F=s("tr"),Kt=s("td"),Ra=o("channels last"),Sa=p(),Ot=s("td"),Na=o("3.30s"),Aa=p(),el=s("td"),Xa=o("x2.88"),Ya=p(),P=s("tr"),tl=s("td"),Qa=o("traced UNet"),za=p(),ll=s("td"),xa=o("3.21s"),Ha=p(),sl=s("td"),Da=o("x2.96"),Fa=p(),q=s("tr"),al=s("td"),Pa=o("memory-efficient attention"),qa=p(),nl=s("td"),La=o("2.63s"),Ka=p(),ol=s("td"),Oa=o("x3.61"),os=p(),L=s("h2"),he=s("a"),rl=s("span"),b(Re.$$.fragment),en=p(),il=s("span"),tn=o("Sliced VAE"),rs=p(),ce=s("p"),ln=o("Sliced VAE enables decoding large batches of images with limited VRAM or batches with 32 images or more by decoding the batches of latents one image at a time. You\u2019ll likely want to couple this with "),nt=s("a"),sn=o("enable_xformers_memory_efficient_attention()"),an=o(" to further reduce memory use."),is=p(),ue=s("p"),nn=o("To use sliced VAE, call "),ot=s("a"),on=o("enable_vae_slicing()"),rn=o(" on your pipeline before inference:"),ps=p(),b(Se.$$.fragment),fs=p(),rt=s("p"),pn=o("You may see a small performance boost in VAE decoding on multi-image batches, and there should be no performance impact on single-image batches."),ms=p(),K=s("h2"),ye=s("a"),pl=s("span"),b(Ne.$$.fragment),fn=p(),fl=s("span"),mn=o("Tiled VAE"),ds=p(),Me=s("p"),dn=o("Tiled VAE processing also enables working with large images on limited VRAM (for example, generating 4k images on 8GB of VRAM) by splitting the image into overlapping tiles, decoding the tiles, and then blending the outputs together to compose the final image. You should also used tiled VAE with "),it=s("a"),hn=o("enable_xformers_memory_efficient_attention()"),cn=o(" to further reduce memory use."),hs=p(),we=s("p"),un=o("To use tiled VAE processing, call "),pt=s("a"),yn=o("enable_vae_tiling()"),Mn=o(" on your pipeline before inference:"),cs=p(),b(Ae.$$.fragment),us=p(),ft=s("p"),wn=o("The output image has some tile-to-tile tone variation because the tiles are decoded separately, but you shouldn\u2019t see any sharp and obvious seams between the tiles. Tiling is turned off for images that are 512x512 or smaller."),ys=p(),O=s("h2"),Je=s("a"),ml=s("span"),b(Xe.$$.fragment),Jn=p(),dl=s("span"),Tn=o("CPU offloading"),Ms=p(),mt=s("p"),bn=o("Offloading the weights to the CPU and only loading them on the GPU when performing the forward pass can also save memory. Often, this technique can reduce memory consumption to less than 3GB."),ws=p(),Te=s("p"),Un=o("To perform CPU offloading, call "),dt=s("a"),_n=o("enable_sequential_cpu_offload()"),vn=o(":"),Js=p(),b(Ye.$$.fragment),Ts=p(),be=s("p"),jn=o("CPU offloading works on submodules rather than whole models. This is the best way to minimize memory consumption, but inference is much slower due to the iterative nature of the diffusion process. The UNet component of the pipeline runs several times (as many as "),hl=s("code"),En=o("num_inference_steps"),In=o("); each time, the different UNet submodules are sequentially onloaded and offloaded as needed, resulting in a large number of memory transfers."),bs=p(),b(Ue.$$.fragment),Us=p(),ht=s("p"),Zn=o("CPU offloading can also be chained with attention slicing to reduce memory consumption to less than 2GB."),_s=p(),b(Qe.$$.fragment),vs=p(),b(_e.$$.fragment),js=p(),ee=s("h2"),ve=s("a"),cl=s("span"),b(ze.$$.fragment),gn=p(),ul=s("span"),Cn=o("Model offloading"),Es=p(),b(je.$$.fragment),Is=p(),xe=s("p"),ct=s("a"),Bn=o("Sequential CPU offloading"),Gn=o(" preserves a lot of memory but it makes inference slower because submodules are moved to GPU as needed, and they\u2019re immediately returned to the CPU when a new module runs."),Zs=p(),N=s("p"),Wn=o("Full-model offloading is an alternative that moves whole models to the GPU, instead of handling each model\u2019s constituent "),yl=s("em"),Vn=o("submodules"),$n=o(". There is a negligible impact on inference time (compared with moving the pipeline to "),Ml=s("code"),kn=o("cuda"),Rn=o("), and it still provides some memory savings."),gs=p(),ut=s("p"),Sn=o(`During model offloading, only one of the main components of the pipeline (typically the text encoder, UNet and VAE)
is placed on the GPU while the others wait on the CPU. Components like the UNet that run for multiple iterations stay on the GPU until they\u2019re no longer needed.`),Cs=p(),Ee=s("p"),Nn=o("Enable model offloading by calling "),yt=s("a"),An=o("enable_model_cpu_offload()"),Xn=o(" on the pipeline:"),Bs=p(),b(He.$$.fragment),Gs=p(),Mt=s("p"),Yn=o("Model offloading can also be combined with attention slicing for additional memory savings."),Ws=p(),b(De.$$.fragment),Vs=p(),b(Ie.$$.fragment),$s=p(),te=s("h2"),Ze=s("a"),wl=s("span"),b(Fe.$$.fragment),Qn=p(),Jl=s("span"),zn=o("Channels-last memory format"),ks=p(),wt=s("p"),xn=o("The channels-last memory format is an alternative way of ordering NCHW tensors in memory to preserve dimension ordering. Channels-last tensors are ordered in such a way that the channels become the densest dimension (storing images pixel-per-pixel). Since not all operators currently support the channels-last format, it may result in worst performance but you should still try and see if it works for your model."),Rs=p(),Jt=s("p"),Hn=o("For example, to set the pipeline\u2019s UNet to use the channels-last format:"),Ss=p(),b(Pe.$$.fragment),Ns=p(),le=s("h2"),ge=s("a"),Tl=s("span"),b(qe.$$.fragment),Dn=p(),bl=s("span"),Fn=o("Tracing"),As=p(),Ce=s("p"),Pn=o("Tracing runs an example input tensor through the model and captures the operations that are performed on it as that input makes its way through the model\u2019s layers. The executable or "),Ul=s("code"),qn=o("ScriptFunction"),Ln=o(" that is returned is optimized with just-in-time compilation."),Xs=p(),Tt=s("p"),Kn=o("To trace a UNet:"),Ys=p(),b(Le.$$.fragment),Qs=p(),Be=s("p"),On=o("Replace the "),_l=s("code"),eo=o("unet"),to=o(" attribute of the pipeline with the traced model:"),zs=p(),b(Ke.$$.fragment),xs=p(),se=s("h2"),Ge=s("a"),vl=s("span"),b(Oe.$$.fragment),lo=p(),jl=s("span"),so=o("Memory-efficient attention"),Hs=p(),A=s("p"),ao=o("Recent work on optimizing bandwidth in the attention block has generated huge speed-ups and reductions in GPU memory usage. The most recent type of memory-efficient attention is "),et=s("a"),no=o("Flash Attention"),oo=o(" (you can check out the original code at "),tt=s("a"),ro=o("HazyResearch/flash-attention"),io=o(")."),Ds=p(),bt=s("p"),po=o("The table below details the speed-ups from a few different Nvidia GPUs when running inference on image sizes of 512x512 and a batch size of 1 (one prompt):"),Fs=p(),We=s("table"),El=s("thead"),ae=s("tr"),Il=s("th"),fo=o("GPU"),mo=p(),Zl=s("th"),ho=o("base attention (fp16)"),co=p(),gl=s("th"),uo=o("memory-efficient attention (fp16)"),yo=p(),C=s("tbody"),ne=s("tr"),Cl=s("td"),Mo=o("NVIDIA Tesla T4"),wo=p(),Bl=s("td"),Jo=o("3.5it/s"),To=p(),Gl=s("td"),bo=o("5.5it/s"),Uo=p(),oe=s("tr"),Wl=s("td"),_o=o("NVIDIA 3060 RTX"),vo=p(),Vl=s("td"),jo=o("4.6it/s"),Eo=p(),$l=s("td"),Io=o("7.8it/s"),Zo=p(),re=s("tr"),kl=s("td"),go=o("NVIDIA A10G"),Co=p(),Rl=s("td"),Bo=o("8.88it/s"),Go=p(),Sl=s("td"),Wo=o("15.6it/s"),Vo=p(),ie=s("tr"),Nl=s("td"),$o=o("NVIDIA RTX A6000"),ko=p(),Al=s("td"),Ro=o("11.7it/s"),So=p(),Xl=s("td"),No=o("21.09it/s"),Ao=p(),pe=s("tr"),Yl=s("td"),Xo=o("NVIDIA TITAN RTX"),Yo=p(),Ql=s("td"),Qo=o("12.51it/s"),zo=p(),zl=s("td"),xo=o("18.22it/s"),Ho=p(),fe=s("tr"),xl=s("td"),Do=o("A100-SXM4-40GB"),Fo=p(),Hl=s("td"),Po=o("18.6it/s"),qo=p(),Dl=s("td"),Lo=o("29.it/s"),Ko=p(),me=s("tr"),Fl=s("td"),Oo=o("A100-SXM-80GB"),er=p(),Pl=s("td"),tr=o("18.7it/s"),lr=p(),ql=s("td"),sr=o("29.5it/s"),Ps=p(),b(Ve.$$.fragment),qs=p(),Ut=s("p"),ar=o("To use Flash Attention, install the following:"),Ls=p(),X=s("ul"),Ll=s("li"),nr=o("PyTorch > 1.12"),or=p(),Kl=s("li"),rr=o("CUDA available"),ir=p(),Ol=s("li"),_t=s("a"),pr=o("xFormers"),Ks=p(),$e=s("p"),fr=o("Then call "),vt=s("a"),mr=o("enable_xformers_memory_efficient_attention()"),dr=o(" on the pipeline:"),Os=p(),b(lt.$$.fragment),this.h()},l(e){const i=sp('[data-svelte="svelte-1phssyn"]',document.head);d=a(i,"META",{name:!0,content:!0}),i.forEach(l),w=f(e),c=a(e,"H1",{class:!0});var st=n(c);u=a(st,"A",{id:!0,class:!0,href:!0});var es=n(u);Z=a(es,"SPAN",{});var ts=n(Z);U(y.$$.fragment,ts),ts.forEach(l),es.forEach(l),M=f(st),I=a(st,"SPAN",{});var ls=n(I);k=r(ls,"Reduce memory usage"),ls.forEach(l),st.forEach(l),B=f(e),J=a(e,"P",{});var ss=n(J);G=r(ss,"A barrier to using diffusion models is the large amount of memory required. To overcome this challenge, there are several memory-reducing techniques you can use to run even some of the largest models on free-tier or consumer GPUs. Some of these techniques can even be combined to further reduce memory usage."),ss.forEach(l),R=f(e),U(T.$$.fragment,e),g=f(e),z=a(e,"P",{});var as=n(z);at=r(as,"The results below are obtained from generating a single 512x512 image from the prompt a photo of an astronaut riding a horse on mars with 50 DDIM steps on a Nvidia Titan RTX, demonstrating the speed-up you can expect as a result of reduced memory consumption."),as.forEach(l),de=f(e),S=a(e,"TABLE",{});var ta=n(S);Qt=a(ta,"THEAD",{});var cr=n(Qt);x=a(cr,"TR",{});var jt=n(x);ns=a(jt,"TH",{}),n(ns).forEach(l),Ta=f(jt),zt=a(jt,"TH",{});var ur=n(zt);ba=r(ur,"latency"),ur.forEach(l),Ua=f(jt),xt=a(jt,"TH",{});var yr=n(xt);_a=r(yr,"speed-up"),yr.forEach(l),jt.forEach(l),cr.forEach(l),va=f(ta),$=a(ta,"TBODY",{});var Y=n($);H=a(Y,"TR",{});var Et=n(H);Ht=a(Et,"TD",{});var Mr=n(Ht);ja=r(Mr,"original"),Mr.forEach(l),Ea=f(Et),Dt=a(Et,"TD",{});var wr=n(Dt);Ia=r(wr,"9.50s"),wr.forEach(l),Za=f(Et),Ft=a(Et,"TD",{});var Jr=n(Ft);ga=r(Jr,"x1"),Jr.forEach(l),Et.forEach(l),Ca=f(Y),D=a(Y,"TR",{});var It=n(D);Pt=a(It,"TD",{});var Tr=n(Pt);Ba=r(Tr,"fp16"),Tr.forEach(l),Ga=f(It),qt=a(It,"TD",{});var br=n(qt);Wa=r(br,"3.61s"),br.forEach(l),Va=f(It),Lt=a(It,"TD",{});var Ur=n(Lt);$a=r(Ur,"x2.63"),Ur.forEach(l),It.forEach(l),ka=f(Y),F=a(Y,"TR",{});var Zt=n(F);Kt=a(Zt,"TD",{});var _r=n(Kt);Ra=r(_r,"channels last"),_r.forEach(l),Sa=f(Zt),Ot=a(Zt,"TD",{});var vr=n(Ot);Na=r(vr,"3.30s"),vr.forEach(l),Aa=f(Zt),el=a(Zt,"TD",{});var jr=n(el);Xa=r(jr,"x2.88"),jr.forEach(l),Zt.forEach(l),Ya=f(Y),P=a(Y,"TR",{});var gt=n(P);tl=a(gt,"TD",{});var Er=n(tl);Qa=r(Er,"traced UNet"),Er.forEach(l),za=f(gt),ll=a(gt,"TD",{});var Ir=n(ll);xa=r(Ir,"3.21s"),Ir.forEach(l),Ha=f(gt),sl=a(gt,"TD",{});var Zr=n(sl);Da=r(Zr,"x2.96"),Zr.forEach(l),gt.forEach(l),Fa=f(Y),q=a(Y,"TR",{});var Ct=n(q);al=a(Ct,"TD",{});var gr=n(al);Pa=r(gr,"memory-efficient attention"),gr.forEach(l),qa=f(Ct),nl=a(Ct,"TD",{});var Cr=n(nl);La=r(Cr,"2.63s"),Cr.forEach(l),Ka=f(Ct),ol=a(Ct,"TD",{});var Br=n(ol);Oa=r(Br,"x3.61"),Br.forEach(l),Ct.forEach(l),Y.forEach(l),ta.forEach(l),os=f(e),L=a(e,"H2",{class:!0});var la=n(L);he=a(la,"A",{id:!0,class:!0,href:!0});var Gr=n(he);rl=a(Gr,"SPAN",{});var Wr=n(rl);U(Re.$$.fragment,Wr),Wr.forEach(l),Gr.forEach(l),en=f(la),il=a(la,"SPAN",{});var Vr=n(il);tn=r(Vr,"Sliced VAE"),Vr.forEach(l),la.forEach(l),rs=f(e),ce=a(e,"P",{});var sa=n(ce);ln=r(sa,"Sliced VAE enables decoding large batches of images with limited VRAM or batches with 32 images or more by decoding the batches of latents one image at a time. You\u2019ll likely want to couple this with "),nt=a(sa,"A",{href:!0});var $r=n(nt);sn=r($r,"enable_xformers_memory_efficient_attention()"),$r.forEach(l),an=r(sa," to further reduce memory use."),sa.forEach(l),is=f(e),ue=a(e,"P",{});var aa=n(ue);nn=r(aa,"To use sliced VAE, call "),ot=a(aa,"A",{href:!0});var kr=n(ot);on=r(kr,"enable_vae_slicing()"),kr.forEach(l),rn=r(aa," on your pipeline before inference:"),aa.forEach(l),ps=f(e),U(Se.$$.fragment,e),fs=f(e),rt=a(e,"P",{});var Rr=n(rt);pn=r(Rr,"You may see a small performance boost in VAE decoding on multi-image batches, and there should be no performance impact on single-image batches."),Rr.forEach(l),ms=f(e),K=a(e,"H2",{class:!0});var na=n(K);ye=a(na,"A",{id:!0,class:!0,href:!0});var Sr=n(ye);pl=a(Sr,"SPAN",{});var Nr=n(pl);U(Ne.$$.fragment,Nr),Nr.forEach(l),Sr.forEach(l),fn=f(na),fl=a(na,"SPAN",{});var Ar=n(fl);mn=r(Ar,"Tiled VAE"),Ar.forEach(l),na.forEach(l),ds=f(e),Me=a(e,"P",{});var oa=n(Me);dn=r(oa,"Tiled VAE processing also enables working with large images on limited VRAM (for example, generating 4k images on 8GB of VRAM) by splitting the image into overlapping tiles, decoding the tiles, and then blending the outputs together to compose the final image. You should also used tiled VAE with "),it=a(oa,"A",{href:!0});var Xr=n(it);hn=r(Xr,"enable_xformers_memory_efficient_attention()"),Xr.forEach(l),cn=r(oa," to further reduce memory use."),oa.forEach(l),hs=f(e),we=a(e,"P",{});var ra=n(we);un=r(ra,"To use tiled VAE processing, call "),pt=a(ra,"A",{href:!0});var Yr=n(pt);yn=r(Yr,"enable_vae_tiling()"),Yr.forEach(l),Mn=r(ra," on your pipeline before inference:"),ra.forEach(l),cs=f(e),U(Ae.$$.fragment,e),us=f(e),ft=a(e,"P",{});var Qr=n(ft);wn=r(Qr,"The output image has some tile-to-tile tone variation because the tiles are decoded separately, but you shouldn\u2019t see any sharp and obvious seams between the tiles. Tiling is turned off for images that are 512x512 or smaller."),Qr.forEach(l),ys=f(e),O=a(e,"H2",{class:!0});var ia=n(O);Je=a(ia,"A",{id:!0,class:!0,href:!0});var zr=n(Je);ml=a(zr,"SPAN",{});var xr=n(ml);U(Xe.$$.fragment,xr),xr.forEach(l),zr.forEach(l),Jn=f(ia),dl=a(ia,"SPAN",{});var Hr=n(dl);Tn=r(Hr,"CPU offloading"),Hr.forEach(l),ia.forEach(l),Ms=f(e),mt=a(e,"P",{});var Dr=n(mt);bn=r(Dr,"Offloading the weights to the CPU and only loading them on the GPU when performing the forward pass can also save memory. Often, this technique can reduce memory consumption to less than 3GB."),Dr.forEach(l),ws=f(e),Te=a(e,"P",{});var pa=n(Te);Un=r(pa,"To perform CPU offloading, call "),dt=a(pa,"A",{href:!0});var Fr=n(dt);_n=r(Fr,"enable_sequential_cpu_offload()"),Fr.forEach(l),vn=r(pa,":"),pa.forEach(l),Js=f(e),U(Ye.$$.fragment,e),Ts=f(e),be=a(e,"P",{});var fa=n(be);jn=r(fa,"CPU offloading works on submodules rather than whole models. This is the best way to minimize memory consumption, but inference is much slower due to the iterative nature of the diffusion process. The UNet component of the pipeline runs several times (as many as "),hl=a(fa,"CODE",{});var Pr=n(hl);En=r(Pr,"num_inference_steps"),Pr.forEach(l),In=r(fa,"); each time, the different UNet submodules are sequentially onloaded and offloaded as needed, resulting in a large number of memory transfers."),fa.forEach(l),bs=f(e),U(Ue.$$.fragment,e),Us=f(e),ht=a(e,"P",{});var qr=n(ht);Zn=r(qr,"CPU offloading can also be chained with attention slicing to reduce memory consumption to less than 2GB."),qr.forEach(l),_s=f(e),U(Qe.$$.fragment,e),vs=f(e),U(_e.$$.fragment,e),js=f(e),ee=a(e,"H2",{class:!0});var ma=n(ee);ve=a(ma,"A",{id:!0,class:!0,href:!0});var Lr=n(ve);cl=a(Lr,"SPAN",{});var Kr=n(cl);U(ze.$$.fragment,Kr),Kr.forEach(l),Lr.forEach(l),gn=f(ma),ul=a(ma,"SPAN",{});var Or=n(ul);Cn=r(Or,"Model offloading"),Or.forEach(l),ma.forEach(l),Es=f(e),U(je.$$.fragment,e),Is=f(e),xe=a(e,"P",{});var hr=n(xe);ct=a(hr,"A",{href:!0});var ei=n(ct);Bn=r(ei,"Sequential CPU offloading"),ei.forEach(l),Gn=r(hr," preserves a lot of memory but it makes inference slower because submodules are moved to GPU as needed, and they\u2019re immediately returned to the CPU when a new module runs."),hr.forEach(l),Zs=f(e),N=a(e,"P",{});var Bt=n(N);Wn=r(Bt,"Full-model offloading is an alternative that moves whole models to the GPU, instead of handling each model\u2019s constituent "),yl=a(Bt,"EM",{});var ti=n(yl);Vn=r(ti,"submodules"),ti.forEach(l),$n=r(Bt,". There is a negligible impact on inference time (compared with moving the pipeline to "),Ml=a(Bt,"CODE",{});var li=n(Ml);kn=r(li,"cuda"),li.forEach(l),Rn=r(Bt,"), and it still provides some memory savings."),Bt.forEach(l),gs=f(e),ut=a(e,"P",{});var si=n(ut);Sn=r(si,`During model offloading, only one of the main components of the pipeline (typically the text encoder, UNet and VAE)
is placed on the GPU while the others wait on the CPU. Components like the UNet that run for multiple iterations stay on the GPU until they\u2019re no longer needed.`),si.forEach(l),Cs=f(e),Ee=a(e,"P",{});var da=n(Ee);Nn=r(da,"Enable model offloading by calling "),yt=a(da,"A",{href:!0});var ai=n(yt);An=r(ai,"enable_model_cpu_offload()"),ai.forEach(l),Xn=r(da," on the pipeline:"),da.forEach(l),Bs=f(e),U(He.$$.fragment,e),Gs=f(e),Mt=a(e,"P",{});var ni=n(Mt);Yn=r(ni,"Model offloading can also be combined with attention slicing for additional memory savings."),ni.forEach(l),Ws=f(e),U(De.$$.fragment,e),Vs=f(e),U(Ie.$$.fragment,e),$s=f(e),te=a(e,"H2",{class:!0});var ha=n(te);Ze=a(ha,"A",{id:!0,class:!0,href:!0});var oi=n(Ze);wl=a(oi,"SPAN",{});var ri=n(wl);U(Fe.$$.fragment,ri),ri.forEach(l),oi.forEach(l),Qn=f(ha),Jl=a(ha,"SPAN",{});var ii=n(Jl);zn=r(ii,"Channels-last memory format"),ii.forEach(l),ha.forEach(l),ks=f(e),wt=a(e,"P",{});var pi=n(wt);xn=r(pi,"The channels-last memory format is an alternative way of ordering NCHW tensors in memory to preserve dimension ordering. Channels-last tensors are ordered in such a way that the channels become the densest dimension (storing images pixel-per-pixel). Since not all operators currently support the channels-last format, it may result in worst performance but you should still try and see if it works for your model."),pi.forEach(l),Rs=f(e),Jt=a(e,"P",{});var fi=n(Jt);Hn=r(fi,"For example, to set the pipeline\u2019s UNet to use the channels-last format:"),fi.forEach(l),Ss=f(e),U(Pe.$$.fragment,e),Ns=f(e),le=a(e,"H2",{class:!0});var ca=n(le);ge=a(ca,"A",{id:!0,class:!0,href:!0});var mi=n(ge);Tl=a(mi,"SPAN",{});var di=n(Tl);U(qe.$$.fragment,di),di.forEach(l),mi.forEach(l),Dn=f(ca),bl=a(ca,"SPAN",{});var hi=n(bl);Fn=r(hi,"Tracing"),hi.forEach(l),ca.forEach(l),As=f(e),Ce=a(e,"P",{});var ua=n(Ce);Pn=r(ua,"Tracing runs an example input tensor through the model and captures the operations that are performed on it as that input makes its way through the model\u2019s layers. The executable or "),Ul=a(ua,"CODE",{});var ci=n(Ul);qn=r(ci,"ScriptFunction"),ci.forEach(l),Ln=r(ua," that is returned is optimized with just-in-time compilation."),ua.forEach(l),Xs=f(e),Tt=a(e,"P",{});var ui=n(Tt);Kn=r(ui,"To trace a UNet:"),ui.forEach(l),Ys=f(e),U(Le.$$.fragment,e),Qs=f(e),Be=a(e,"P",{});var ya=n(Be);On=r(ya,"Replace the "),_l=a(ya,"CODE",{});var yi=n(_l);eo=r(yi,"unet"),yi.forEach(l),to=r(ya," attribute of the pipeline with the traced model:"),ya.forEach(l),zs=f(e),U(Ke.$$.fragment,e),xs=f(e),se=a(e,"H2",{class:!0});var Ma=n(se);Ge=a(Ma,"A",{id:!0,class:!0,href:!0});var Mi=n(Ge);vl=a(Mi,"SPAN",{});var wi=n(vl);U(Oe.$$.fragment,wi),wi.forEach(l),Mi.forEach(l),lo=f(Ma),jl=a(Ma,"SPAN",{});var Ji=n(jl);so=r(Ji,"Memory-efficient attention"),Ji.forEach(l),Ma.forEach(l),Hs=f(e),A=a(e,"P",{});var Gt=n(A);ao=r(Gt,"Recent work on optimizing bandwidth in the attention block has generated huge speed-ups and reductions in GPU memory usage. The most recent type of memory-efficient attention is "),et=a(Gt,"A",{href:!0,rel:!0});var Ti=n(et);no=r(Ti,"Flash Attention"),Ti.forEach(l),oo=r(Gt," (you can check out the original code at "),tt=a(Gt,"A",{href:!0,rel:!0});var bi=n(tt);ro=r(bi,"HazyResearch/flash-attention"),bi.forEach(l),io=r(Gt,")."),Gt.forEach(l),Ds=f(e),bt=a(e,"P",{});var Ui=n(bt);po=r(Ui,"The table below details the speed-ups from a few different Nvidia GPUs when running inference on image sizes of 512x512 and a batch size of 1 (one prompt):"),Ui.forEach(l),Fs=f(e),We=a(e,"TABLE",{});var wa=n(We);El=a(wa,"THEAD",{});var _i=n(El);ae=a(_i,"TR",{});var Wt=n(ae);Il=a(Wt,"TH",{});var vi=n(Il);fo=r(vi,"GPU"),vi.forEach(l),mo=f(Wt),Zl=a(Wt,"TH",{});var ji=n(Zl);ho=r(ji,"base attention (fp16)"),ji.forEach(l),co=f(Wt),gl=a(Wt,"TH",{});var Ei=n(gl);uo=r(Ei,"memory-efficient attention (fp16)"),Ei.forEach(l),Wt.forEach(l),_i.forEach(l),yo=f(wa),C=a(wa,"TBODY",{});var V=n(C);ne=a(V,"TR",{});var Vt=n(ne);Cl=a(Vt,"TD",{});var Ii=n(Cl);Mo=r(Ii,"NVIDIA Tesla T4"),Ii.forEach(l),wo=f(Vt),Bl=a(Vt,"TD",{});var Zi=n(Bl);Jo=r(Zi,"3.5it/s"),Zi.forEach(l),To=f(Vt),Gl=a(Vt,"TD",{});var gi=n(Gl);bo=r(gi,"5.5it/s"),gi.forEach(l),Vt.forEach(l),Uo=f(V),oe=a(V,"TR",{});var $t=n(oe);Wl=a($t,"TD",{});var Ci=n(Wl);_o=r(Ci,"NVIDIA 3060 RTX"),Ci.forEach(l),vo=f($t),Vl=a($t,"TD",{});var Bi=n(Vl);jo=r(Bi,"4.6it/s"),Bi.forEach(l),Eo=f($t),$l=a($t,"TD",{});var Gi=n($l);Io=r(Gi,"7.8it/s"),Gi.forEach(l),$t.forEach(l),Zo=f(V),re=a(V,"TR",{});var kt=n(re);kl=a(kt,"TD",{});var Wi=n(kl);go=r(Wi,"NVIDIA A10G"),Wi.forEach(l),Co=f(kt),Rl=a(kt,"TD",{});var Vi=n(Rl);Bo=r(Vi,"8.88it/s"),Vi.forEach(l),Go=f(kt),Sl=a(kt,"TD",{});var $i=n(Sl);Wo=r($i,"15.6it/s"),$i.forEach(l),kt.forEach(l),Vo=f(V),ie=a(V,"TR",{});var Rt=n(ie);Nl=a(Rt,"TD",{});var ki=n(Nl);$o=r(ki,"NVIDIA RTX A6000"),ki.forEach(l),ko=f(Rt),Al=a(Rt,"TD",{});var Ri=n(Al);Ro=r(Ri,"11.7it/s"),Ri.forEach(l),So=f(Rt),Xl=a(Rt,"TD",{});var Si=n(Xl);No=r(Si,"21.09it/s"),Si.forEach(l),Rt.forEach(l),Ao=f(V),pe=a(V,"TR",{});var St=n(pe);Yl=a(St,"TD",{});var Ni=n(Yl);Xo=r(Ni,"NVIDIA TITAN RTX"),Ni.forEach(l),Yo=f(St),Ql=a(St,"TD",{});var Ai=n(Ql);Qo=r(Ai,"12.51it/s"),Ai.forEach(l),zo=f(St),zl=a(St,"TD",{});var Xi=n(zl);xo=r(Xi,"18.22it/s"),Xi.forEach(l),St.forEach(l),Ho=f(V),fe=a(V,"TR",{});var Nt=n(fe);xl=a(Nt,"TD",{});var Yi=n(xl);Do=r(Yi,"A100-SXM4-40GB"),Yi.forEach(l),Fo=f(Nt),Hl=a(Nt,"TD",{});var Qi=n(Hl);Po=r(Qi,"18.6it/s"),Qi.forEach(l),qo=f(Nt),Dl=a(Nt,"TD",{});var zi=n(Dl);Lo=r(zi,"29.it/s"),zi.forEach(l),Nt.forEach(l),Ko=f(V),me=a(V,"TR",{});var At=n(me);Fl=a(At,"TD",{});var xi=n(Fl);Oo=r(xi,"A100-SXM-80GB"),xi.forEach(l),er=f(At),Pl=a(At,"TD",{});var Hi=n(Pl);tr=r(Hi,"18.7it/s"),Hi.forEach(l),lr=f(At),ql=a(At,"TD",{});var Di=n(ql);sr=r(Di,"29.5it/s"),Di.forEach(l),At.forEach(l),V.forEach(l),wa.forEach(l),Ps=f(e),U(Ve.$$.fragment,e),qs=f(e),Ut=a(e,"P",{});var Fi=n(Ut);ar=r(Fi,"To use Flash Attention, install the following:"),Fi.forEach(l),Ls=f(e),X=a(e,"UL",{});var Xt=n(X);Ll=a(Xt,"LI",{});var Pi=n(Ll);nr=r(Pi,"PyTorch > 1.12"),Pi.forEach(l),or=f(Xt),Kl=a(Xt,"LI",{});var qi=n(Kl);rr=r(qi,"CUDA available"),qi.forEach(l),ir=f(Xt),Ol=a(Xt,"LI",{});var Li=n(Ol);_t=a(Li,"A",{href:!0});var Ki=n(_t);pr=r(Ki,"xFormers"),Ki.forEach(l),Li.forEach(l),Xt.forEach(l),Ks=f(e),$e=a(e,"P",{});var Ja=n($e);fr=r(Ja,"Then call "),vt=a(Ja,"A",{href:!0});var Oi=n(vt);mr=r(Oi,"enable_xformers_memory_efficient_attention()"),Oi.forEach(l),dr=r(Ja," on the pipeline:"),Ja.forEach(l),Os=f(e),U(lt.$$.fragment,e),this.h()},h(){h(d,"name","hf:doc:metadata"),h(d,"content",JSON.stringify(dp)),h(u,"id","reduce-memory-usage"),h(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(u,"href","#reduce-memory-usage"),h(c,"class","relative group"),h(he,"id","sliced-vae"),h(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(he,"href","#sliced-vae"),h(L,"class","relative group"),h(nt,"href","/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.enable_xformers_memory_efficient_attention"),h(ot,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline.enable_vae_slicing"),h(ye,"id","tiled-vae"),h(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ye,"href","#tiled-vae"),h(K,"class","relative group"),h(it,"href","/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.enable_xformers_memory_efficient_attention"),h(pt,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline.enable_vae_tiling"),h(Je,"id","cpu-offloading"),h(Je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Je,"href","#cpu-offloading"),h(O,"class","relative group"),h(dt,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/latent_upscale#diffusers.StableDiffusionLatentUpscalePipeline.enable_sequential_cpu_offload"),h(ve,"id","model-offloading"),h(ve,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ve,"href","#model-offloading"),h(ee,"class","relative group"),h(ct,"href","#cpu-offloading"),h(yt,"href","/docs/diffusers/main/en/api/pipelines/stable_diffusion/gligen#diffusers.StableDiffusionGLIGENTextImagePipeline.enable_model_cpu_offload"),h(Ze,"id","channelslast-memory-format"),h(Ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ze,"href","#channelslast-memory-format"),h(te,"class","relative group"),h(ge,"id","tracing"),h(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ge,"href","#tracing"),h(le,"class","relative group"),h(Ge,"id","memoryefficient-attention"),h(Ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ge,"href","#memoryefficient-attention"),h(se,"class","relative group"),h(et,"href","https://arxiv.org/pdf/2205.14135.pdf"),h(et,"rel","nofollow"),h(tt,"href","https://github.com/HazyResearch/flash-attention"),h(tt,"rel","nofollow"),h(_t,"href","xformers"),h(vt,"href","/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.enable_xformers_memory_efficient_attention")},m(e,i){t(document.head,d),m(e,w,i),m(e,c,i),t(c,u),t(u,Z),_(y,Z,null),t(c,M),t(c,I),t(I,k),m(e,B,i),m(e,J,i),t(J,G),m(e,R,i),_(T,e,i),m(e,g,i),m(e,z,i),t(z,at),m(e,de,i),m(e,S,i),t(S,Qt),t(Qt,x),t(x,ns),t(x,Ta),t(x,zt),t(zt,ba),t(x,Ua),t(x,xt),t(xt,_a),t(S,va),t(S,$),t($,H),t(H,Ht),t(Ht,ja),t(H,Ea),t(H,Dt),t(Dt,Ia),t(H,Za),t(H,Ft),t(Ft,ga),t($,Ca),t($,D),t(D,Pt),t(Pt,Ba),t(D,Ga),t(D,qt),t(qt,Wa),t(D,Va),t(D,Lt),t(Lt,$a),t($,ka),t($,F),t(F,Kt),t(Kt,Ra),t(F,Sa),t(F,Ot),t(Ot,Na),t(F,Aa),t(F,el),t(el,Xa),t($,Ya),t($,P),t(P,tl),t(tl,Qa),t(P,za),t(P,ll),t(ll,xa),t(P,Ha),t(P,sl),t(sl,Da),t($,Fa),t($,q),t(q,al),t(al,Pa),t(q,qa),t(q,nl),t(nl,La),t(q,Ka),t(q,ol),t(ol,Oa),m(e,os,i),m(e,L,i),t(L,he),t(he,rl),_(Re,rl,null),t(L,en),t(L,il),t(il,tn),m(e,rs,i),m(e,ce,i),t(ce,ln),t(ce,nt),t(nt,sn),t(ce,an),m(e,is,i),m(e,ue,i),t(ue,nn),t(ue,ot),t(ot,on),t(ue,rn),m(e,ps,i),_(Se,e,i),m(e,fs,i),m(e,rt,i),t(rt,pn),m(e,ms,i),m(e,K,i),t(K,ye),t(ye,pl),_(Ne,pl,null),t(K,fn),t(K,fl),t(fl,mn),m(e,ds,i),m(e,Me,i),t(Me,dn),t(Me,it),t(it,hn),t(Me,cn),m(e,hs,i),m(e,we,i),t(we,un),t(we,pt),t(pt,yn),t(we,Mn),m(e,cs,i),_(Ae,e,i),m(e,us,i),m(e,ft,i),t(ft,wn),m(e,ys,i),m(e,O,i),t(O,Je),t(Je,ml),_(Xe,ml,null),t(O,Jn),t(O,dl),t(dl,Tn),m(e,Ms,i),m(e,mt,i),t(mt,bn),m(e,ws,i),m(e,Te,i),t(Te,Un),t(Te,dt),t(dt,_n),t(Te,vn),m(e,Js,i),_(Ye,e,i),m(e,Ts,i),m(e,be,i),t(be,jn),t(be,hl),t(hl,En),t(be,In),m(e,bs,i),_(Ue,e,i),m(e,Us,i),m(e,ht,i),t(ht,Zn),m(e,_s,i),_(Qe,e,i),m(e,vs,i),_(_e,e,i),m(e,js,i),m(e,ee,i),t(ee,ve),t(ve,cl),_(ze,cl,null),t(ee,gn),t(ee,ul),t(ul,Cn),m(e,Es,i),_(je,e,i),m(e,Is,i),m(e,xe,i),t(xe,ct),t(ct,Bn),t(xe,Gn),m(e,Zs,i),m(e,N,i),t(N,Wn),t(N,yl),t(yl,Vn),t(N,$n),t(N,Ml),t(Ml,kn),t(N,Rn),m(e,gs,i),m(e,ut,i),t(ut,Sn),m(e,Cs,i),m(e,Ee,i),t(Ee,Nn),t(Ee,yt),t(yt,An),t(Ee,Xn),m(e,Bs,i),_(He,e,i),m(e,Gs,i),m(e,Mt,i),t(Mt,Yn),m(e,Ws,i),_(De,e,i),m(e,Vs,i),_(Ie,e,i),m(e,$s,i),m(e,te,i),t(te,Ze),t(Ze,wl),_(Fe,wl,null),t(te,Qn),t(te,Jl),t(Jl,zn),m(e,ks,i),m(e,wt,i),t(wt,xn),m(e,Rs,i),m(e,Jt,i),t(Jt,Hn),m(e,Ss,i),_(Pe,e,i),m(e,Ns,i),m(e,le,i),t(le,ge),t(ge,Tl),_(qe,Tl,null),t(le,Dn),t(le,bl),t(bl,Fn),m(e,As,i),m(e,Ce,i),t(Ce,Pn),t(Ce,Ul),t(Ul,qn),t(Ce,Ln),m(e,Xs,i),m(e,Tt,i),t(Tt,Kn),m(e,Ys,i),_(Le,e,i),m(e,Qs,i),m(e,Be,i),t(Be,On),t(Be,_l),t(_l,eo),t(Be,to),m(e,zs,i),_(Ke,e,i),m(e,xs,i),m(e,se,i),t(se,Ge),t(Ge,vl),_(Oe,vl,null),t(se,lo),t(se,jl),t(jl,so),m(e,Hs,i),m(e,A,i),t(A,ao),t(A,et),t(et,no),t(A,oo),t(A,tt),t(tt,ro),t(A,io),m(e,Ds,i),m(e,bt,i),t(bt,po),m(e,Fs,i),m(e,We,i),t(We,El),t(El,ae),t(ae,Il),t(Il,fo),t(ae,mo),t(ae,Zl),t(Zl,ho),t(ae,co),t(ae,gl),t(gl,uo),t(We,yo),t(We,C),t(C,ne),t(ne,Cl),t(Cl,Mo),t(ne,wo),t(ne,Bl),t(Bl,Jo),t(ne,To),t(ne,Gl),t(Gl,bo),t(C,Uo),t(C,oe),t(oe,Wl),t(Wl,_o),t(oe,vo),t(oe,Vl),t(Vl,jo),t(oe,Eo),t(oe,$l),t($l,Io),t(C,Zo),t(C,re),t(re,kl),t(kl,go),t(re,Co),t(re,Rl),t(Rl,Bo),t(re,Go),t(re,Sl),t(Sl,Wo),t(C,Vo),t(C,ie),t(ie,Nl),t(Nl,$o),t(ie,ko),t(ie,Al),t(Al,Ro),t(ie,So),t(ie,Xl),t(Xl,No),t(C,Ao),t(C,pe),t(pe,Yl),t(Yl,Xo),t(pe,Yo),t(pe,Ql),t(Ql,Qo),t(pe,zo),t(pe,zl),t(zl,xo),t(C,Ho),t(C,fe),t(fe,xl),t(xl,Do),t(fe,Fo),t(fe,Hl),t(Hl,Po),t(fe,qo),t(fe,Dl),t(Dl,Lo),t(C,Ko),t(C,me),t(me,Fl),t(Fl,Oo),t(me,er),t(me,Pl),t(Pl,tr),t(me,lr),t(me,ql),t(ql,sr),m(e,Ps,i),_(Ve,e,i),m(e,qs,i),m(e,Ut,i),t(Ut,ar),m(e,Ls,i),m(e,X,i),t(X,Ll),t(Ll,nr),t(X,or),t(X,Kl),t(Kl,rr),t(X,ir),t(X,Ol),t(Ol,_t),t(_t,pr),m(e,Ks,i),m(e,$e,i),t($e,fr),t($e,vt),t(vt,mr),t($e,dr),m(e,Os,i),_(lt,e,i),ea=!0},p(e,[i]){const st={};i&2&&(st.$$scope={dirty:i,ctx:e}),T.$set(st);const es={};i&2&&(es.$$scope={dirty:i,ctx:e}),Ue.$set(es);const ts={};i&2&&(ts.$$scope={dirty:i,ctx:e}),_e.$set(ts);const ls={};i&2&&(ls.$$scope={dirty:i,ctx:e}),je.$set(ls);const ss={};i&2&&(ss.$$scope={dirty:i,ctx:e}),Ie.$set(ss);const as={};i&2&&(as.$$scope={dirty:i,ctx:e}),Ve.$set(as)},i(e){ea||(v(y.$$.fragment,e),v(T.$$.fragment,e),v(Re.$$.fragment,e),v(Se.$$.fragment,e),v(Ne.$$.fragment,e),v(Ae.$$.fragment,e),v(Xe.$$.fragment,e),v(Ye.$$.fragment,e),v(Ue.$$.fragment,e),v(Qe.$$.fragment,e),v(_e.$$.fragment,e),v(ze.$$.fragment,e),v(je.$$.fragment,e),v(He.$$.fragment,e),v(De.$$.fragment,e),v(Ie.$$.fragment,e),v(Fe.$$.fragment,e),v(Pe.$$.fragment,e),v(qe.$$.fragment,e),v(Le.$$.fragment,e),v(Ke.$$.fragment,e),v(Oe.$$.fragment,e),v(Ve.$$.fragment,e),v(lt.$$.fragment,e),ea=!0)},o(e){j(y.$$.fragment,e),j(T.$$.fragment,e),j(Re.$$.fragment,e),j(Se.$$.fragment,e),j(Ne.$$.fragment,e),j(Ae.$$.fragment,e),j(Xe.$$.fragment,e),j(Ye.$$.fragment,e),j(Ue.$$.fragment,e),j(Qe.$$.fragment,e),j(_e.$$.fragment,e),j(ze.$$.fragment,e),j(je.$$.fragment,e),j(He.$$.fragment,e),j(De.$$.fragment,e),j(Ie.$$.fragment,e),j(Fe.$$.fragment,e),j(Pe.$$.fragment,e),j(qe.$$.fragment,e),j(Le.$$.fragment,e),j(Ke.$$.fragment,e),j(Oe.$$.fragment,e),j(Ve.$$.fragment,e),j(lt.$$.fragment,e),ea=!1},d(e){l(d),e&&l(w),e&&l(c),E(y),e&&l(B),e&&l(J),e&&l(R),E(T,e),e&&l(g),e&&l(z),e&&l(de),e&&l(S),e&&l(os),e&&l(L),E(Re),e&&l(rs),e&&l(ce),e&&l(is),e&&l(ue),e&&l(ps),E(Se,e),e&&l(fs),e&&l(rt),e&&l(ms),e&&l(K),E(Ne),e&&l(ds),e&&l(Me),e&&l(hs),e&&l(we),e&&l(cs),E(Ae,e),e&&l(us),e&&l(ft),e&&l(ys),e&&l(O),E(Xe),e&&l(Ms),e&&l(mt),e&&l(ws),e&&l(Te),e&&l(Js),E(Ye,e),e&&l(Ts),e&&l(be),e&&l(bs),E(Ue,e),e&&l(Us),e&&l(ht),e&&l(_s),E(Qe,e),e&&l(vs),E(_e,e),e&&l(js),e&&l(ee),E(ze),e&&l(Es),E(je,e),e&&l(Is),e&&l(xe),e&&l(Zs),e&&l(N),e&&l(gs),e&&l(ut),e&&l(Cs),e&&l(Ee),e&&l(Bs),E(He,e),e&&l(Gs),e&&l(Mt),e&&l(Ws),E(De,e),e&&l(Vs),E(Ie,e),e&&l($s),e&&l(te),E(Fe),e&&l(ks),e&&l(wt),e&&l(Rs),e&&l(Jt),e&&l(Ss),E(Pe,e),e&&l(Ns),e&&l(le),E(qe),e&&l(As),e&&l(Ce),e&&l(Xs),e&&l(Tt),e&&l(Ys),E(Le,e),e&&l(Qs),e&&l(Be),e&&l(zs),E(Ke,e),e&&l(xs),e&&l(se),E(Oe),e&&l(Hs),e&&l(A),e&&l(Ds),e&&l(bt),e&&l(Fs),e&&l(We),e&&l(Ps),E(Ve,e),e&&l(qs),e&&l(Ut),e&&l(Ls),e&&l(X),e&&l(Ks),e&&l($e),e&&l(Os),E(lt,e)}}}const dp={local:"reduce-memory-usage",sections:[{local:"sliced-vae",title:"Sliced VAE"},{local:"tiled-vae",title:"Tiled VAE"},{local:"cpu-offloading",title:"CPU offloading"},{local:"model-offloading",title:"Model offloading"},{local:"channelslast-memory-format",title:"Channels-last memory format"},{local:"tracing",title:"Tracing"},{local:"memoryefficient-attention",title:"Memory-efficient attention"}],title:"Reduce memory usage"};function hp(W){return ap(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class wp extends ep{constructor(d){super();tp(this,d,hp,mp,lp,{})}}export{wp as default,dp as metadata};