accelerate/pr_4057/en/_app/immutable/nodes/38.fbb656fb.js

import{s as it,n as ot,o as pt}from"../chunks/scheduler.b9285784.js";import{S as rt,i as mt,e as i,s as n,c as r,h as ct,a as o,d as l,b as a,f as at,g as m,j as p,k as He,l as dt,m as s,n as c,t as d,o as f,p as h}from"../chunks/index.26bc89a1.js";import{C as ft,H as We,E as ht}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.52b7ec29.js";import{Y as ut}from"../chunks/Youtube.fd47a3ef.js";import{C as z}from"../chunks/CodeBlock.20a2bf0a.js";function yt(Pe){let u,O,K,ee,b,te,w,le,_,Ee='One of the biggest advancements Accelerate provides is <a href="../concept_guides/big_model_inference">Big Model Inference</a>, which allows you to perform inference with models that don’t fully fit on your graphics card.',se,g,Qe="This tutorial will show you how to use Big Model Inference in Accelerate and the Hugging Face ecosystem.",ne,$,ae,v,Le="A typical workflow for loading a PyTorch model is shown below. <code>ModelClass</code> is a model that exceeds the GPU memory of your device (mps or cuda or xpu).",ie,j,oe,T,Ve="With Big Model Inference, the first step is to init an empty skeleton of the model with the <code>init_empty_weights</code> context manager. This doesn’t require any memory because <code>my_model</code> is “parameterless”.",pe,C,re,U,Se="Next, the weights are loaded into the model for inference.",me,k,Ne='The <a href="/docs/accelerate/pr_4057/en/package_reference/big_modeling#accelerate.load_checkpoint_and_dispatch">load_checkpoint_and_dispatch()</a> method loads a checkpoint inside your empty model and dispatches the weights for each layer across all available devices, starting with the fastest devices (GPU, MPS, XPU, NPU, MLU, SDAA, MUSA) first before moving to the slower ones (CPU and hard drive).',ce,G,Fe="Setting <code>device_map=&quot;auto&quot;</code> automatically fills all available space on the GPU(s) first, then the CPU, and finally, the hard drive (the absolute slowest option) if there is still not enough memory.",de,y,Ye='<p>Refer to the <a href="../concept_guides/big_model_inference#designing-a-device-map">Designing a device map</a> guide for more details on how to design your own device map.</p>',fe,J,he,x,Ie='If there are certain “chunks” of layers that shouldn’t be split, pass them to <code>no_split_module_classes</code> (see <a href="../concept_guides/big_model_inference#loading-weights">here</a> for more details).',ue,Z,Ae='A models weights can also be sharded into multiple checkpoints to save memory, such as when the <code>state_dict</code> doesn’t fit in memory (see <a href="../concept_guides/big_model_inference#sharded-checkpoints">here</a> for more details).',ye,R,qe="Now that the model is fully dispatched, you can perform inference.",Me,X,be,B,ze="Each time an input is passed through a layer, it is sent from the CPU to the GPU (or disk to CPU to GPU), the output is calculated, and the layer is removed from the GPU going back down the line. While this adds some overhead to inference, it enables you to run any size model on your system, as long as the largest layer fits on your GPU.",we,H,Ke="Multiple GPUs, or “model parallelism”, can be utilized but only one GPU will be active at any given moment. This forces the GPU to wait for the previous GPU to send it the output. You should launch your script normally with Python instead of other tools like torchrun and accelerate launch.",_e,M,De='<p>You may also be interested in <em>pipeline parallelism</em> which utilizes all available GPUs at once, instead of only having one GPU active at a time. This approach is less flexible though. For more details, refer to the <a href="./distributed_inference#memory-efficient-pipeline-parallelism-experimental">Memory-efficient pipeline parallelism</a> guide.</p>',ge,W,$e,P,Oe="Take a look at a full example of Big Model Inference below.",ve,E,je,Q,Te,L,et='Other libraries in the Hugging Face ecosystem, like Transformers or Diffusers, supports Big Model Inference in their <a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained" rel="nofollow">from_pretrained</a> constructors.',Ce,V,tt='You just need to add <code>device_map=&quot;auto&quot;</code> in <a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained" rel="nofollow">from_pretrained</a> to enable Big Model Inference.',Ue,S,lt="For example, load Big Sciences T0pp 11 billion parameter model with Big Model Inference.",ke,N,Ge,F,st="After loading the model, the empty init and smart dispatch steps from before are executed and the model is fully ready to make use of all the resources in your machine. Through these constructors, you can also save more memory by specifying the <code>torch_dtype</code> parameter to load a model in a lower precision.",Je,Y,xe,I,Ze,A,nt='For a more detailed explanation of Big Model Inference, make sure to check out the <a href="../concept_guides/big_model_inference">conceptual guide</a>!',Re,q,Xe,D,Be;return b=new ft({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new We({props:{title:"Big Model Inference",local:"big-model-inference",headingTag:"h1"}}),$=new We({props:{title:"Accelerate",local:"accelerate",headingTag:"h2"}}),j=new z({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFteV9tb2RlbCUyMCUzRCUyME1vZGVsQ2xhc3MoLi4uKSUwQXN0YXRlX2RpY3QlMjAlM0QlMjB0b3JjaC5sb2FkKGNoZWNrcG9pbnRfZmlsZSklMEFteV9tb2RlbC5sb2FkX3N0YXRlX2RpY3Qoc3RhdGVfZGljdCk=",highlighted:`<span class="hljs-keyword">import</span> torch

my_model = ModelClass(...)
state_dict = torch.load(checkpoint_file)
my_model.load_state_dict(state_dict)`,lang:"py",wrap:!1}}),C=new z({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBpbml0X2VtcHR5X3dlaWdodHMlMEF3aXRoJTIwaW5pdF9lbXB0eV93ZWlnaHRzKCklM0ElMEElMjAlMjAlMjAlMjBteV9tb2RlbCUyMCUzRCUyME1vZGVsQ2xhc3MoLi4uKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> init_empty_weights
<span class="hljs-keyword">with</span> init_empty_weights():
    my_model = ModelClass(...)`,lang:"py",wrap:!1}}),J=new z({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBsb2FkX2NoZWNrcG9pbnRfYW5kX2Rpc3BhdGNoJTBBJTBBbW9kZWwlMjAlM0QlMjBsb2FkX2NoZWNrcG9pbnRfYW5kX2Rpc3BhdGNoKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTJDJTIwY2hlY2twb2ludCUzRGNoZWNrcG9pbnRfZmlsZSUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> load_checkpoint_and_dispatch

model = load_checkpoint_and_dispatch(
    model, checkpoint=checkpoint_file, device_map=<span class="hljs-string">&quot;auto&quot;</span>
)`,lang:"py",wrap:!1}}),X=new z({props:{code:"aW5wdXQlMjAlM0QlMjB0b3JjaC5yYW5kbigyJTJDMyklMEFkZXZpY2VfdHlwZSUyMCUzRCUyMG5leHQoaXRlcihtb2RlbC5wYXJhbWV0ZXJzKCkpKS5kZXZpY2UudHlwZSUwQWlucHV0JTIwJTNEJTIwaW5wdXQudG8oZGV2aWNlX3R5cGUpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoaW5wdXQp",highlighted:`<span class="hljs-built_in">input</span> = torch.randn(<span class="hljs-number">2</span>,<span class="hljs-number">3</span>)
device_type = <span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(model.parameters())).device.<span class="hljs-built_in">type</span>
<span class="hljs-built_in">input</span> = <span class="hljs-built_in">input</span>.to(device_type)
output = model(<span class="hljs-built_in">input</span>)`,lang:"py",wrap:!1}}),W=new ut({props:{id:"MWCSGj9jEAo"}}),E=new z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwYWNjZWxlcmF0ZSUyMGltcG9ydCUyMGluaXRfZW1wdHlfd2VpZ2h0cyUyQyUyMGxvYWRfY2hlY2twb2ludF9hbmRfZGlzcGF0Y2glMEElMEF3aXRoJTIwaW5pdF9lbXB0eV93ZWlnaHRzKCklM0ElMEElMjAlMjAlMjAlMjBtb2RlbCUyMCUzRCUyME15TW9kZWwoLi4uKSUwQSUwQW1vZGVsJTIwJTNEJTIwbG9hZF9jaGVja3BvaW50X2FuZF9kaXNwYXRjaCglMEElMjAlMjAlMjAlMjBtb2RlbCUyQyUyMGNoZWNrcG9pbnQlM0RjaGVja3BvaW50X2ZpbGUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUwQSklMEElMEFpbnB1dCUyMCUzRCUyMHRvcmNoLnJhbmRuKDIlMkMzKSUwQWRldmljZV90eXBlJTIwJTNEJTIwbmV4dChpdGVyKG1vZGVsLnBhcmFtZXRlcnMoKSkpLmRldmljZS50eXBlJTBBaW5wdXQlMjAlM0QlMjBpbnB1dC50byhkZXZpY2VfdHlwZSklMEFvdXRwdXQlMjAlM0QlMjBtb2RlbChpbnB1dCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> init_empty_weights, load_checkpoint_and_dispatch

<span class="hljs-keyword">with</span> init_empty_weights():
    model = MyModel(...)

model = load_checkpoint_and_dispatch(
    model, checkpoint=checkpoint_file, device_map=<span class="hljs-string">&quot;auto&quot;</span>
)

<span class="hljs-built_in">input</span> = torch.randn(<span class="hljs-number">2</span>,<span class="hljs-number">3</span>)
device_type = <span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(model.parameters())).device.<span class="hljs-built_in">type</span>
<span class="hljs-built_in">input</span> = <span class="hljs-built_in">input</span>.to(device_type)
output = model(<span class="hljs-built_in">input</span>)`,lang:"py",wrap:!1}}),Q=new We({props:{title:"Hugging Face ecosystem",local:"hugging-face-ecosystem",headingTag:"h2"}}),N=new z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxMlNlcUxNLmZyb21fcHJldHJhaW5lZCglMjJiaWdzY2llbmNlJTJGVDBwcCUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM

model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;bigscience/T0pp&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,lang:"py",wrap:!1}}),Y=new z({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclNlcTJTZXFMTSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yU2VxMlNlcUxNLmZyb21fcHJldHJhaW5lZCglMjJiaWdzY2llbmNlJTJGVDBwcCUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSeq2SeqLM

model = AutoModelForSeq2SeqLM.from_pretrained(<span class="hljs-string">&quot;bigscience/T0pp&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, torch_dtype=torch.float16)`,lang:"py",wrap:!1}}),I=new We({props:{title:"Next steps",local:"next-steps",headingTag:"h2"}}),q=new ht({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/big_modeling.md"}}),{c(){u=i("meta"),O=n(),K=i("p"),ee=n(),r(b.$$.fragment),te=n(),r(w.$$.fragment),le=n(),_=i("p"),_.innerHTML=Ee,se=n(),g=i("p"),g.textContent=Qe,ne=n(),r($.$$.fragment),ae=n(),v=i("p"),v.innerHTML=Le,ie=n(),r(j.$$.fragment),oe=n(),T=i("p"),T.innerHTML=Ve,pe=n(),r(C.$$.fragment),re=n(),U=i("p"),U.textContent=Se,me=n(),k=i("p"),k.innerHTML=Ne,ce=n(),G=i("p"),G.innerHTML=Fe,de=n(),y=i("blockquote"),y.innerHTML=Ye,fe=n(),r(J.$$.fragment),he=n(),x=i("p"),x.innerHTML=Ie,ue=n(),Z=i("p"),Z.innerHTML=Ae,ye=n(),R=i("p"),R.textContent=qe,Me=n(),r(X.$$.fragment),be=n(),B=i("p"),B.textContent=ze,we=n(),H=i("p"),H.textContent=Ke,_e=n(),M=i("blockquote"),M.innerHTML=De,ge=n(),r(W.$$.fragment),$e=n(),P=i("p"),P.textContent=Oe,ve=n(),r(E.$$.fragment),je=n(),r(Q.$$.fragment),Te=n(),L=i("p"),L.innerHTML=et,Ce=n(),V=i("p"),V.innerHTML=tt,Ue=n(),S=i("p"),S.textContent=lt,ke=n(),r(N.$$.fragment),Ge=n(),F=i("p"),F.innerHTML=st,Je=n(),r(Y.$$.fragment),xe=n(),r(I.$$.fragment),Ze=n(),A=i("p"),A.innerHTML=nt,Re=n(),r(q.$$.fragment),Xe=n(),D=i("p"),this.h()},l(e){const t=ct("svelte-u9bgzb",document.head);u=o(t,"META",{name:!0,content:!0}),t.forEach(l),O=a(e),K=o(e,"P",{}),at(K).forEach(l),ee=a(e),m(b.$$.fragment,e),te=a(e),m(w.$$.fragment,e),le=a(e),_=o(e,"P",{"data-svelte-h":!0}),p(_)!=="svelte-1njjewm"&&(_.innerHTML=Ee),se=a(e),g=o(e,"P",{"data-svelte-h":!0}),p(g)!=="svelte-1hrff2w"&&(g.textContent=Qe),ne=a(e),m($.$$.fragment,e),ae=a(e),v=o(e,"P",{"data-svelte-h":!0}),p(v)!=="svelte-w5qfqd"&&(v.innerHTML=Le),ie=a(e),m(j.$$.fragment,e),oe=a(e),T=o(e,"P",{"data-svelte-h":!0}),p(T)!=="svelte-91fu20"&&(T.innerHTML=Ve),pe=a(e),m(C.$$.fragment,e),re=a(e),U=o(e,"P",{"data-svelte-h":!0}),p(U)!=="svelte-158zzkm"&&(U.textContent=Se),me=a(e),k=o(e,"P",{"data-svelte-h":!0}),p(k)!=="svelte-ipiioo"&&(k.innerHTML=Ne),ce=a(e),G=o(e,"P",{"data-svelte-h":!0}),p(G)!=="svelte-6ilppp"&&(G.innerHTML=Fe),de=a(e),y=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),p(y)!=="svelte-1s1ycch"&&(y.innerHTML=Ye),fe=a(e),m(J.$$.fragment,e),he=a(e),x=o(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-iajmu4"&&(x.innerHTML=Ie),ue=a(e),Z=o(e,"P",{"data-svelte-h":!0}),p(Z)!=="svelte-lf7k2g"&&(Z.innerHTML=Ae),ye=a(e),R=o(e,"P",{"data-svelte-h":!0}),p(R)!=="svelte-7amkht"&&(R.textContent=qe),Me=a(e),m(X.$$.fragment,e),be=a(e),B=o(e,"P",{"data-svelte-h":!0}),p(B)!=="svelte-3b5yt7"&&(B.textContent=ze),we=a(e),H=o(e,"P",{"data-svelte-h":!0}),p(H)!=="svelte-1y01w5b"&&(H.textContent=Ke),_e=a(e),M=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),p(M)!=="svelte-lzl68u"&&(M.innerHTML=De),ge=a(e),m(W.$$.fragment,e),$e=a(e),P=o(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-1yv5ji4"&&(P.textContent=Oe),ve=a(e),m(E.$$.fragment,e),je=a(e),m(Q.$$.fragment,e),Te=a(e),L=o(e,"P",{"data-svelte-h":!0}),p(L)!=="svelte-1sxxcso"&&(L.innerHTML=et),Ce=a(e),V=o(e,"P",{"data-svelte-h":!0}),p(V)!=="svelte-owwo1o"&&(V.innerHTML=tt),Ue=a(e),S=o(e,"P",{"data-svelte-h":!0}),p(S)!=="svelte-ois3cz"&&(S.textContent=lt),ke=a(e),m(N.$$.fragment,e),Ge=a(e),F=o(e,"P",{"data-svelte-h":!0}),p(F)!=="svelte-25ai00"&&(F.innerHTML=st),Je=a(e),m(Y.$$.fragment,e),xe=a(e),m(I.$$.fragment,e),Ze=a(e),A=o(e,"P",{"data-svelte-h":!0}),p(A)!=="svelte-2pv49y"&&(A.innerHTML=nt),Re=a(e),m(q.$$.fragment,e),Xe=a(e),D=o(e,"P",{}),at(D).forEach(l),this.h()},h(){He(u,"name","hf:doc:metadata"),He(u,"content",Mt),He(y,"class","tip"),He(M,"class","tip")},m(e,t){dt(document.head,u),s(e,O,t),s(e,K,t),s(e,ee,t),c(b,e,t),s(e,te,t),c(w,e,t),s(e,le,t),s(e,_,t),s(e,se,t),s(e,g,t),s(e,ne,t),c($,e,t),s(e,ae,t),s(e,v,t),s(e,ie,t),c(j,e,t),s(e,oe,t),s(e,T,t),s(e,pe,t),c(C,e,t),s(e,re,t),s(e,U,t),s(e,me,t),s(e,k,t),s(e,ce,t),s(e,G,t),s(e,de,t),s(e,y,t),s(e,fe,t),c(J,e,t),s(e,he,t),s(e,x,t),s(e,ue,t),s(e,Z,t),s(e,ye,t),s(e,R,t),s(e,Me,t),c(X,e,t),s(e,be,t),s(e,B,t),s(e,we,t),s(e,H,t),s(e,_e,t),s(e,M,t),s(e,ge,t),c(W,e,t),s(e,$e,t),s(e,P,t),s(e,ve,t),c(E,e,t),s(e,je,t),c(Q,e,t),s(e,Te,t),s(e,L,t),s(e,Ce,t),s(e,V,t),s(e,Ue,t),s(e,S,t),s(e,ke,t),c(N,e,t),s(e,Ge,t),s(e,F,t),s(e,Je,t),c(Y,e,t),s(e,xe,t),c(I,e,t),s(e,Ze,t),s(e,A,t),s(e,Re,t),c(q,e,t),s(e,Xe,t),s(e,D,t),Be=!0},p:ot,i(e){Be||(d(b.$$.fragment,e),d(w.$$.fragment,e),d($.$$.fragment,e),d(j.$$.fragment,e),d(C.$$.fragment,e),d(J.$$.fragment,e),d(X.$$.fragment,e),d(W.$$.fragment,e),d(E.$$.fragment,e),d(Q.$$.fragment,e),d(N.$$.fragment,e),d(Y.$$.fragment,e),d(I.$$.fragment,e),d(q.$$.fragment,e),Be=!0)},o(e){f(b.$$.fragment,e),f(w.$$.fragment,e),f($.$$.fragment,e),f(j.$$.fragment,e),f(C.$$.fragment,e),f(J.$$.fragment,e),f(X.$$.fragment,e),f(W.$$.fragment,e),f(E.$$.fragment,e),f(Q.$$.fragment,e),f(N.$$.fragment,e),f(Y.$$.fragment,e),f(I.$$.fragment,e),f(q.$$.fragment,e),Be=!1},d(e){e&&(l(O),l(K),l(ee),l(te),l(le),l(_),l(se),l(g),l(ne),l(ae),l(v),l(ie),l(oe),l(T),l(pe),l(re),l(U),l(me),l(k),l(ce),l(G),l(de),l(y),l(fe),l(he),l(x),l(ue),l(Z),l(ye),l(R),l(Me),l(be),l(B),l(we),l(H),l(_e),l(M),l(ge),l($e),l(P),l(ve),l(je),l(Te),l(L),l(Ce),l(V),l(Ue),l(S),l(ke),l(Ge),l(F),l(Je),l(xe),l(Ze),l(A),l(Re),l(Xe),l(D)),l(u),h(b,e),h(w,e),h($,e),h(j,e),h(C,e),h(J,e),h(X,e),h(W,e),h(E,e),h(Q,e),h(N,e),h(Y,e),h(I,e),h(q,e)}}}const Mt='{"title":"Big Model Inference","local":"big-model-inference","sections":[{"title":"Accelerate","local":"accelerate","sections":[],"depth":2},{"title":"Hugging Face ecosystem","local":"hugging-face-ecosystem","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';function bt(Pe){return pt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jt extends rt{constructor(u){super(),mt(this,u,bt,yt,it,{})}}export{jt as component};