Buckets:
| import{s as Tn,o as Un,n as oe}from"../chunks/scheduler.b9285784.js";import{S as Cn,i as jn,e as f,s as a,c as r,h as Jn,a as _,d as o,b as n,f as y,g as c,j as b,k as w,l as d,m as l,n as i,t as m,o as h,p}from"../chunks/index.26bc89a1.js";import{T as da}from"../chunks/Tip.e4eba3d6.js";import{C as Gn,H as x,E as In}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.7a0ae628.js";import{D as M}from"../chunks/Docstring.3b3b5305.js";import{C as qt}from"../chunks/CodeBlock.844ff9c3.js";import{E as Qt}from"../chunks/ExampleCodeBlock.e3a326b9.js";function Nn(C){let s,k="Example:",g,v,$;return v=new qt({props:{code:"aW1wb3J0JTIwdG9yY2gubm4lMjBhcyUyMG5uJTBBZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBpbml0X2VtcHR5X3dlaWdodHMlMEElMEElMjMlMjBJbml0aWFsaXplJTIwYSUyMG1vZGVsJTIwd2l0aCUyMDEwMCUyMGJpbGxpb25zJTIwcGFyYW1ldGVycyUyMGluJTIwbm8lMjB0aW1lJTIwYW5kJTIwd2l0aG91dCUyMHVzaW5nJTIwYW55JTIwUkFNLiUwQXdpdGglMjBpbml0X2VtcHR5X3dlaWdodHMoKSUzQSUwQSUyMCUyMCUyMCUyMHRzdCUyMCUzRCUyMG5uLlNlcXVlbnRpYWwoKiU1Qm5uLkxpbmVhcigxMDAwMCUyQyUyMDEwMDAwKSUyMGZvciUyMF8lMjBpbiUyMHJhbmdlKDEwMDApJTVEKQ==",highlighted:`<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn | |
| <span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> init_empty_weights | |
| <span class="hljs-comment"># Initialize a model with 100 billions parameters in no time and without using any RAM.</span> | |
| <span class="hljs-keyword">with</span> init_empty_weights(): | |
| tst = nn.Sequential(*[nn.Linear(<span class="hljs-number">10000</span>, <span class="hljs-number">10000</span>) <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>)])`,wrap:!1}}),{c(){s=f("p"),s.textContent=k,g=a(),r(v.$$.fragment)},l(u){s=_(u,"P",{"data-svelte-h":!0}),b(s)!=="svelte-11lpom8"&&(s.textContent=k),g=n(u),c(v.$$.fragment,u)},m(u,T){l(u,s,T),l(u,g,T),i(v,u,T),$=!0},p:oe,i(u){$||(m(v.$$.fragment,u),$=!0)},o(u){h(v.$$.fragment,u),$=!1},d(u){u&&(o(s),o(g)),p(v,u)}}}function Fn(C){let s,k=`Any model created under this context manager has no weights. As such you can’t do something like | |
| <code>model.to(some_device)</code> with it. To load weights inside your empty model, see <a href="/docs/accelerate/pr_4021/en/package_reference/big_modeling#accelerate.load_checkpoint_and_dispatch">load_checkpoint_and_dispatch()</a>. | |
| Make sure to overwrite the default device_map param for <a href="/docs/accelerate/pr_4021/en/package_reference/big_modeling#accelerate.load_checkpoint_and_dispatch">load_checkpoint_and_dispatch()</a>, otherwise dispatch is not | |
| called.`;return{c(){s=f("p"),s.innerHTML=k},l(g){s=_(g,"P",{"data-svelte-h":!0}),b(s)!=="svelte-4cd706"&&(s.innerHTML=k)},m(g,v){l(g,s,v)},p:oe,d(g){g&&o(s)}}}function Hn(C){let s,k="Example:",g,v,$;return v=new qt({props:{code:"bW9kZWxfMSUyQyUyMGhvb2tfMSUyMCUzRCUyMGNwdV9vZmZsb2FkX3dpdGhfaG9vayhtb2RlbF8xJTJDJTIwZGV2aWNlKSUwQW1vZGVsXzIlMkMlMjBob29rXzIlMjAlM0QlMjBjcHVfb2ZmbG9hZF93aXRoX2hvb2sobW9kZWxfMiUyQyUyMGRldmljZSUyQyUyMHByZXZfbW9kdWxlX2hvb2slM0Rob29rXzEpJTBBbW9kZWxfMyUyQyUyMGhvb2tfMyUyMCUzRCUyMGNwdV9vZmZsb2FkX3dpdGhfaG9vayhtb2RlbF8zJTJDJTIwZGV2aWNlJTJDJTIwcHJldl9tb2R1bGVfaG9vayUzRGhvb2tfMiklMEElMEFoaWRfMSUyMCUzRCUyMG1vZGVsXzEoaW5wdXQpJTBBZm9yJTIwaSUyMGluJTIwcmFuZ2UoNTApJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwbW9kZWwxJTIwaXMlMjBvZmZsb2FkZWQlMjBvbiUyMHRoZSUyMENQVSUyMGF0JTIwdGhlJTIwZmlyc3QlMjBpdGVyYXRpb24lMkMlMjBtb2RlbCUyMDIlMjBzdGF5cyUyMG9uJTIwdGhlJTIwR1BVJTIwZm9yJTIwdGhpcyUyMHdob2xlJTIwbG9vcC4lMEElMjAlMjAlMjAlMjBoaWRfMiUyMCUzRCUyMG1vZGVsXzIoaGlkXzEpJTBBJTIzJTIwbW9kZWwyJTIwaXMlMjBvZmZsb2FkZWQlMjB0byUyMHRoZSUyMENQVSUyMGp1c3QlMjBiZWZvcmUlMjB0aGlzJTIwZm9yd2FyZC4lMEFoaWRfMyUyMCUzRCUyMG1vZGVsXzMoaGlkXzMpJTBBJTBBJTIzJTIwRm9yJTIwbW9kZWwzJTJDJTIweW91JTIwbmVlZCUyMHRvJTIwbWFudWFsbHklMjBjYWxsJTIwdGhlJTIwaG9vayUyMG9mZmxvYWQlMjBtZXRob2QuJTBBaG9va18zLm9mZmxvYWQoKQ==",highlighted:`model_1, hook_1 = cpu_offload_with_hook(model_1, device) | |
| model_2, hook_2 = cpu_offload_with_hook(model_2, device, prev_module_hook=hook_1) | |
| model_3, hook_3 = cpu_offload_with_hook(model_3, device, prev_module_hook=hook_2) | |
| hid_1 = model_1(<span class="hljs-built_in">input</span>) | |
| <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">50</span>): | |
| <span class="hljs-comment"># model1 is offloaded on the CPU at the first iteration, model 2 stays on the GPU for this whole loop.</span> | |
| hid_2 = model_2(hid_1) | |
| <span class="hljs-comment"># model2 is offloaded to the CPU just before this forward.</span> | |
| hid_3 = model_3(hid_3) | |
| <span class="hljs-comment"># For model3, you need to manually call the hook offload method.</span> | |
| hook_3.offload()`,wrap:!1}}),{c(){s=f("p"),s.textContent=k,g=a(),r(v.$$.fragment)},l(u){s=_(u,"P",{"data-svelte-h":!0}),b(s)!=="svelte-11lpom8"&&(s.textContent=k),g=n(u),c(v.$$.fragment,u)},m(u,T){l(u,s,T),l(u,g,T),i(v,u,T),$=!0},p:oe,i(u){$||(m(v.$$.fragment,u),$=!0)},o(u){h(v.$$.fragment,u),$=!1},d(u){u&&(o(s),o(g)),p(v,u)}}}function Wn(C){let s,k="Example:",g,v,$;return v=new qt({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBpbml0X2VtcHR5X3dlaWdodHMlMkMlMjBsb2FkX2NoZWNrcG9pbnRfYW5kX2Rpc3BhdGNoJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGhmX2h1Yl9kb3dubG9hZCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvQ29uZmlnJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEElMjMlMjBEb3dubG9hZCUyMHRoZSUyMFdlaWdodHMlMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyRWxldXRoZXJBSSUyRmdwdC1qLTZCJTIyJTBBd2VpZ2h0c19sb2NhdGlvbiUyMCUzRCUyMGhmX2h1Yl9kb3dubG9hZChjaGVja3BvaW50JTJDJTIwJTIycHl0b3JjaF9tb2RlbC5iaW4lMjIpJTBBJTBBJTIzJTIwQ3JlYXRlJTIwYSUyMG1vZGVsJTIwYW5kJTIwaW5pdGlhbGl6ZSUyMGl0JTIwd2l0aCUyMGVtcHR5JTIwd2VpZ2h0cyUwQWNvbmZpZyUyMCUzRCUyMEF1dG9Db25maWcuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBd2l0aCUyMGluaXRfZW1wdHlfd2VpZ2h0cygpJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX2NvbmZpZyhjb25maWcpJTBBJTBBJTIzJTIwTG9hZCUyMHRoZSUyMGNoZWNrcG9pbnQlMjBhbmQlMjBkaXNwYXRjaCUyMGl0JTIwdG8lMjB0aGUlMjByaWdodCUyMGRldmljZXMlMEFtb2RlbCUyMCUzRCUyMGxvYWRfY2hlY2twb2ludF9hbmRfZGlzcGF0Y2goJTBBJTIwJTIwJTIwJTIwbW9kZWwlMkMlMjB3ZWlnaHRzX2xvY2F0aW9uJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBub19zcGxpdF9tb2R1bGVfY2xhc3NlcyUzRCU1QiUyMkdQVEpCbG9jayUyMiU1RCUwQSk=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> init_empty_weights, load_checkpoint_and_dispatch | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hf_hub_download | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoConfig, AutoModelForCausalLM | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Download the Weights</span> | |
| <span class="hljs-meta">>>> </span>checkpoint = <span class="hljs-string">"EleutherAI/gpt-j-6B"</span> | |
| <span class="hljs-meta">>>> </span>weights_location = hf_hub_download(checkpoint, <span class="hljs-string">"pytorch_model.bin"</span>) | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Create a model and initialize it with empty weights</span> | |
| <span class="hljs-meta">>>> </span>config = AutoConfig.from_pretrained(checkpoint) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> init_empty_weights(): | |
| <span class="hljs-meta">... </span> model = AutoModelForCausalLM.from_config(config) | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Load the checkpoint and dispatch it to the right devices</span> | |
| <span class="hljs-meta">>>> </span>model = load_checkpoint_and_dispatch( | |
| <span class="hljs-meta">... </span> model, weights_location, device_map=<span class="hljs-string">"auto"</span>, no_split_module_classes=[<span class="hljs-string">"GPTJBlock"</span>] | |
| <span class="hljs-meta">... </span>)`,wrap:!1}}),{c(){s=f("p"),s.textContent=k,g=a(),r(v.$$.fragment)},l(u){s=_(u,"P",{"data-svelte-h":!0}),b(s)!=="svelte-11lpom8"&&(s.textContent=k),g=n(u),c(v.$$.fragment,u)},m(u,T){l(u,s,T),l(u,g,T),i(v,u,T),$=!0},p:oe,i(u){$||(m(v.$$.fragment,u),$=!0)},o(u){h(v.$$.fragment,u),$=!1},d(u){u&&(o(s),o(g)),p(v,u)}}}function Dn(C){let s,k=`Once loaded across devices, you still need to call <a href="/docs/accelerate/pr_4021/en/package_reference/big_modeling#accelerate.dispatch_model">dispatch_model()</a> on your model to make it able to run. To | |
| group the checkpoint loading and dispatch in one single call, use <a href="/docs/accelerate/pr_4021/en/package_reference/big_modeling#accelerate.load_checkpoint_and_dispatch">load_checkpoint_and_dispatch()</a>.`;return{c(){s=f("p"),s.innerHTML=k},l(g){s=_(g,"P",{"data-svelte-h":!0}),b(s)!=="svelte-vw3bsx"&&(s.innerHTML=k)},m(g,v){l(g,s,v)},p:oe,d(g){g&&o(s)}}}function Zn(C){let s,k=`All computation is done analyzing sizes and dtypes of the model parameters. As a result, the model can be on the | |
| meta device (as it would if initialized within the <code>init_empty_weights</code> context manager).`;return{c(){s=f("p"),s.innerHTML=k},l(g){s=_(g,"P",{"data-svelte-h":!0}),b(s)!=="svelte-1d9m3bn"&&(s.innerHTML=k)},m(g,v){l(g,s,v)},p:oe,d(g){g&&o(s)}}}function Bn(C){let s,k=`If the module already contains a hook, this will replace it with the new hook passed by default. To chain two hooks | |
| together, pass <code>append=True</code>, so it chains the current and new hook into an instance of the <code>SequentialHook</code> class.`;return{c(){s=f("p"),s.innerHTML=k},l(g){s=_(g,"P",{"data-svelte-h":!0}),b(s)!=="svelte-1dclc89"&&(s.innerHTML=k)},m(g,v){l(g,s,v)},p:oe,d(g){g&&o(s)}}}function Ln(C){let s,k="Example:",g,v,$;return v=new qt({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUuaG9va3MlMjBpbXBvcnQlMjBhdHRhY2hfbGF5ZXJ3aXNlX2Nhc3RpbmdfaG9va3MlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEFpbXBvcnQlMjB0b3JjaCUwQSUwQSUyMyUyME1vZGVsJTBBY2hlY2twb2ludCUyMCUzRCUyMCUyMkVsZXV0aGVyQUklMkZncHQtai02QiUyMiUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBJTBBJTIzJTIwQXR0YWNoJTIwaG9va3MlMjBhbmQlMjBwZXJmb3JtJTIwaW5mZXJlbmNlJTBBYXR0YWNoX2xheWVyd2lzZV9jYXN0aW5nX2hvb2tzKG1vZGVsJTJDJTIwc3RvcmFnZV9kdHlwZSUzRHRvcmNoLmZsb2F0OF9lNG0zZm4lMkMlMjBjb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYpJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsKC4uLik=",highlighted:`<span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> accelerate.hooks <span class="hljs-keyword">import</span> attach_layerwise_casting_hooks | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">import</span> torch | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Model</span> | |
| <span class="hljs-meta">>>> </span>checkpoint = <span class="hljs-string">"EleutherAI/gpt-j-6B"</span> | |
| <span class="hljs-meta">>>> </span>model = AutoModelForCausalLM.from_pretrained(checkpoint) | |
| <span class="hljs-meta">>>> </span><span class="hljs-comment"># Attach hooks and perform inference</span> | |
| <span class="hljs-meta">>>> </span>attach_layerwise_casting_hooks(model, storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16) | |
| <span class="hljs-meta">>>> </span><span class="hljs-keyword">with</span> torch.no_grad(): | |
| <span class="hljs-meta">... </span> model(...)`,wrap:!1}}),{c(){s=f("p"),s.textContent=k,g=a(),r(v.$$.fragment)},l(u){s=_(u,"P",{"data-svelte-h":!0}),b(s)!=="svelte-11lpom8"&&(s.textContent=k),g=n(u),c(v.$$.fragment,u)},m(u,T){l(u,s,T),l(u,g,T),i(v,u,T),$=!0},p:oe,i(u){$||(m(v.$$.fragment,u),$=!0)},o(u){h(v.$$.fragment,u),$=!1},d(u){u&&(o(s),o(g)),p(v,u)}}}function An(C){let s,k;return s=new qt({props:{code:"YXR0YWNoX2xheWVyd2lzZV9jYXN0aW5nX2hvb2tzKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTJDJTIwc3RvcmFnZV9kdHlwZSUzRHRvcmNoLmZsb2F0OF9lNG0zZm4lMkMlMjBjb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMjBza2lwX21vZHVsZXNfcGF0dGVybiUzRCU1QiUyMm5vcm0lMjIlNUQlMEEp",highlighted:`<span class="hljs-meta">>>> </span>attach_layerwise_casting_hooks( | |
| <span class="hljs-meta">... </span> model, storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16, skip_modules_pattern=[<span class="hljs-string">"norm"</span>] | |
| <span class="hljs-meta">... </span>)`,wrap:!1}}),{c(){r(s.$$.fragment)},l(g){c(s.$$.fragment,g)},m(g,v){i(s,g,v),k=!0},p:oe,i(g){k||(m(s.$$.fragment,g),k=!0)},o(g){h(s.$$.fragment,g),k=!1},d(g){p(s,g)}}}function Vn(C){let s,k,g,v,$,u,T,St,ue,Yt,ge,Ot,G,ve,ra,gt,Sa=`A context manager under which models are initialized with all parameters on the meta device, therefore creating an | |
| empty model. Useful when just initializing the model would blow the available RAM.`,ca,ae,ia,ne,Kt,be,eo,L,ke,ma,vt,Ya=`Activates full CPU offload for a model. As a result, all parameters of the model will be offloaded and only one | |
| copy of the state dict of the model will be kept. During the forward pass, parameters will be extracted from that | |
| state dict and put on the execution device passed as they are needed, then offloaded again.`,to,$e,oo,N,ye,ha,bt,Oa=`Offloads a model on the CPU and puts it back to an execution device when executed. The difference with | |
| <a href="/docs/accelerate/pr_4021/en/package_reference/big_modeling#accelerate.cpu_offload">cpu_offload()</a> is that the model stays on the execution device after the forward and is only offloaded again when | |
| the <code>offload</code> method of the returned <code>hook</code> is called. Useful for pipelines running a model in a loop.`,pa,le,ao,we,no,A,xe,fa,kt,Ka=`Activates full disk offload for a model. As a result, all parameters of the model will be offloaded as | |
| memory-mapped array in a given folder. During the forward pass, parameters will be accessed from that folder and | |
| put on the execution device passed as they are needed, then offloaded again.`,lo,Me,so,V,Te,_a,$t,en=`Dispatches a model according to a given device map. Layers of the model might be spread across GPUs, offloaded on | |
| the CPU or even the disk.`,ro,Ue,co,F,Ce,ua,yt,tn=`Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are | |
| loaded and adds the various hooks that will make this model run properly (even if split across devices).`,ga,se,io,je,mo,H,Je,va,wt,on=`Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are | |
| loaded.`,ba,de,ho,Ge,po,I,Ie,ka,xt,an=`Compute a device map for a given model giving priority to GPUs, then offload on CPU and finally offload to disk, | |
| such that:`,$a,Mt,nn=`<li>we don’t exceed the memory available of any of the GPU.</li> <li>if offload to the CPU is needed, there is always room left on GPU 0 to put back the layer offloaded on CPU that | |
| has the largest size.</li> <li>if offload to the CPU is needed,we don’t exceed the RAM available on the CPU.</li> <li>if offload to the disk is needed, there is always room left on the CPU to put back the layer offloaded on disk | |
| that has the largest size.</li>`,ya,re,fo,Ne,_o,Fe,uo,U,He,wa,Tt,ln=`A hook that contains callbacks to be executed just before and after the forward method of a model. The difference | |
| with PyTorch existing hooks is that they get passed along the kwargs.`,xa,Ut,sn="Class attribute:",Ma,Ct,dn=`<li><strong>no_grad</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — Whether or not to execute the actual forward pass under | |
| the <code>torch.no_grad()</code> context manager.</li>`,Ta,ce,We,Ua,jt,rn="To be executed when the hook is detached from a module.",Ca,ie,De,ja,Jt,cn="To be executed when the hook is attached to the module.",Ja,me,Ze,Ga,Gt,mn="To be executed just after the forward method of the model.",Ia,he,Be,Na,It,hn="To be executed just before the forward method of the model.",go,Le,vo,P,Ae,Fa,Nt,pn=`A generic <code>ModelHook</code> that ensures inputs and model weights are on the same device for the forward pass of the | |
| associated module, potentially offloading the weights after the forward pass.`,bo,Ve,ko,R,Pe,Ha,Ft,fn="A hook that can contain several hooks and iterates through them at each event.",$o,Re,yo,X,Xe,Wa,Ht,_n=`A hook that casts the weights of a module to a high precision dtype for computation, and to a low precision dtype | |
| for storage. This process may lead to quality loss in the output, but can significantly reduce the memory | |
| footprint.`,wo,Ee,xo,ze,Mo,W,Qe,Da,Wt,un=`Adds a hook to a given module. This will rewrite the <code>forward</code> method of the module to include the hook, to remove | |
| this behavior and restore the original <code>forward</code> method, use <code>remove_hook_from_module</code>.`,Za,pe,To,qe,Uo,E,Se,Ba,Dt,gn=`Recursively attaches <code>AlignDevicesHook</code> to all submodules of a given model to make sure they have the right | |
| execution device`,Co,Ye,jo,z,Oe,La,Zt,vn=`Recursively attaches <code>AlignDevicesHook</code> to all submodules of a given model that have direct parameters and/or | |
| buffers.`,Jo,Ke,Go,Q,et,Aa,Bt,bn="Attaches <code>AlignDevicesHook</code> to all blocks of a given model as needed.",Io,tt,No,J,ot,Va,Lt,kn=`Applies layerwise casting to a given module. The module expected here is a PyTorch <code>nn.Module</code>. This is helpful for | |
| reducing memory requirements when one doesn’t want to fully quantize a model. Model params can be kept in say, | |
| <code>torch.float8_e4m3fn</code> and upcasted to a higher precision like <code>torch.bfloat16</code> during forward pass and downcasted | |
| back to <code>torch.float8_e4m3fn</code> to realize memory savings.`,Pa,fe,Ra,At,$n="Users can also pass modules they want to avoid from getting downcasted.",Xa,_e,Fo,at,Ho,nt,Wo,q,lt,Ea,Vt,yn="Removes any hook attached to a module via <code>add_hook_to_module</code>.",Do,st,Zo,S,dt,za,Pt,wn="Recursively removes all hooks attached on the submodules of a given model.",Bo,rt,Lo,ct,Ao,Y,it,Qa,Rt,xn=`Checks if a module has offloaded parameters by checking if the given module has a AlignDevicesHook attached with | |
| offloading enabled`,Vo,mt,Po,O,ht,qa,Xt,Mn="Context manager that moves a module’s parameters to the specified execution device.",Ro,pt,Xo,zt,Eo;return $=new Gn({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),T=new x({props:{title:"Working with large models",local:"working-with-large-models",headingTag:"h1"}}),ue=new x({props:{title:"Dispatch and offload",local:"dispatch-and-offload",headingTag:"h2"}}),ge=new x({props:{title:"init_empty_weights",local:"accelerate.init_empty_weights",headingTag:"h3"}}),ve=new M({props:{name:"accelerate.init_empty_weights",anchor:"accelerate.init_empty_weights",parameters:[{name:"include_buffers",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"accelerate.init_empty_weights.include_buffers",description:`<strong>include_buffers</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to also put all buffers on the meta device while initializing.`,name:"include_buffers"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/big_modeling.py#L61"}}),ae=new Qt({props:{anchor:"accelerate.init_empty_weights.example",$$slots:{default:[Nn]},$$scope:{ctx:C}}}),ne=new da({props:{warning:!0,$$slots:{default:[Fn]},$$scope:{ctx:C}}}),be=new x({props:{title:"cpu_offload",local:"accelerate.cpu_offload",headingTag:"h3"}}),ke=new M({props:{name:"accelerate.cpu_offload",anchor:"accelerate.cpu_offload",parameters:[{name:"model",val:": Module"},{name:"execution_device",val:": typing.Optional[torch.device] = None"},{name:"offload_buffers",val:": bool = False"},{name:"state_dict",val:": typing.Optional[dict[str, torch.Tensor]] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"}],parametersDescription:[{anchor:"accelerate.cpu_offload.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model to offload.`,name:"model"},{anchor:"accelerate.cpu_offload.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) — | |
| The device on which the forward pass of the model will be executed (should be a GPU). Will default to the | |
| model first parameter device.`,name:"execution_device"},{anchor:"accelerate.cpu_offload.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to offload the buffers with the model parameters.`,name:"offload_buffers"},{anchor:"accelerate.cpu_offload.state_dict",description:`<strong>state_dict</strong> (<code>Dict[str, torch.Tensor]</code>, <em>optional</em>) — | |
| The state dict of the model that will be kept on CPU.`,name:"state_dict"},{anchor:"accelerate.cpu_offload.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of classes whose instances should load all their weights (even in the submodules) at the beginning | |
| of the forward. This should only be used for classes that have submodules which are registered but not | |
| called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward, | |
| <code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/big_modeling.py#L179"}}),$e=new x({props:{title:"cpu_offload_with_hook",local:"accelerate.cpu_offload_with_hook",headingTag:"h3"}}),ye=new M({props:{name:"accelerate.cpu_offload_with_hook",anchor:"accelerate.cpu_offload_with_hook",parameters:[{name:"model",val:": Module"},{name:"execution_device",val:": typing.Union[str, torch.device, int, NoneType] = None"},{name:"prev_module_hook",val:": typing.Optional[accelerate.hooks.UserCpuOffloadHook] = None"}],parametersDescription:[{anchor:"accelerate.cpu_offload_with_hook.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model to offload.`,name:"model"},{anchor:"accelerate.cpu_offload_with_hook.execution_device(str,",description:`<strong>execution_device(<code>str</code>,</strong> <code>int</code> or <code>torch.device</code>, <em>optional</em>) — | |
| The device on which the model should be executed. Will default to the MPS device if it’s available, then | |
| device 0 if there is an accelerator device, and finally to the CPU.`,name:"execution_device(str,"},{anchor:"accelerate.cpu_offload_with_hook.prev_module_hook",description:`<strong>prev_module_hook</strong> (<code>UserCpuOffloadHook</code>, <em>optional</em>) — | |
| The hook sent back by this function for a previous model in the pipeline you are running. If passed, its | |
| offload method will be called just before the forward of the model to which this hook is attached.`,name:"prev_module_hook"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/big_modeling.py#L225"}}),le=new Qt({props:{anchor:"accelerate.cpu_offload_with_hook.example",$$slots:{default:[Hn]},$$scope:{ctx:C}}}),we=new x({props:{title:"disk_offload",local:"accelerate.disk_offload",headingTag:"h3"}}),xe=new M({props:{name:"accelerate.disk_offload",anchor:"accelerate.disk_offload",parameters:[{name:"model",val:": Module"},{name:"offload_dir",val:": typing.Union[str, os.PathLike]"},{name:"execution_device",val:": typing.Optional[torch.device] = None"},{name:"offload_buffers",val:": bool = False"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"}],parametersDescription:[{anchor:"accelerate.disk_offload.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) — The model to offload.",name:"model"},{anchor:"accelerate.disk_offload.offload_dir",description:`<strong>offload_dir</strong> (<code>str</code> or <code>os.PathLike</code>) — | |
| The folder in which to offload the model weights (or where the model weights are already offloaded).`,name:"offload_dir"},{anchor:"accelerate.disk_offload.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) — | |
| The device on which the forward pass of the model will be executed (should be a GPU). Will default to the | |
| model’s first parameter device.`,name:"execution_device"},{anchor:"accelerate.disk_offload.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to offload the buffers with the model parameters.`,name:"offload_buffers"},{anchor:"accelerate.disk_offload.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of classes whose instances should load all their weights (even in the submodules) at the beginning | |
| of the forward. This should only be used for classes that have submodules which are registered but not | |
| called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward, | |
| <code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/big_modeling.py#L269"}}),Me=new x({props:{title:"dispatch_model",local:"accelerate.dispatch_model",headingTag:"h3"}}),Te=new M({props:{name:"accelerate.dispatch_model",anchor:"accelerate.dispatch_model",parameters:[{name:"model",val:": Module"},{name:"device_map",val:": dict"},{name:"main_device",val:": typing.Optional[torch.device] = None"},{name:"state_dict",val:": typing.Optional[dict[str, torch.Tensor]] = None"},{name:"offload_dir",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"offload_index",val:": typing.Optional[dict[str, str]] = None"},{name:"offload_buffers",val:": bool = False"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"force_hooks",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.dispatch_model.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model to dispatch.`,name:"model"},{anchor:"accelerate.dispatch_model.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[str, int, torch.device]]</code>) — | |
| A dictionary mapping module names in the models <code>state_dict</code> to the device they should go to. Note that | |
| <code>"disk"</code> is accepted even if it’s not a proper value for <code>torch.device</code>.`,name:"device_map"},{anchor:"accelerate.dispatch_model.main_device",description:`<strong>main_device</strong> (<code>str</code>, <code>int</code> or <code>torch.device</code>, <em>optional</em>) — | |
| The main execution device. Will default to the first device in the <code>device_map</code> different from <code>"cpu"</code> or | |
| <code>"disk"</code>.`,name:"main_device"},{anchor:"accelerate.dispatch_model.state_dict",description:`<strong>state_dict</strong> (<code>Dict[str, torch.Tensor]</code>, <em>optional</em>) — | |
| The state dict of the part of the model that will be kept on CPU.`,name:"state_dict"},{anchor:"accelerate.dispatch_model.offload_dir",description:`<strong>offload_dir</strong> (<code>str</code> or <code>os.PathLike</code>) — | |
| The folder in which to offload the model weights (or where the model weights are already offloaded).`,name:"offload_dir"},{anchor:"accelerate.dispatch_model.offload_index",description:`<strong>offload_index</strong> (<code>Dict</code>, <em>optional</em>) — | |
| A dictionary from weight name to their information (<code>dtype</code>/ <code>shape</code> or safetensors filename). Will default | |
| to the index saved in <code>save_folder</code>.`,name:"offload_index"},{anchor:"accelerate.dispatch_model.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to offload the buffers with the model parameters.`,name:"offload_buffers"},{anchor:"accelerate.dispatch_model.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) — | |
| A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.dispatch_model.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of classes whose instances should load all their weights (even in the submodules) at the beginning | |
| of the forward. This should only be used for classes that have submodules which are registered but not | |
| called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward, | |
| <code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.dispatch_model.force_hooks",description:`<strong>force_hooks</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to force device hooks to be attached to the model even if all layers are dispatched to a | |
| single device.`,name:"force_hooks"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/big_modeling.py#L315"}}),Ue=new x({props:{title:"load_checkpoint_and_dispatch",local:"accelerate.load_checkpoint_and_dispatch",headingTag:"h3"}}),Ce=new M({props:{name:"accelerate.load_checkpoint_and_dispatch",anchor:"accelerate.load_checkpoint_and_dispatch",parameters:[{name:"model",val:": Module"},{name:"checkpoint",val:": typing.Union[str, os.PathLike]"},{name:"device_map",val:": typing.Union[str, dict[str, typing.Union[int, str, torch.device]], NoneType] = None"},{name:"max_memory",val:": typing.Optional[dict[typing.Union[int, str], typing.Union[int, str]]] = None"},{name:"no_split_module_classes",val:": typing.Optional[list[str]] = None"},{name:"offload_folder",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"offload_buffers",val:": bool = False"},{name:"dtype",val:": typing.Union[str, torch.dtype, NoneType] = None"},{name:"offload_state_dict",val:": typing.Optional[bool] = None"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"force_hooks",val:": bool = False"},{name:"strict",val:": bool = False"},{name:"full_state_dict",val:": bool = True"},{name:"broadcast_from_rank0",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.load_checkpoint_and_dispatch.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) — The model in which we want to load a checkpoint.",name:"model"},{anchor:"accelerate.load_checkpoint_and_dispatch.checkpoint",description:`<strong>checkpoint</strong> (<code>str</code> or <code>os.PathLike</code>) — | |
| The folder checkpoint to load. It can be:</p> | |
| <ul> | |
| <li>a path to a file containing a whole model state dict</li> | |
| <li>a path to a <code>.json</code> file containing the index to a sharded checkpoint</li> | |
| <li>a path to a folder containing a unique <code>.index.json</code> file and the shards of a checkpoint.</li> | |
| </ul>`,name:"checkpoint"},{anchor:"accelerate.load_checkpoint_and_dispatch.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[int, str, torch.device]]</code>, <em>optional</em>) — | |
| A map that specifies where each submodule should go. It doesn’t need to be refined to each parameter/buffer | |
| name, once a given module name is inside, every submodule of it will be sent to the same device.</p> | |
| <p>To have Accelerate compute the most optimized <code>device_map</code> automatically, set <code>device_map="auto"</code>. For more | |
| information about each option see <a href="../concept_guides/big_model_inference#designing-a-device-map">here</a>. | |
| Defaults to None, which means <a href="/docs/accelerate/pr_4021/en/package_reference/big_modeling#accelerate.dispatch_model">dispatch_model()</a> will not be called.`,name:"device_map"},{anchor:"accelerate.load_checkpoint_and_dispatch.max_memory",description:`<strong>max_memory</strong> (<code>Dict</code>, <em>optional</em>) — | |
| A dictionary device identifier to maximum memory. Will default to the maximum memory available for each GPU | |
| and the available CPU RAM if unset.`,name:"max_memory"},{anchor:"accelerate.load_checkpoint_and_dispatch.no_split_module_classes",description:`<strong>no_split_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of layer class names that should never be split across device (for instance any layer that has a | |
| residual connection).`,name:"no_split_module_classes"},{anchor:"accelerate.load_checkpoint_and_dispatch.offload_folder",description:`<strong>offload_folder</strong> (<code>str</code> or <code>os.PathLike</code>, <em>optional</em>) — | |
| If the <code>device_map</code> contains any value <code>"disk"</code>, the folder where we will offload weights.`,name:"offload_folder"},{anchor:"accelerate.load_checkpoint_and_dispatch.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as | |
| well as the parameters.`,name:"offload_buffers"},{anchor:"accelerate.load_checkpoint_and_dispatch.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) — | |
| If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.load_checkpoint_and_dispatch.offload_state_dict",description:`<strong>offload_state_dict</strong> (<code>bool</code>, <em>optional</em>) — | |
| If <code>True</code>, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if | |
| the weight of the CPU state dict + the biggest shard does not fit. Will default to <code>True</code> if the device map | |
| picked contains <code>"disk"</code> values.`,name:"offload_state_dict"},{anchor:"accelerate.load_checkpoint_and_dispatch.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) — | |
| A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.load_checkpoint_and_dispatch.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of classes whose instances should load all their weights (even in the submodules) at the beginning | |
| of the forward. This should only be used for classes that have submodules which are registered but not | |
| called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward, | |
| <code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.load_checkpoint_and_dispatch.force_hooks",description:`<strong>force_hooks</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to force device hooks to be attached to the model even if all layers are dispatched to a | |
| single device.`,name:"force_hooks"},{anchor:"accelerate.load_checkpoint_and_dispatch.strict",description:`<strong>strict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model’s | |
| state_dict.`,name:"strict"},{anchor:"accelerate.load_checkpoint_and_dispatch.full_state_dict",description:`<strong>full_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — if this is set to <code>True</code>, all the tensors in the | |
| loaded state_dict will be gathered. No ShardedTensor and DTensor will be in the loaded state_dict.`,name:"full_state_dict"},{anchor:"accelerate.load_checkpoint_and_dispatch.broadcast_from_rank0",description:`<strong>broadcast_from_rank0</strong> (<code>False</code>, <em>optional</em>, defaults to <code>False</code>) — when the option is <code>True</code>, a distributed | |
| <code>ProcessGroup</code> must be initialized. rank0 should receive a full state_dict and will broadcast the tensors | |
| in the state_dict one by one to other ranks. Other ranks will receive the tensors and shard (if applicable) | |
| according to the local shards in the model.`,name:"broadcast_from_rank0"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/big_modeling.py#L522"}}),se=new Qt({props:{anchor:"accelerate.load_checkpoint_and_dispatch.example",$$slots:{default:[Wn]},$$scope:{ctx:C}}}),je=new x({props:{title:"load_checkpoint_in_model",local:"accelerate.load_checkpoint_in_model",headingTag:"h3"}}),Je=new M({props:{name:"accelerate.load_checkpoint_in_model",anchor:"accelerate.load_checkpoint_in_model",parameters:[{name:"model",val:": Module"},{name:"checkpoint",val:": typing.Union[str, os.PathLike]"},{name:"device_map",val:": typing.Optional[dict[str, typing.Union[int, str, torch.device]]] = None"},{name:"offload_folder",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"dtype",val:": typing.Union[str, torch.dtype, NoneType] = None"},{name:"offload_state_dict",val:": bool = False"},{name:"offload_buffers",val:": bool = False"},{name:"keep_in_fp32_modules",val:": typing.Optional[list[str]] = None"},{name:"offload_8bit_bnb",val:": bool = False"},{name:"strict",val:": bool = False"},{name:"full_state_dict",val:": bool = True"},{name:"broadcast_from_rank0",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.load_checkpoint_in_model.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model in which we want to load a checkpoint.`,name:"model"},{anchor:"accelerate.load_checkpoint_in_model.checkpoint",description:`<strong>checkpoint</strong> (<code>str</code> or <code>os.PathLike</code>) — | |
| The folder checkpoint to load. It can be: | |
| <ul> | |
| <li>a path to a file containing a whole model state dict</li> | |
| <li>a path to a <code>.json</code> file containing the index to a sharded checkpoint</li> | |
| <li>a path to a folder containing a unique <code>.index.json</code> file and the shards of a checkpoint.</li> | |
| <li>a path to a folder containing a unique pytorch_model.bin or a model.safetensors file.</li> | |
| </ul>`,name:"checkpoint"},{anchor:"accelerate.load_checkpoint_in_model.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[int, str, torch.device]]</code>, <em>optional</em>) — | |
| A map that specifies where each submodule should go. It doesn’t need to be refined to each parameter/buffer | |
| name, once a given module name is inside, every submodule of it will be sent to the same device.`,name:"device_map"},{anchor:"accelerate.load_checkpoint_in_model.offload_folder",description:`<strong>offload_folder</strong> (<code>str</code> or <code>os.PathLike</code>, <em>optional</em>) — | |
| If the <code>device_map</code> contains any value <code>"disk"</code>, the folder where we will offload weights.`,name:"offload_folder"},{anchor:"accelerate.load_checkpoint_in_model.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) — | |
| If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.load_checkpoint_in_model.offload_state_dict",description:`<strong>offload_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| If <code>True</code>, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if | |
| the weight of the CPU state dict + the biggest shard does not fit.`,name:"offload_state_dict"},{anchor:"accelerate.load_checkpoint_in_model.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to include the buffers in the weights offloaded to disk.`,name:"offload_buffers"},{anchor:"accelerate.load_checkpoint_in_model.keep_in_fp32_modules(List[str],",description:`<strong>keep_in_fp32_modules(<code>List[str]</code>,</strong> <em>optional</em>) — | |
| A list of the modules that we keep in <code>torch.float32</code> dtype.`,name:"keep_in_fp32_modules(List[str],"},{anchor:"accelerate.load_checkpoint_in_model.offload_8bit_bnb",description:`<strong>offload_8bit_bnb</strong> (<code>bool</code>, <em>optional</em>) — | |
| Whether or not to enable offload of 8-bit modules on cpu/disk.`,name:"offload_8bit_bnb"},{anchor:"accelerate.load_checkpoint_in_model.strict",description:`<strong>strict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model’s | |
| state_dict.`,name:"strict"},{anchor:"accelerate.load_checkpoint_in_model.full_state_dict",description:`<strong>full_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — if this is set to <code>True</code>, all the tensors in the | |
| loaded state_dict will be gathered. No ShardedTensor and DTensor will be in the loaded state_dict.`,name:"full_state_dict"},{anchor:"accelerate.load_checkpoint_in_model.broadcast_from_rank0",description:`<strong>broadcast_from_rank0</strong> (<code>False</code>, <em>optional</em>, defaults to <code>False</code>) — when the option is <code>True</code>, a distributed | |
| <code>ProcessGroup</code> must be initialized. rank0 should receive a full state_dict and will broadcast the tensors | |
| in the state_dict one by one to other ranks. Other ranks will receive the tensors and shard (if applicable) | |
| according to the local shards in the model.`,name:"broadcast_from_rank0"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/utils/modeling.py#L1791"}}),de=new da({props:{warning:!0,$$slots:{default:[Dn]},$$scope:{ctx:C}}}),Ge=new x({props:{title:"infer_auto_device_map",local:"accelerate.infer_auto_device_map",headingTag:"h3"}}),Ie=new M({props:{name:"accelerate.infer_auto_device_map",anchor:"accelerate.infer_auto_device_map",parameters:[{name:"model",val:": Module"},{name:"max_memory",val:": typing.Optional[dict[typing.Union[int, str], typing.Union[int, str]]] = None"},{name:"no_split_module_classes",val:": typing.Optional[list[str]] = None"},{name:"dtype",val:": typing.Union[str, torch.dtype, NoneType] = None"},{name:"special_dtypes",val:": typing.Optional[dict[str, typing.Union[str, torch.dtype]]] = None"},{name:"verbose",val:": bool = False"},{name:"clean_result",val:": bool = True"},{name:"offload_buffers",val:": bool = False"},{name:"fallback_allocation",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.infer_auto_device_map.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) — | |
| The model to analyze.`,name:"model"},{anchor:"accelerate.infer_auto_device_map.max_memory",description:`<strong>max_memory</strong> (<code>Dict</code>, <em>optional</em>) — | |
| A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset. | |
| Example: <code>max_memory={0: "1GB"}</code>.`,name:"max_memory"},{anchor:"accelerate.infer_auto_device_map.no_split_module_classes",description:`<strong>no_split_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of layer class names that should never be split across device (for instance any layer that has a | |
| residual connection).`,name:"no_split_module_classes"},{anchor:"accelerate.infer_auto_device_map.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) — | |
| If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.infer_auto_device_map.special_dtypes",description:`<strong>special_dtypes</strong> (<code>Dict[str, Union[str, torch.device]]</code>, <em>optional</em>) — | |
| If provided, special dtypes to consider for some specific weights (will override dtype used as default for | |
| all weights).`,name:"special_dtypes"},{anchor:"accelerate.infer_auto_device_map.verbose",description:`<strong>verbose</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to provide debugging statements as the function builds the device_map.`,name:"verbose"},{anchor:"accelerate.infer_auto_device_map.clean_result",description:`<strong>clean_result</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) — | |
| Clean the resulting device_map by grouping all submodules that go on the same device together.`,name:"clean_result"},{anchor:"accelerate.infer_auto_device_map.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as | |
| well as the parameters.`,name:"offload_buffers"},{anchor:"accelerate.infer_auto_device_map.fallback_allocation",description:`<strong>fallback_allocation</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| When regular allocation fails, try to allocate a module that fits in the size limit using BFS.`,name:"fallback_allocation"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/utils/modeling.py#L1281"}}),re=new da({props:{$$slots:{default:[Zn]},$$scope:{ctx:C}}}),Ne=new x({props:{title:"Hooks",local:"hooks",headingTag:"h2"}}),Fe=new x({props:{title:"ModelHook",local:"accelerate.hooks.ModelHook",headingTag:"h3"}}),He=new M({props:{name:"class accelerate.hooks.ModelHook",anchor:"accelerate.hooks.ModelHook",parameters:[],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L58"}}),We=new M({props:{name:"detach_hook",anchor:"accelerate.hooks.ModelHook.detach_hook",parameters:[{name:"module",val:""}],parametersDescription:[{anchor:"accelerate.hooks.ModelHook.detach_hook.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) — The module detached from this hook.",name:"module"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L106"}}),De=new M({props:{name:"init_hook",anchor:"accelerate.hooks.ModelHook.init_hook",parameters:[{name:"module",val:""}],parametersDescription:[{anchor:"accelerate.hooks.ModelHook.init_hook.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) — The module attached to this hook.",name:"module"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L70"}}),Ze=new M({props:{name:"post_forward",anchor:"accelerate.hooks.ModelHook.post_forward",parameters:[{name:"module",val:""},{name:"output",val:""}],parametersDescription:[{anchor:"accelerate.hooks.ModelHook.post_forward.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) — The module whose forward pass been executed just before this event.",name:"module"},{anchor:"accelerate.hooks.ModelHook.post_forward.output",description:"<strong>output</strong> (<code>Any</code>) — The output of the module.",name:"output"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L93",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The processed <code>output</code>.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>Any</code></p> | |
| `}}),Be=new M({props:{name:"pre_forward",anchor:"accelerate.hooks.ModelHook.pre_forward",parameters:[{name:"module",val:""},{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"accelerate.hooks.ModelHook.pre_forward.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) — The module whose forward pass will be executed just after this event.",name:"module"},{anchor:"accelerate.hooks.ModelHook.pre_forward.args",description:"<strong>args</strong> (<code>Tuple[Any]</code>) — The positional arguments passed to the module.",name:"args"},{anchor:"accelerate.hooks.ModelHook.pre_forward.kwargs",description:"<strong>kwargs</strong> (<code>Dict[Str, Any]</code>) — The keyword arguments passed to the module.",name:"kwargs"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L79",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>A tuple with the treated <code>args</code> and <code>kwargs</code>.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>Tuple[Tuple[Any], Dict[Str, Any]]</code></p> | |
| `}}),Le=new x({props:{title:"AlignDevicesHook",local:"accelerate.hooks.AlignDevicesHook",headingTag:"h3"}}),Ae=new M({props:{name:"class accelerate.hooks.AlignDevicesHook",anchor:"accelerate.hooks.AlignDevicesHook",parameters:[{name:"execution_device",val:": typing.Union[str, torch.device, int, NoneType] = None"},{name:"offload",val:": bool = False"},{name:"io_same_device",val:": bool = False"},{name:"weights_map",val:": typing.Optional[collections.abc.Mapping] = None"},{name:"offload_buffers",val:": bool = False"},{name:"place_submodules",val:": bool = False"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"tied_params_map",val:": typing.Optional[dict[int, dict[torch.device, torch.Tensor]]] = None"}],parametersDescription:[{anchor:"accelerate.hooks.AlignDevicesHook.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) — | |
| The device on which inputs and model weights should be placed before the forward pass.`,name:"execution_device"},{anchor:"accelerate.hooks.AlignDevicesHook.offload",description:`<strong>offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the weights should be offloaded after the forward pass.`,name:"offload"},{anchor:"accelerate.hooks.AlignDevicesHook.io_same_device",description:`<strong>io_same_device</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the output should be placed on the same device as the input was.`,name:"io_same_device"},{anchor:"accelerate.hooks.AlignDevicesHook.weights_map",description:`<strong>weights_map</strong> (<code>Mapping[str, torch.Tensor]</code>, <em>optional</em>) — | |
| When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.`,name:"weights_map"},{anchor:"accelerate.hooks.AlignDevicesHook.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to include the associated module’s buffers when offloading.`,name:"offload_buffers"},{anchor:"accelerate.hooks.AlignDevicesHook.place_submodules",description:`<strong>place_submodules</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether to place the submodules on <code>execution_device</code> during the <code>init_hook</code> event.`,name:"place_submodules"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L242"}}),Ve=new x({props:{title:"SequentialHook",local:"accelerate.hooks.SequentialHook",headingTag:"h3"}}),Pe=new M({props:{name:"class accelerate.hooks.SequentialHook",anchor:"accelerate.hooks.SequentialHook",parameters:[{name:"*hooks",val:""}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L116"}}),Re=new x({props:{title:"LayerwiseCastingHook",local:"accelerate.hooks.LayerwiseCastingHook",headingTag:"h3"}}),Xe=new M({props:{name:"class accelerate.hooks.LayerwiseCastingHook",anchor:"accelerate.hooks.LayerwiseCastingHook",parameters:[{name:"storage_dtype",val:": dtype"},{name:"compute_dtype",val:": dtype"},{name:"non_blocking",val:": bool"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L772"}}),Ee=new x({props:{title:"Adding Hooks",local:"adding-hooks",headingTag:"h2"}}),ze=new x({props:{title:"add_hook_to_module",local:"accelerate.hooks.add_hook_to_module",headingTag:"h3"}}),Qe=new M({props:{name:"accelerate.hooks.add_hook_to_module",anchor:"accelerate.hooks.add_hook_to_module",parameters:[{name:"module",val:": Module"},{name:"hook",val:": ModelHook"},{name:"append",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.hooks.add_hook_to_module.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) — | |
| The module to attach a hook to.`,name:"module"},{anchor:"accelerate.hooks.add_hook_to_module.hook",description:`<strong>hook</strong> (<code>ModelHook</code>) — | |
| The hook to attach.`,name:"hook"},{anchor:"accelerate.hooks.add_hook_to_module.append",description:`<strong>append</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether the hook should be chained with an existing one (if module already contains a hook) or not.`,name:"append"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L147",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same module, with the hook attached (the module is modified in place, so the result can | |
| be discarded).</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>torch.nn.Module</code></p> | |
| `}}),pe=new da({props:{warning:!0,$$slots:{default:[Bn]},$$scope:{ctx:C}}}),qe=new x({props:{title:"attach_execution_device_hook",local:"accelerate.hooks.attach_execution_device_hook",headingTag:"h3"}}),Se=new M({props:{name:"accelerate.hooks.attach_execution_device_hook",anchor:"accelerate.hooks.attach_execution_device_hook",parameters:[{name:"module",val:": Module"},{name:"execution_device",val:": typing.Union[int, str, torch.device]"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"tied_params_map",val:": typing.Optional[dict[int, dict[torch.device, torch.Tensor]]] = None"}],parametersDescription:[{anchor:"accelerate.hooks.attach_execution_device_hook.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) — | |
| The module where we want to attach the hooks.`,name:"module"},{anchor:"accelerate.hooks.attach_execution_device_hook.execution_device",description:`<strong>execution_device</strong> (<code>int</code>, <code>str</code> or <code>torch.device</code>) — | |
| The device on which inputs and model weights should be placed before the forward pass.`,name:"execution_device"},{anchor:"accelerate.hooks.attach_execution_device_hook.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) — | |
| A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.hooks.attach_execution_device_hook.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of classes whose instances should load all their weights (even in the submodules) at the beginning | |
| of the forward. This should only be used for classes that have submodules which are registered but not | |
| called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward, | |
| <code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.hooks.attach_execution_device_hook.tied_params_map",description:`<strong>tied_params_map</strong> (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], <em>optional</em>, defaults to <code>None</code>) — | |
| A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution | |
| device, this parameter is useful to reuse the first available pointer of a shared weight for all others, | |
| instead of duplicating memory.`,name:"tied_params_map"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L431"}}),Ye=new x({props:{title:"attach_align_device_hook",local:"accelerate.hooks.attach_align_device_hook",headingTag:"h3"}}),Oe=new M({props:{name:"accelerate.hooks.attach_align_device_hook",anchor:"accelerate.hooks.attach_align_device_hook",parameters:[{name:"module",val:": Module"},{name:"execution_device",val:": typing.Optional[torch.device] = None"},{name:"offload",val:": bool = False"},{name:"weights_map",val:": typing.Optional[collections.abc.Mapping] = None"},{name:"offload_buffers",val:": bool = False"},{name:"module_name",val:": str = ''"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"tied_params_map",val:": typing.Optional[dict[int, dict[torch.device, torch.Tensor]]] = None"}],parametersDescription:[{anchor:"accelerate.hooks.attach_align_device_hook.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) — | |
| The module where we want to attach the hooks.`,name:"module"},{anchor:"accelerate.hooks.attach_align_device_hook.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) — | |
| The device on which inputs and model weights should be placed before the forward pass.`,name:"execution_device"},{anchor:"accelerate.hooks.attach_align_device_hook.offload",description:`<strong>offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the weights should be offloaded after the forward pass.`,name:"offload"},{anchor:"accelerate.hooks.attach_align_device_hook.weights_map",description:`<strong>weights_map</strong> (<code>Mapping[str, torch.Tensor]</code>, <em>optional</em>) — | |
| When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.`,name:"weights_map"},{anchor:"accelerate.hooks.attach_align_device_hook.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to include the associated module’s buffers when offloading.`,name:"offload_buffers"},{anchor:"accelerate.hooks.attach_align_device_hook.module_name",description:`<strong>module_name</strong> (<code>str</code>, <em>optional</em>, defaults to <code>""</code>) — | |
| The name of the module.`,name:"module_name"},{anchor:"accelerate.hooks.attach_align_device_hook.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) — | |
| A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.hooks.attach_align_device_hook.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of classes whose instances should load all their weights (even in the submodules) at the beginning | |
| of the forward. This should only be used for classes that have submodules which are registered but not | |
| called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward, | |
| <code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.hooks.attach_align_device_hook.tied_params_map",description:`<strong>tied_params_map</strong> (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], <em>optional</em>, defaults to <code>None</code>) — | |
| A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution | |
| device, this parameter is useful to reuse the first available pointer of a shared weight for all others, | |
| instead of duplicating memory.`,name:"tied_params_map"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L479"}}),Ke=new x({props:{title:"attach_align_device_hook_on_blocks",local:"accelerate.hooks.attach_align_device_hook_on_blocks",headingTag:"h3"}}),et=new M({props:{name:"accelerate.hooks.attach_align_device_hook_on_blocks",anchor:"accelerate.hooks.attach_align_device_hook_on_blocks",parameters:[{name:"module",val:": Module"},{name:"execution_device",val:": typing.Union[torch.device, dict[str, torch.device], NoneType] = None"},{name:"offload",val:": typing.Union[bool, dict[str, bool]] = False"},{name:"weights_map",val:": typing.Optional[collections.abc.Mapping] = None"},{name:"offload_buffers",val:": bool = False"},{name:"module_name",val:": str = ''"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"tied_params_map",val:": typing.Optional[dict[int, dict[torch.device, torch.Tensor]]] = None"}],parametersDescription:[{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) — | |
| The module where we want to attach the hooks.`,name:"module"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code> or <code>Dict[str, torch.device]</code>, <em>optional</em>) — | |
| The device on which inputs and model weights should be placed before the forward pass. It can be one device | |
| for the whole module, or a dictionary mapping module name to device.`,name:"execution_device"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.offload",description:`<strong>offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not the weights should be offloaded after the forward pass. It can be one boolean for the whole | |
| module, or a dictionary mapping module name to boolean.`,name:"offload"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.weights_map",description:`<strong>weights_map</strong> (<code>Mapping[str, torch.Tensor]</code>, <em>optional</em>) — | |
| When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.`,name:"weights_map"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — | |
| Whether or not to include the associated module’s buffers when offloading.`,name:"offload_buffers"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.module_name",description:`<strong>module_name</strong> (<code>str</code>, <em>optional</em>, defaults to <code>""</code>) — | |
| The name of the module.`,name:"module_name"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) — | |
| A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) — | |
| A list of classes whose instances should load all their weights (even in the submodules) at the beginning | |
| of the forward. This should only be used for classes that have submodules which are registered but not | |
| called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward, | |
| <code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.tied_params_map",description:`<strong>tied_params_map</strong> (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], <em>optional</em>, defaults to <code>None</code>) — | |
| A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution | |
| device, this parameter is useful to reuse the first available pointer of a shared weight for all others, | |
| instead of duplicating memory.`,name:"tied_params_map"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L574"}}),tt=new x({props:{title:"attach_layerwise_casting_hooks",local:"accelerate.big_modeling.attach_layerwise_casting_hooks",headingTag:"h3"}}),ot=new M({props:{name:"accelerate.big_modeling.attach_layerwise_casting_hooks",anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks",parameters:[{name:"module",val:": Module"},{name:"storage_dtype",val:": dtype"},{name:"compute_dtype",val:": dtype"},{name:"skip_modules_pattern",val:": typing.Union[str, tuple[str, ...], NoneType] = None"},{name:"skip_modules_classes",val:": typing.Optional[tuple[type[torch.nn.modules.module.Module], ...]] = None"},{name:"non_blocking",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) — | |
| The module whose leaf modules will be cast to a high precision dtype for computation, and to a low | |
| precision dtype for storage.`,name:"module"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.storage_dtype",description:`<strong>storage_dtype</strong> (<code>torch.dtype</code>) — | |
| The dtype to cast the module to before/after the forward pass for storage.`,name:"storage_dtype"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.compute_dtype",description:`<strong>compute_dtype</strong> (<code>torch.dtype</code>) — | |
| The dtype to cast the module to during the forward pass for computation.`,name:"compute_dtype"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.skip_modules_pattern",description:`<strong>skip_modules_pattern</strong> (<code>tuple[str, ...]</code>, defaults to <code>None</code>) — | |
| A list of patterns to match the names of the modules to skip during the layerwise casting process. If set | |
| to <code>None</code> alongside <code>skip_modules_classes</code> being <code>None</code>, the layerwise casting is applied directly to the | |
| module instead of its internal submodules.`,name:"skip_modules_pattern"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.skip_modules_classes",description:`<strong>skip_modules_classes</strong> (<code>tuple[type[torch.nn.Module], ...]</code>, defaults to <code>None</code>) — | |
| A list of module classes to skip during the layerwise casting process.`,name:"skip_modules_classes"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.non_blocking",description:`<strong>non_blocking</strong> (<code>bool</code>, defaults to <code>False</code>) — | |
| If <code>True</code>, the weight casting operations are non-blocking.`,name:"non_blocking"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/big_modeling.py#L663"}}),fe=new Qt({props:{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.example",$$slots:{default:[Ln]},$$scope:{ctx:C}}}),_e=new Qt({props:{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.example-2",$$slots:{default:[An]},$$scope:{ctx:C}}}),at=new x({props:{title:"Removing Hooks",local:"removing-hooks",headingTag:"h2"}}),nt=new x({props:{title:"remove_hook_from_module",local:"accelerate.hooks.remove_hook_from_module",headingTag:"h3"}}),lt=new M({props:{name:"accelerate.hooks.remove_hook_from_module",anchor:"accelerate.hooks.remove_hook_from_module",parameters:[{name:"module",val:": Module"},{name:"recurse",val:" = False"}],parametersDescription:[{anchor:"accelerate.hooks.remove_hook_from_module.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) — The module to attach a hook to.",name:"module"},{anchor:"accelerate.hooks.remove_hook_from_module.recurse",description:"<strong>recurse</strong> (<code>bool</code>, <strong>optional</strong>) — Whether to remove the hooks recursively",name:"recurse"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L205",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>The same module, with the hook detached (the module is modified in place, so the result can | |
| be discarded).</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>torch.nn.Module</code></p> | |
| `}}),st=new x({props:{title:"remove_hook_from_submodules",local:"accelerate.hooks.remove_hook_from_submodules",headingTag:"h3"}}),dt=new M({props:{name:"accelerate.hooks.remove_hook_from_submodules",anchor:"accelerate.hooks.remove_hook_from_submodules",parameters:[{name:"module",val:": Module"}],parametersDescription:[{anchor:"accelerate.hooks.remove_hook_from_submodules.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) — The module on which to remove all hooks.",name:"module"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/hooks.py#L562"}}),rt=new x({props:{title:"Utilities",local:"utilities",headingTag:"h2"}}),ct=new x({props:{title:"has_offloaded_params",local:"accelerate.utils.has_offloaded_params",headingTag:"h3"}}),it=new M({props:{name:"accelerate.utils.has_offloaded_params",anchor:"accelerate.utils.has_offloaded_params",parameters:[{name:"module",val:": Module"}],parametersDescription:[{anchor:"accelerate.utils.has_offloaded_params.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) — The module to check for an offload hook.",name:"module"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/utils/modeling.py#L2138",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p><code>True</code> if the module has an offload hook and offloading is enabled, <code>False</code> otherwise.</p> | |
| `,returnType:`<script context="module">export const metadata = 'undefined';<\/script> | |
| <p>bool</p> | |
| `}}),mt=new x({props:{title:"align_module_device",local:"accelerate.utils.align_module_device",headingTag:"h3"}}),ht=new M({props:{name:"accelerate.utils.align_module_device",anchor:"accelerate.utils.align_module_device",parameters:[{name:"module",val:": Module"},{name:"execution_device",val:": typing.Optional[torch.device] = None"}],parametersDescription:[{anchor:"accelerate.utils.align_module_device.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) — | |
| Module with parameters to align.`,name:"module"},{anchor:"accelerate.utils.align_module_device.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) — | |
| If provided, overrides the module’s execution device within the context. Otherwise, use hook execution | |
| device or pass`,name:"execution_device"}],source:"https://github.com/huggingface/accelerate/blob/vr_4021/src/accelerate/utils/modeling.py#L2154"}}),pt=new In({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/package_reference/big_modeling.md"}}),{c(){s=f("meta"),k=a(),g=f("p"),v=a(),r($.$$.fragment),u=a(),r(T.$$.fragment),St=a(),r(ue.$$.fragment),Yt=a(),r(ge.$$.fragment),Ot=a(),G=f("div"),r(ve.$$.fragment),ra=a(),gt=f("p"),gt.textContent=Sa,ca=a(),r(ae.$$.fragment),ia=a(),r(ne.$$.fragment),Kt=a(),r(be.$$.fragment),eo=a(),L=f("div"),r(ke.$$.fragment),ma=a(),vt=f("p"),vt.textContent=Ya,to=a(),r($e.$$.fragment),oo=a(),N=f("div"),r(ye.$$.fragment),ha=a(),bt=f("p"),bt.innerHTML=Oa,pa=a(),r(le.$$.fragment),ao=a(),r(we.$$.fragment),no=a(),A=f("div"),r(xe.$$.fragment),fa=a(),kt=f("p"),kt.textContent=Ka,lo=a(),r(Me.$$.fragment),so=a(),V=f("div"),r(Te.$$.fragment),_a=a(),$t=f("p"),$t.textContent=en,ro=a(),r(Ue.$$.fragment),co=a(),F=f("div"),r(Ce.$$.fragment),ua=a(),yt=f("p"),yt.textContent=tn,ga=a(),r(se.$$.fragment),io=a(),r(je.$$.fragment),mo=a(),H=f("div"),r(Je.$$.fragment),va=a(),wt=f("p"),wt.textContent=on,ba=a(),r(de.$$.fragment),ho=a(),r(Ge.$$.fragment),po=a(),I=f("div"),r(Ie.$$.fragment),ka=a(),xt=f("p"),xt.textContent=an,$a=a(),Mt=f("ul"),Mt.innerHTML=nn,ya=a(),r(re.$$.fragment),fo=a(),r(Ne.$$.fragment),_o=a(),r(Fe.$$.fragment),uo=a(),U=f("div"),r(He.$$.fragment),wa=a(),Tt=f("p"),Tt.textContent=ln,xa=a(),Ut=f("p"),Ut.textContent=sn,Ma=a(),Ct=f("ul"),Ct.innerHTML=dn,Ta=a(),ce=f("div"),r(We.$$.fragment),Ua=a(),jt=f("p"),jt.textContent=rn,Ca=a(),ie=f("div"),r(De.$$.fragment),ja=a(),Jt=f("p"),Jt.textContent=cn,Ja=a(),me=f("div"),r(Ze.$$.fragment),Ga=a(),Gt=f("p"),Gt.textContent=mn,Ia=a(),he=f("div"),r(Be.$$.fragment),Na=a(),It=f("p"),It.textContent=hn,go=a(),r(Le.$$.fragment),vo=a(),P=f("div"),r(Ae.$$.fragment),Fa=a(),Nt=f("p"),Nt.innerHTML=pn,bo=a(),r(Ve.$$.fragment),ko=a(),R=f("div"),r(Pe.$$.fragment),Ha=a(),Ft=f("p"),Ft.textContent=fn,$o=a(),r(Re.$$.fragment),yo=a(),X=f("div"),r(Xe.$$.fragment),Wa=a(),Ht=f("p"),Ht.textContent=_n,wo=a(),r(Ee.$$.fragment),xo=a(),r(ze.$$.fragment),Mo=a(),W=f("div"),r(Qe.$$.fragment),Da=a(),Wt=f("p"),Wt.innerHTML=un,Za=a(),r(pe.$$.fragment),To=a(),r(qe.$$.fragment),Uo=a(),E=f("div"),r(Se.$$.fragment),Ba=a(),Dt=f("p"),Dt.innerHTML=gn,Co=a(),r(Ye.$$.fragment),jo=a(),z=f("div"),r(Oe.$$.fragment),La=a(),Zt=f("p"),Zt.innerHTML=vn,Jo=a(),r(Ke.$$.fragment),Go=a(),Q=f("div"),r(et.$$.fragment),Aa=a(),Bt=f("p"),Bt.innerHTML=bn,Io=a(),r(tt.$$.fragment),No=a(),J=f("div"),r(ot.$$.fragment),Va=a(),Lt=f("p"),Lt.innerHTML=kn,Pa=a(),r(fe.$$.fragment),Ra=a(),At=f("p"),At.textContent=$n,Xa=a(),r(_e.$$.fragment),Fo=a(),r(at.$$.fragment),Ho=a(),r(nt.$$.fragment),Wo=a(),q=f("div"),r(lt.$$.fragment),Ea=a(),Vt=f("p"),Vt.innerHTML=yn,Do=a(),r(st.$$.fragment),Zo=a(),S=f("div"),r(dt.$$.fragment),za=a(),Pt=f("p"),Pt.textContent=wn,Bo=a(),r(rt.$$.fragment),Lo=a(),r(ct.$$.fragment),Ao=a(),Y=f("div"),r(it.$$.fragment),Qa=a(),Rt=f("p"),Rt.textContent=xn,Vo=a(),r(mt.$$.fragment),Po=a(),O=f("div"),r(ht.$$.fragment),qa=a(),Xt=f("p"),Xt.textContent=Mn,Ro=a(),r(pt.$$.fragment),Xo=a(),zt=f("p"),this.h()},l(e){const t=Jn("svelte-u9bgzb",document.head);s=_(t,"META",{name:!0,content:!0}),t.forEach(o),k=n(e),g=_(e,"P",{}),y(g).forEach(o),v=n(e),c($.$$.fragment,e),u=n(e),c(T.$$.fragment,e),St=n(e),c(ue.$$.fragment,e),Yt=n(e),c(ge.$$.fragment,e),Ot=n(e),G=_(e,"DIV",{class:!0});var D=y(G);c(ve.$$.fragment,D),ra=n(D),gt=_(D,"P",{"data-svelte-h":!0}),b(gt)!=="svelte-1u5mepw"&&(gt.textContent=Sa),ca=n(D),c(ae.$$.fragment,D),ia=n(D),c(ne.$$.fragment,D),D.forEach(o),Kt=n(e),c(be.$$.fragment,e),eo=n(e),L=_(e,"DIV",{class:!0});var ft=y(L);c(ke.$$.fragment,ft),ma=n(ft),vt=_(ft,"P",{"data-svelte-h":!0}),b(vt)!=="svelte-1gp9ghs"&&(vt.textContent=Ya),ft.forEach(o),to=n(e),c($e.$$.fragment,e),oo=n(e),N=_(e,"DIV",{class:!0});var K=y(N);c(ye.$$.fragment,K),ha=n(K),bt=_(K,"P",{"data-svelte-h":!0}),b(bt)!=="svelte-arhkns"&&(bt.innerHTML=Oa),pa=n(K),c(le.$$.fragment,K),K.forEach(o),ao=n(e),c(we.$$.fragment,e),no=n(e),A=_(e,"DIV",{class:!0});var _t=y(A);c(xe.$$.fragment,_t),fa=n(_t),kt=_(_t,"P",{"data-svelte-h":!0}),b(kt)!=="svelte-js7ro5"&&(kt.textContent=Ka),_t.forEach(o),lo=n(e),c(Me.$$.fragment,e),so=n(e),V=_(e,"DIV",{class:!0});var ut=y(V);c(Te.$$.fragment,ut),_a=n(ut),$t=_(ut,"P",{"data-svelte-h":!0}),b($t)!=="svelte-1p13mzc"&&($t.textContent=en),ut.forEach(o),ro=n(e),c(Ue.$$.fragment,e),co=n(e),F=_(e,"DIV",{class:!0});var ee=y(F);c(Ce.$$.fragment,ee),ua=n(ee),yt=_(ee,"P",{"data-svelte-h":!0}),b(yt)!=="svelte-1ikrez4"&&(yt.textContent=tn),ga=n(ee),c(se.$$.fragment,ee),ee.forEach(o),io=n(e),c(je.$$.fragment,e),mo=n(e),H=_(e,"DIV",{class:!0});var te=y(H);c(Je.$$.fragment,te),va=n(te),wt=_(te,"P",{"data-svelte-h":!0}),b(wt)!=="svelte-qft9nv"&&(wt.textContent=on),ba=n(te),c(de.$$.fragment,te),te.forEach(o),ho=n(e),c(Ge.$$.fragment,e),po=n(e),I=_(e,"DIV",{class:!0});var Z=y(I);c(Ie.$$.fragment,Z),ka=n(Z),xt=_(Z,"P",{"data-svelte-h":!0}),b(xt)!=="svelte-ou57jw"&&(xt.textContent=an),$a=n(Z),Mt=_(Z,"UL",{"data-svelte-h":!0}),b(Mt)!=="svelte-17tb5w2"&&(Mt.innerHTML=nn),ya=n(Z),c(re.$$.fragment,Z),Z.forEach(o),fo=n(e),c(Ne.$$.fragment,e),_o=n(e),c(Fe.$$.fragment,e),uo=n(e),U=_(e,"DIV",{class:!0});var j=y(U);c(He.$$.fragment,j),wa=n(j),Tt=_(j,"P",{"data-svelte-h":!0}),b(Tt)!=="svelte-jjeczj"&&(Tt.textContent=ln),xa=n(j),Ut=_(j,"P",{"data-svelte-h":!0}),b(Ut)!=="svelte-1aqkkwa"&&(Ut.textContent=sn),Ma=n(j),Ct=_(j,"UL",{"data-svelte-h":!0}),b(Ct)!=="svelte-1ny2lfh"&&(Ct.innerHTML=dn),Ta=n(j),ce=_(j,"DIV",{class:!0});var zo=y(ce);c(We.$$.fragment,zo),Ua=n(zo),jt=_(zo,"P",{"data-svelte-h":!0}),b(jt)!=="svelte-13obmts"&&(jt.textContent=rn),zo.forEach(o),Ca=n(j),ie=_(j,"DIV",{class:!0});var Qo=y(ie);c(De.$$.fragment,Qo),ja=n(Qo),Jt=_(Qo,"P",{"data-svelte-h":!0}),b(Jt)!=="svelte-2npthr"&&(Jt.textContent=cn),Qo.forEach(o),Ja=n(j),me=_(j,"DIV",{class:!0});var qo=y(me);c(Ze.$$.fragment,qo),Ga=n(qo),Gt=_(qo,"P",{"data-svelte-h":!0}),b(Gt)!=="svelte-17hlov5"&&(Gt.textContent=mn),qo.forEach(o),Ia=n(j),he=_(j,"DIV",{class:!0});var So=y(he);c(Be.$$.fragment,So),Na=n(So),It=_(So,"P",{"data-svelte-h":!0}),b(It)!=="svelte-53qfnq"&&(It.textContent=hn),So.forEach(o),j.forEach(o),go=n(e),c(Le.$$.fragment,e),vo=n(e),P=_(e,"DIV",{class:!0});var Yo=y(P);c(Ae.$$.fragment,Yo),Fa=n(Yo),Nt=_(Yo,"P",{"data-svelte-h":!0}),b(Nt)!=="svelte-15d0s2c"&&(Nt.innerHTML=pn),Yo.forEach(o),bo=n(e),c(Ve.$$.fragment,e),ko=n(e),R=_(e,"DIV",{class:!0});var Oo=y(R);c(Pe.$$.fragment,Oo),Ha=n(Oo),Ft=_(Oo,"P",{"data-svelte-h":!0}),b(Ft)!=="svelte-kxjaug"&&(Ft.textContent=fn),Oo.forEach(o),$o=n(e),c(Re.$$.fragment,e),yo=n(e),X=_(e,"DIV",{class:!0});var Ko=y(X);c(Xe.$$.fragment,Ko),Wa=n(Ko),Ht=_(Ko,"P",{"data-svelte-h":!0}),b(Ht)!=="svelte-v8znqi"&&(Ht.textContent=_n),Ko.forEach(o),wo=n(e),c(Ee.$$.fragment,e),xo=n(e),c(ze.$$.fragment,e),Mo=n(e),W=_(e,"DIV",{class:!0});var Et=y(W);c(Qe.$$.fragment,Et),Da=n(Et),Wt=_(Et,"P",{"data-svelte-h":!0}),b(Wt)!=="svelte-12byttf"&&(Wt.innerHTML=un),Za=n(Et),c(pe.$$.fragment,Et),Et.forEach(o),To=n(e),c(qe.$$.fragment,e),Uo=n(e),E=_(e,"DIV",{class:!0});var ea=y(E);c(Se.$$.fragment,ea),Ba=n(ea),Dt=_(ea,"P",{"data-svelte-h":!0}),b(Dt)!=="svelte-rf1vym"&&(Dt.innerHTML=gn),ea.forEach(o),Co=n(e),c(Ye.$$.fragment,e),jo=n(e),z=_(e,"DIV",{class:!0});var ta=y(z);c(Oe.$$.fragment,ta),La=n(ta),Zt=_(ta,"P",{"data-svelte-h":!0}),b(Zt)!=="svelte-5orloz"&&(Zt.innerHTML=vn),ta.forEach(o),Jo=n(e),c(Ke.$$.fragment,e),Go=n(e),Q=_(e,"DIV",{class:!0});var oa=y(Q);c(et.$$.fragment,oa),Aa=n(oa),Bt=_(oa,"P",{"data-svelte-h":!0}),b(Bt)!=="svelte-1anoa02"&&(Bt.innerHTML=bn),oa.forEach(o),Io=n(e),c(tt.$$.fragment,e),No=n(e),J=_(e,"DIV",{class:!0});var B=y(J);c(ot.$$.fragment,B),Va=n(B),Lt=_(B,"P",{"data-svelte-h":!0}),b(Lt)!=="svelte-q70bcf"&&(Lt.innerHTML=kn),Pa=n(B),c(fe.$$.fragment,B),Ra=n(B),At=_(B,"P",{"data-svelte-h":!0}),b(At)!=="svelte-3cloev"&&(At.textContent=$n),Xa=n(B),c(_e.$$.fragment,B),B.forEach(o),Fo=n(e),c(at.$$.fragment,e),Ho=n(e),c(nt.$$.fragment,e),Wo=n(e),q=_(e,"DIV",{class:!0});var aa=y(q);c(lt.$$.fragment,aa),Ea=n(aa),Vt=_(aa,"P",{"data-svelte-h":!0}),b(Vt)!=="svelte-1kj8409"&&(Vt.innerHTML=yn),aa.forEach(o),Do=n(e),c(st.$$.fragment,e),Zo=n(e),S=_(e,"DIV",{class:!0});var na=y(S);c(dt.$$.fragment,na),za=n(na),Pt=_(na,"P",{"data-svelte-h":!0}),b(Pt)!=="svelte-xlirjm"&&(Pt.textContent=wn),na.forEach(o),Bo=n(e),c(rt.$$.fragment,e),Lo=n(e),c(ct.$$.fragment,e),Ao=n(e),Y=_(e,"DIV",{class:!0});var la=y(Y);c(it.$$.fragment,la),Qa=n(la),Rt=_(la,"P",{"data-svelte-h":!0}),b(Rt)!=="svelte-195dvsx"&&(Rt.textContent=xn),la.forEach(o),Vo=n(e),c(mt.$$.fragment,e),Po=n(e),O=_(e,"DIV",{class:!0});var sa=y(O);c(ht.$$.fragment,sa),qa=n(sa),Xt=_(sa,"P",{"data-svelte-h":!0}),b(Xt)!=="svelte-17fy5vm"&&(Xt.textContent=Mn),sa.forEach(o),Ro=n(e),c(pt.$$.fragment,e),Xo=n(e),zt=_(e,"P",{}),y(zt).forEach(o),this.h()},h(){w(s,"name","hf:doc:metadata"),w(s,"content",Pn),w(G,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(V,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(I,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(ce,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(ie,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(me,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(he,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(R,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(X,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(E,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(Q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(S,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){d(document.head,s),l(e,k,t),l(e,g,t),l(e,v,t),i($,e,t),l(e,u,t),i(T,e,t),l(e,St,t),i(ue,e,t),l(e,Yt,t),i(ge,e,t),l(e,Ot,t),l(e,G,t),i(ve,G,null),d(G,ra),d(G,gt),d(G,ca),i(ae,G,null),d(G,ia),i(ne,G,null),l(e,Kt,t),i(be,e,t),l(e,eo,t),l(e,L,t),i(ke,L,null),d(L,ma),d(L,vt),l(e,to,t),i($e,e,t),l(e,oo,t),l(e,N,t),i(ye,N,null),d(N,ha),d(N,bt),d(N,pa),i(le,N,null),l(e,ao,t),i(we,e,t),l(e,no,t),l(e,A,t),i(xe,A,null),d(A,fa),d(A,kt),l(e,lo,t),i(Me,e,t),l(e,so,t),l(e,V,t),i(Te,V,null),d(V,_a),d(V,$t),l(e,ro,t),i(Ue,e,t),l(e,co,t),l(e,F,t),i(Ce,F,null),d(F,ua),d(F,yt),d(F,ga),i(se,F,null),l(e,io,t),i(je,e,t),l(e,mo,t),l(e,H,t),i(Je,H,null),d(H,va),d(H,wt),d(H,ba),i(de,H,null),l(e,ho,t),i(Ge,e,t),l(e,po,t),l(e,I,t),i(Ie,I,null),d(I,ka),d(I,xt),d(I,$a),d(I,Mt),d(I,ya),i(re,I,null),l(e,fo,t),i(Ne,e,t),l(e,_o,t),i(Fe,e,t),l(e,uo,t),l(e,U,t),i(He,U,null),d(U,wa),d(U,Tt),d(U,xa),d(U,Ut),d(U,Ma),d(U,Ct),d(U,Ta),d(U,ce),i(We,ce,null),d(ce,Ua),d(ce,jt),d(U,Ca),d(U,ie),i(De,ie,null),d(ie,ja),d(ie,Jt),d(U,Ja),d(U,me),i(Ze,me,null),d(me,Ga),d(me,Gt),d(U,Ia),d(U,he),i(Be,he,null),d(he,Na),d(he,It),l(e,go,t),i(Le,e,t),l(e,vo,t),l(e,P,t),i(Ae,P,null),d(P,Fa),d(P,Nt),l(e,bo,t),i(Ve,e,t),l(e,ko,t),l(e,R,t),i(Pe,R,null),d(R,Ha),d(R,Ft),l(e,$o,t),i(Re,e,t),l(e,yo,t),l(e,X,t),i(Xe,X,null),d(X,Wa),d(X,Ht),l(e,wo,t),i(Ee,e,t),l(e,xo,t),i(ze,e,t),l(e,Mo,t),l(e,W,t),i(Qe,W,null),d(W,Da),d(W,Wt),d(W,Za),i(pe,W,null),l(e,To,t),i(qe,e,t),l(e,Uo,t),l(e,E,t),i(Se,E,null),d(E,Ba),d(E,Dt),l(e,Co,t),i(Ye,e,t),l(e,jo,t),l(e,z,t),i(Oe,z,null),d(z,La),d(z,Zt),l(e,Jo,t),i(Ke,e,t),l(e,Go,t),l(e,Q,t),i(et,Q,null),d(Q,Aa),d(Q,Bt),l(e,Io,t),i(tt,e,t),l(e,No,t),l(e,J,t),i(ot,J,null),d(J,Va),d(J,Lt),d(J,Pa),i(fe,J,null),d(J,Ra),d(J,At),d(J,Xa),i(_e,J,null),l(e,Fo,t),i(at,e,t),l(e,Ho,t),i(nt,e,t),l(e,Wo,t),l(e,q,t),i(lt,q,null),d(q,Ea),d(q,Vt),l(e,Do,t),i(st,e,t),l(e,Zo,t),l(e,S,t),i(dt,S,null),d(S,za),d(S,Pt),l(e,Bo,t),i(rt,e,t),l(e,Lo,t),i(ct,e,t),l(e,Ao,t),l(e,Y,t),i(it,Y,null),d(Y,Qa),d(Y,Rt),l(e,Vo,t),i(mt,e,t),l(e,Po,t),l(e,O,t),i(ht,O,null),d(O,qa),d(O,Xt),l(e,Ro,t),i(pt,e,t),l(e,Xo,t),l(e,zt,t),Eo=!0},p(e,[t]){const D={};t&2&&(D.$$scope={dirty:t,ctx:e}),ae.$set(D);const ft={};t&2&&(ft.$$scope={dirty:t,ctx:e}),ne.$set(ft);const K={};t&2&&(K.$$scope={dirty:t,ctx:e}),le.$set(K);const _t={};t&2&&(_t.$$scope={dirty:t,ctx:e}),se.$set(_t);const ut={};t&2&&(ut.$$scope={dirty:t,ctx:e}),de.$set(ut);const ee={};t&2&&(ee.$$scope={dirty:t,ctx:e}),re.$set(ee);const te={};t&2&&(te.$$scope={dirty:t,ctx:e}),pe.$set(te);const Z={};t&2&&(Z.$$scope={dirty:t,ctx:e}),fe.$set(Z);const j={};t&2&&(j.$$scope={dirty:t,ctx:e}),_e.$set(j)},i(e){Eo||(m($.$$.fragment,e),m(T.$$.fragment,e),m(ue.$$.fragment,e),m(ge.$$.fragment,e),m(ve.$$.fragment,e),m(ae.$$.fragment,e),m(ne.$$.fragment,e),m(be.$$.fragment,e),m(ke.$$.fragment,e),m($e.$$.fragment,e),m(ye.$$.fragment,e),m(le.$$.fragment,e),m(we.$$.fragment,e),m(xe.$$.fragment,e),m(Me.$$.fragment,e),m(Te.$$.fragment,e),m(Ue.$$.fragment,e),m(Ce.$$.fragment,e),m(se.$$.fragment,e),m(je.$$.fragment,e),m(Je.$$.fragment,e),m(de.$$.fragment,e),m(Ge.$$.fragment,e),m(Ie.$$.fragment,e),m(re.$$.fragment,e),m(Ne.$$.fragment,e),m(Fe.$$.fragment,e),m(He.$$.fragment,e),m(We.$$.fragment,e),m(De.$$.fragment,e),m(Ze.$$.fragment,e),m(Be.$$.fragment,e),m(Le.$$.fragment,e),m(Ae.$$.fragment,e),m(Ve.$$.fragment,e),m(Pe.$$.fragment,e),m(Re.$$.fragment,e),m(Xe.$$.fragment,e),m(Ee.$$.fragment,e),m(ze.$$.fragment,e),m(Qe.$$.fragment,e),m(pe.$$.fragment,e),m(qe.$$.fragment,e),m(Se.$$.fragment,e),m(Ye.$$.fragment,e),m(Oe.$$.fragment,e),m(Ke.$$.fragment,e),m(et.$$.fragment,e),m(tt.$$.fragment,e),m(ot.$$.fragment,e),m(fe.$$.fragment,e),m(_e.$$.fragment,e),m(at.$$.fragment,e),m(nt.$$.fragment,e),m(lt.$$.fragment,e),m(st.$$.fragment,e),m(dt.$$.fragment,e),m(rt.$$.fragment,e),m(ct.$$.fragment,e),m(it.$$.fragment,e),m(mt.$$.fragment,e),m(ht.$$.fragment,e),m(pt.$$.fragment,e),Eo=!0)},o(e){h($.$$.fragment,e),h(T.$$.fragment,e),h(ue.$$.fragment,e),h(ge.$$.fragment,e),h(ve.$$.fragment,e),h(ae.$$.fragment,e),h(ne.$$.fragment,e),h(be.$$.fragment,e),h(ke.$$.fragment,e),h($e.$$.fragment,e),h(ye.$$.fragment,e),h(le.$$.fragment,e),h(we.$$.fragment,e),h(xe.$$.fragment,e),h(Me.$$.fragment,e),h(Te.$$.fragment,e),h(Ue.$$.fragment,e),h(Ce.$$.fragment,e),h(se.$$.fragment,e),h(je.$$.fragment,e),h(Je.$$.fragment,e),h(de.$$.fragment,e),h(Ge.$$.fragment,e),h(Ie.$$.fragment,e),h(re.$$.fragment,e),h(Ne.$$.fragment,e),h(Fe.$$.fragment,e),h(He.$$.fragment,e),h(We.$$.fragment,e),h(De.$$.fragment,e),h(Ze.$$.fragment,e),h(Be.$$.fragment,e),h(Le.$$.fragment,e),h(Ae.$$.fragment,e),h(Ve.$$.fragment,e),h(Pe.$$.fragment,e),h(Re.$$.fragment,e),h(Xe.$$.fragment,e),h(Ee.$$.fragment,e),h(ze.$$.fragment,e),h(Qe.$$.fragment,e),h(pe.$$.fragment,e),h(qe.$$.fragment,e),h(Se.$$.fragment,e),h(Ye.$$.fragment,e),h(Oe.$$.fragment,e),h(Ke.$$.fragment,e),h(et.$$.fragment,e),h(tt.$$.fragment,e),h(ot.$$.fragment,e),h(fe.$$.fragment,e),h(_e.$$.fragment,e),h(at.$$.fragment,e),h(nt.$$.fragment,e),h(lt.$$.fragment,e),h(st.$$.fragment,e),h(dt.$$.fragment,e),h(rt.$$.fragment,e),h(ct.$$.fragment,e),h(it.$$.fragment,e),h(mt.$$.fragment,e),h(ht.$$.fragment,e),h(pt.$$.fragment,e),Eo=!1},d(e){e&&(o(k),o(g),o(v),o(u),o(St),o(Yt),o(Ot),o(G),o(Kt),o(eo),o(L),o(to),o(oo),o(N),o(ao),o(no),o(A),o(lo),o(so),o(V),o(ro),o(co),o(F),o(io),o(mo),o(H),o(ho),o(po),o(I),o(fo),o(_o),o(uo),o(U),o(go),o(vo),o(P),o(bo),o(ko),o(R),o($o),o(yo),o(X),o(wo),o(xo),o(Mo),o(W),o(To),o(Uo),o(E),o(Co),o(jo),o(z),o(Jo),o(Go),o(Q),o(Io),o(No),o(J),o(Fo),o(Ho),o(Wo),o(q),o(Do),o(Zo),o(S),o(Bo),o(Lo),o(Ao),o(Y),o(Vo),o(Po),o(O),o(Ro),o(Xo),o(zt)),o(s),p($,e),p(T,e),p(ue,e),p(ge,e),p(ve),p(ae),p(ne),p(be,e),p(ke),p($e,e),p(ye),p(le),p(we,e),p(xe),p(Me,e),p(Te),p(Ue,e),p(Ce),p(se),p(je,e),p(Je),p(de),p(Ge,e),p(Ie),p(re),p(Ne,e),p(Fe,e),p(He),p(We),p(De),p(Ze),p(Be),p(Le,e),p(Ae),p(Ve,e),p(Pe),p(Re,e),p(Xe),p(Ee,e),p(ze,e),p(Qe),p(pe),p(qe,e),p(Se),p(Ye,e),p(Oe),p(Ke,e),p(et),p(tt,e),p(ot),p(fe),p(_e),p(at,e),p(nt,e),p(lt),p(st,e),p(dt),p(rt,e),p(ct,e),p(it),p(mt,e),p(ht),p(pt,e)}}}const Pn='{"title":"Working with large models","local":"working-with-large-models","sections":[{"title":"Dispatch and offload","local":"dispatch-and-offload","sections":[{"title":"init_empty_weights","local":"accelerate.init_empty_weights","sections":[],"depth":3},{"title":"cpu_offload","local":"accelerate.cpu_offload","sections":[],"depth":3},{"title":"cpu_offload_with_hook","local":"accelerate.cpu_offload_with_hook","sections":[],"depth":3},{"title":"disk_offload","local":"accelerate.disk_offload","sections":[],"depth":3},{"title":"dispatch_model","local":"accelerate.dispatch_model","sections":[],"depth":3},{"title":"load_checkpoint_and_dispatch","local":"accelerate.load_checkpoint_and_dispatch","sections":[],"depth":3},{"title":"load_checkpoint_in_model","local":"accelerate.load_checkpoint_in_model","sections":[],"depth":3},{"title":"infer_auto_device_map","local":"accelerate.infer_auto_device_map","sections":[],"depth":3}],"depth":2},{"title":"Hooks","local":"hooks","sections":[{"title":"ModelHook","local":"accelerate.hooks.ModelHook","sections":[],"depth":3},{"title":"AlignDevicesHook","local":"accelerate.hooks.AlignDevicesHook","sections":[],"depth":3},{"title":"SequentialHook","local":"accelerate.hooks.SequentialHook","sections":[],"depth":3},{"title":"LayerwiseCastingHook","local":"accelerate.hooks.LayerwiseCastingHook","sections":[],"depth":3}],"depth":2},{"title":"Adding Hooks","local":"adding-hooks","sections":[{"title":"add_hook_to_module","local":"accelerate.hooks.add_hook_to_module","sections":[],"depth":3},{"title":"attach_execution_device_hook","local":"accelerate.hooks.attach_execution_device_hook","sections":[],"depth":3},{"title":"attach_align_device_hook","local":"accelerate.hooks.attach_align_device_hook","sections":[],"depth":3},{"title":"attach_align_device_hook_on_blocks","local":"accelerate.hooks.attach_align_device_hook_on_blocks","sections":[],"depth":3},{"title":"attach_layerwise_casting_hooks","local":"accelerate.big_modeling.attach_layerwise_casting_hooks","sections":[],"depth":3}],"depth":2},{"title":"Removing Hooks","local":"removing-hooks","sections":[{"title":"remove_hook_from_module","local":"accelerate.hooks.remove_hook_from_module","sections":[],"depth":3},{"title":"remove_hook_from_submodules","local":"accelerate.hooks.remove_hook_from_submodules","sections":[],"depth":3}],"depth":2},{"title":"Utilities","local":"utilities","sections":[{"title":"has_offloaded_params","local":"accelerate.utils.has_offloaded_params","sections":[],"depth":3},{"title":"align_module_device","local":"accelerate.utils.align_module_device","sections":[],"depth":3}],"depth":2}],"depth":1}';function Rn(C){return Un(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class On extends Cn{constructor(s){super(),jn(this,s,Rn,Vn,Tn,{})}}export{On as component}; | |
Xet Storage Details
- Size:
- 92.5 kB
- Xet hash:
- a58de881f0e79f0f0d543bac843e4168f841e5db2d4ae4efdfff7549493e27d5
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.