accelerate/pr_4143/en/_app/immutable/nodes/3.DL5yip__.js

import"../chunks/DsnmJJEf.js";import{i as eo,h as oo,C as ao,H as o,D as a,E as to,s as no,a as p}from"../chunks/CePRIh9G.js";import{p as co,o as so,s as e,f as m,a as i,b as lo,c as t,d as h,r as n,n as s}from"../chunks/CpNKRKmM.js";import{T as u}from"../chunks/DXPdOTzx.js";import{E as _}from"../chunks/DXNZ3u5l.js";const io='{"title":"Working with large models","local":"working-with-large-models","sections":[{"title":"Dispatch and offload","local":"dispatch-and-offload","sections":[{"title":"init_empty_weights","local":"accelerate.init_empty_weights","sections":[],"depth":3},{"title":"cpu_offload","local":"accelerate.cpu_offload","sections":[],"depth":3},{"title":"cpu_offload_with_hook","local":"accelerate.cpu_offload_with_hook","sections":[],"depth":3},{"title":"disk_offload","local":"accelerate.disk_offload","sections":[],"depth":3},{"title":"dispatch_model","local":"accelerate.dispatch_model","sections":[],"depth":3},{"title":"load_checkpoint_and_dispatch","local":"accelerate.load_checkpoint_and_dispatch","sections":[],"depth":3},{"title":"load_checkpoint_in_model","local":"accelerate.load_checkpoint_in_model","sections":[],"depth":3},{"title":"infer_auto_device_map","local":"accelerate.infer_auto_device_map","sections":[],"depth":3}],"depth":2},{"title":"Hooks","local":"hooks","sections":[{"title":"ModelHook","local":"accelerate.hooks.ModelHook","sections":[],"depth":3},{"title":"AlignDevicesHook","local":"accelerate.hooks.AlignDevicesHook","sections":[],"depth":3},{"title":"SequentialHook","local":"accelerate.hooks.SequentialHook","sections":[],"depth":3},{"title":"LayerwiseCastingHook","local":"accelerate.hooks.LayerwiseCastingHook","sections":[],"depth":3}],"depth":2},{"title":"Adding Hooks","local":"adding-hooks","sections":[{"title":"add_hook_to_module","local":"accelerate.hooks.add_hook_to_module","sections":[],"depth":3},{"title":"attach_execution_device_hook","local":"accelerate.hooks.attach_execution_device_hook","sections":[],"depth":3},{"title":"attach_align_device_hook","local":"accelerate.hooks.attach_align_device_hook","sections":[],"depth":3},{"title":"attach_align_device_hook_on_blocks","local":"accelerate.hooks.attach_align_device_hook_on_blocks","sections":[],"depth":3},{"title":"attach_layerwise_casting_hooks","local":"accelerate.big_modeling.attach_layerwise_casting_hooks","sections":[],"depth":3}],"depth":2},{"title":"Removing Hooks","local":"removing-hooks","sections":[{"title":"remove_hook_from_module","local":"accelerate.hooks.remove_hook_from_module","sections":[],"depth":3},{"title":"remove_hook_from_submodules","local":"accelerate.hooks.remove_hook_from_submodules","sections":[],"depth":3}],"depth":2},{"title":"Utilities","local":"utilities","sections":[{"title":"has_offloaded_params","local":"accelerate.utils.has_offloaded_params","sections":[],"depth":3},{"title":"align_module_device","local":"accelerate.utils.align_module_device","sections":[],"depth":3}],"depth":2}],"depth":1}';var ro=h('<meta name="hf:doc:metadata"/>'),f=h("<p>Example:</p> <!>",1),ho=h(`<p>Any model created under this context manager has no weights. As such you can’t do something like <code>model.to(some_device)</code> with it. To load weights inside your empty model, see <a href="/docs/accelerate/pr_4143/en/package_reference/big_modeling#accelerate.load_checkpoint_and_dispatch">load_checkpoint_and_dispatch()</a>.
Make sure to overwrite the default device_map param for <a href="/docs/accelerate/pr_4143/en/package_reference/big_modeling#accelerate.load_checkpoint_and_dispatch">load_checkpoint_and_dispatch()</a>, otherwise dispatch is not
called.</p>`),po=h(`<p>Once loaded across devices, you still need to call <a href="/docs/accelerate/pr_4143/en/package_reference/big_modeling#accelerate.dispatch_model">dispatch_model()</a> on your model to make it able to run. To
group the checkpoint loading and dispatch in one single call, use <a href="/docs/accelerate/pr_4143/en/package_reference/big_modeling#accelerate.load_checkpoint_and_dispatch">load_checkpoint_and_dispatch()</a>.</p>`),mo=h(`<p>All computation is done analyzing sizes and dtypes of the model parameters. As a result, the model can be on the
meta device (as it would if initialized within the <code>init_empty_weights</code> context manager).</p>`),_o=h(`<p>If the module already contains a hook, this will replace it with the new hook passed by default. To chain two hooks
together, pass <code>append=True</code>, so it chains the current and new hook into an instance of the <code>SequentialHook</code> class.</p>`),uo=h(`<p></p> <!> <!> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A context manager under which models are initialized with all parameters on the meta device, therefore creating an
empty model. Useful when just initializing the model would blow the available RAM.</p> <!> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Activates full CPU offload for a model. As a result, all parameters of the model will be offloaded and only one
copy of the state dict of the model will be kept. During the forward pass, parameters will be extracted from that
state dict and put on the execution device passed as they are needed, then offloaded again.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Offloads a model on the CPU and puts it back to an execution device when executed. The difference with <a href="/docs/accelerate/pr_4143/en/package_reference/big_modeling#accelerate.cpu_offload">cpu_offload()</a> is that the model stays on the execution device after the forward and is only offloaded again when
the <code>offload</code> method of the returned <code>hook</code> is called. Useful for pipelines running a model in a loop.</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Activates full disk offload for a model. As a result, all parameters of the model will be offloaded as
memory-mapped array in a given folder. During the forward pass, parameters will be accessed from that folder and
put on the execution device passed as they are needed, then offloaded again.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Dispatches a model according to a given device map. Layers of the model might be spread across GPUs, offloaded on
the CPU or even the disk.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are
loaded and adds the various hooks that will make this model run properly (even if split across devices).</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Loads a (potentially sharded) checkpoint inside a model, potentially sending weights to a given device as they are
loaded.</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Compute a device map for a given model giving priority to GPUs, then offload on CPU and finally offload to disk,
such that:</p> <ul><li>we don’t exceed the memory available of any of the GPU.</li> <li>if offload to the CPU is needed, there is always room left on GPU 0 to put back the layer offloaded on CPU that
has the largest size.</li> <li>if offload to the CPU is needed,we don’t exceed the RAM available on the CPU.</li> <li>if offload to the disk is needed, there is always room left on the CPU to put back the layer offloaded on disk
that has the largest size.</li></ul> <!></div> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A hook that contains callbacks to be executed just before and after the forward method of a model. The difference
with PyTorch existing hooks is that they get passed along the kwargs.</p> <p>Class attribute:</p> <ul><li><strong>no_grad</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) — Whether or not to execute the actual forward pass under
the <code>torch.no_grad()</code> context manager.</li></ul> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>To be executed when the hook is detached from a module.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>To be executed when the hook is attached to the module.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>To be executed just after the forward method of the model.</p></div> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>To be executed just before the forward method of the model.</p></div></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A generic <code>ModelHook</code> that ensures inputs and model weights are on the same device for the forward pass of the
associated module, potentially offloading the weights after the forward pass.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A hook that can contain several hooks and iterates through them at each event.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>A hook that casts the weights of a module to a high precision dtype for computation, and to a low precision dtype
for storage. This process may lead to quality loss in the output, but can significantly reduce the memory
footprint.</p></div> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Adds a hook to a given module. This will rewrite the <code>forward</code> method of the module to include the hook, to remove
this behavior and restore the original <code>forward</code> method, use <code>remove_hook_from_module</code>.</p> <!></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively attaches <code>AlignDevicesHook</code> to all submodules of a given model to make sure they have the right
execution device</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively attaches <code>AlignDevicesHook</code> to all submodules of a given model that have direct parameters and/or
buffers.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Attaches <code>AlignDevicesHook</code> to all blocks of a given model as needed.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Applies layerwise casting to a given module. The module expected here is a PyTorch <code>nn.Module</code>. This is helpful for
reducing memory requirements when one doesn’t want to fully quantize a model. Model params can be kept in say, <code>torch.float8_e4m3fn</code> and upcasted to a higher precision like <code>torch.bfloat16</code> during forward pass and downcasted
back to <code>torch.float8_e4m3fn</code> to realize memory savings.</p> <!> <p>Users can also pass modules they want to avoid from getting downcasted.</p> <!></div> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Removes any hook attached to a module via <code>add_hook_to_module</code>.</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Recursively removes all hooks attached on the submodules of a given model.</p></div> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Checks if a module has offloaded parameters by checking if the given module has a AlignDevicesHook attached with
offloading enabled</p></div> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Context manager that moves a module’s parameters to the specified execution device.</p></div> <!> <p></p>`,1);function yo(je,Je){co(Je,!1),so(()=>{new URLSearchParams(window.location.search).get("fw")}),eo();var X=uo();oo("1sxvz4o",d=>{var l=ro();no(l,"content",io),i(d,l)});var L=e(m(X),2);ao(L,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var z=e(L,2);o(z,{title:"Working with large models",local:"working-with-large-models",headingTag:"h1"});var V=e(z,2);o(V,{title:"Dispatch and offload",local:"dispatch-and-offload",headingTag:"h2"});var P=e(V,2);o(P,{title:"init_empty_weights",local:"accelerate.init_empty_weights",headingTag:"h3"});var g=e(P,2),Q=t(g);a(Q,{name:"accelerate.init_empty_weights",anchor:"accelerate.init_empty_weights",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/big_modeling.py#L61",parameters:[{name:"include_buffers",val:": typing.Optional[bool] = None"}],parametersDescription:[{anchor:"accelerate.init_empty_weights.include_buffers",description:`<strong>include_buffers</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to also put all buffers on the meta device while initializing.`,name:"include_buffers"}]});var S=e(Q,4);_(S,{anchor:"accelerate.init_empty_weights.example",children:(d,l)=>{var c=f(),r=e(m(c),2);p(r,{code:"aW1wb3J0JTIwdG9yY2gubm4lMjBhcyUyMG5uJTBBZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBpbml0X2VtcHR5X3dlaWdodHMlMEElMEElMjMlMjBJbml0aWFsaXplJTIwYSUyMG1vZGVsJTIwd2l0aCUyMDEwMCUyMGJpbGxpb25zJTIwcGFyYW1ldGVycyUyMGluJTIwbm8lMjB0aW1lJTIwYW5kJTIwd2l0aG91dCUyMHVzaW5nJTIwYW55JTIwUkFNLiUwQXdpdGglMjBpbml0X2VtcHR5X3dlaWdodHMoKSUzQSUwQSUyMCUyMCUyMCUyMHRzdCUyMCUzRCUyMG5uLlNlcXVlbnRpYWwoKiU1Qm5uLkxpbmVhcigxMDAwMCUyQyUyMDEwMDAwKSUyMGZvciUyMF8lMjBpbiUyMHJhbmdlKDEwMDApJTVEKQ==",highlighted:`<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn
<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> init_empty_weights

<span class="hljs-comment"># Initialize a model with 100 billions parameters in no time and without using any RAM.</span>
<span class="hljs-keyword">with</span> init_empty_weights():
    tst = nn.Sequential(*[nn.Linear(<span class="hljs-number">10000</span>, <span class="hljs-number">10000</span>) <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1000</span>)])`,lang:"python",wrap:!1}),i(d,c)},$$slots:{default:!0}});var Ge=e(S,2);u(Ge,{warning:!0,children:(d,l)=>{var c=ho();i(d,c)},$$slots:{default:!0}}),n(g);var E=e(g,2);o(E,{title:"cpu_offload",local:"accelerate.cpu_offload",headingTag:"h3"});var v=e(E,2),Ne=t(v);a(Ne,{name:"accelerate.cpu_offload",anchor:"accelerate.cpu_offload",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/big_modeling.py#L179",parameters:[{name:"model",val:": Module"},{name:"execution_device",val:": typing.Optional[torch.device] = None"},{name:"offload_buffers",val:": bool = False"},{name:"state_dict",val:": typing.Optional[dict[str, torch.Tensor]] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"}],parametersDescription:[{anchor:"accelerate.cpu_offload.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014;
The model to offload.`,name:"model"},{anchor:"accelerate.cpu_offload.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) &#x2014;
The device on which the forward pass of the model will be executed (should be a GPU). Will default to the
model first parameter device.`,name:"execution_device"},{anchor:"accelerate.cpu_offload.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to offload the buffers with the model parameters.`,name:"offload_buffers"},{anchor:"accelerate.cpu_offload.state_dict",description:`<strong>state_dict</strong> (<code>Dict[str, torch.Tensor]</code>, <em>optional</em>) &#x2014;
The state dict of the model that will be kept on CPU.`,name:"state_dict"},{anchor:"accelerate.cpu_offload.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of classes whose instances should load all their weights (even in the submodules) at the beginning
of the forward. This should only be used for classes that have submodules which are registered but not
called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward,
<code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"}]}),s(2),n(v);var q=e(v,2);o(q,{title:"cpu_offload_with_hook",local:"accelerate.cpu_offload_with_hook",headingTag:"h3"});var b=e(q,2),Y=t(b);a(Y,{name:"accelerate.cpu_offload_with_hook",anchor:"accelerate.cpu_offload_with_hook",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/big_modeling.py#L225",parameters:[{name:"model",val:": Module"},{name:"execution_device",val:": typing.Union[int, str, torch.device, NoneType] = None"},{name:"prev_module_hook",val:": typing.Optional[accelerate.hooks.UserCpuOffloadHook] = None"}],parametersDescription:[{anchor:"accelerate.cpu_offload_with_hook.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014;
The model to offload.`,name:"model"},{anchor:"accelerate.cpu_offload_with_hook.execution_device(str,",description:`<strong>execution_device(<code>str</code>,</strong> <code>int</code> or <code>torch.device</code>, <em>optional</em>) &#x2014;
The device on which the model should be executed. Will default to the MPS device if it&#x2019;s available, then
device 0 if there is an accelerator device, and finally to the CPU.`,name:"execution_device(str,"},{anchor:"accelerate.cpu_offload_with_hook.prev_module_hook",description:`<strong>prev_module_hook</strong> (<code>UserCpuOffloadHook</code>, <em>optional</em>) &#x2014;
The hook sent back by this function for a previous model in the pipeline you are running. If passed, its
offload method will be called just before the forward of the model to which this hook is attached.`,name:"prev_module_hook"}]});var Fe=e(Y,4);_(Fe,{anchor:"accelerate.cpu_offload_with_hook.example",children:(d,l)=>{var c=f(),r=e(m(c),2);p(r,{code:"bW9kZWxfMSUyQyUyMGhvb2tfMSUyMCUzRCUyMGNwdV9vZmZsb2FkX3dpdGhfaG9vayhtb2RlbF8xJTJDJTIwZGV2aWNlKSUwQW1vZGVsXzIlMkMlMjBob29rXzIlMjAlM0QlMjBjcHVfb2ZmbG9hZF93aXRoX2hvb2sobW9kZWxfMiUyQyUyMGRldmljZSUyQyUyMHByZXZfbW9kdWxlX2hvb2slM0Rob29rXzEpJTBBbW9kZWxfMyUyQyUyMGhvb2tfMyUyMCUzRCUyMGNwdV9vZmZsb2FkX3dpdGhfaG9vayhtb2RlbF8zJTJDJTIwZGV2aWNlJTJDJTIwcHJldl9tb2R1bGVfaG9vayUzRGhvb2tfMiklMEElMEFoaWRfMSUyMCUzRCUyMG1vZGVsXzEoaW5wdXQpJTBBZm9yJTIwaSUyMGluJTIwcmFuZ2UoNTApJTNBJTBBJTIwJTIwJTIwJTIwJTIzJTIwbW9kZWwxJTIwaXMlMjBvZmZsb2FkZWQlMjBvbiUyMHRoZSUyMENQVSUyMGF0JTIwdGhlJTIwZmlyc3QlMjBpdGVyYXRpb24lMkMlMjBtb2RlbCUyMDIlMjBzdGF5cyUyMG9uJTIwdGhlJTIwR1BVJTIwZm9yJTIwdGhpcyUyMHdob2xlJTIwbG9vcC4lMEElMjAlMjAlMjAlMjBoaWRfMiUyMCUzRCUyMG1vZGVsXzIoaGlkXzEpJTBBJTIzJTIwbW9kZWwyJTIwaXMlMjBvZmZsb2FkZWQlMjB0byUyMHRoZSUyMENQVSUyMGp1c3QlMjBiZWZvcmUlMjB0aGlzJTIwZm9yd2FyZC4lMEFoaWRfMyUyMCUzRCUyMG1vZGVsXzMoaGlkXzMpJTBBJTBBJTIzJTIwRm9yJTIwbW9kZWwzJTJDJTIweW91JTIwbmVlZCUyMHRvJTIwbWFudWFsbHklMjBjYWxsJTIwdGhlJTIwaG9vayUyMG9mZmxvYWQlMjBtZXRob2QuJTBBaG9va18zLm9mZmxvYWQoKQ==",highlighted:`model_1, hook_1 = cpu_offload_with_hook(model_1, device)
model_2, hook_2 = cpu_offload_with_hook(model_2, device, prev_module_hook=hook_1)
model_3, hook_3 = cpu_offload_with_hook(model_3, device, prev_module_hook=hook_2)

hid_1 = model_1(<span class="hljs-built_in">input</span>)
<span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">50</span>):
    <span class="hljs-comment"># model1 is offloaded on the CPU at the first iteration, model 2 stays on the GPU for this whole loop.</span>
    hid_2 = model_2(hid_1)
<span class="hljs-comment"># model2 is offloaded to the CPU just before this forward.</span>
hid_3 = model_3(hid_3)

<span class="hljs-comment"># For model3, you need to manually call the hook offload method.</span>
hook_3.offload()`,lang:"py",wrap:!1}),i(d,c)},$$slots:{default:!0}}),n(b);var O=e(b,2);o(O,{title:"disk_offload",local:"accelerate.disk_offload",headingTag:"h3"});var k=e(O,2),We=t(k);a(We,{name:"accelerate.disk_offload",anchor:"accelerate.disk_offload",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/big_modeling.py#L269",parameters:[{name:"model",val:": Module"},{name:"offload_dir",val:": typing.Union[str, os.PathLike]"},{name:"execution_device",val:": typing.Optional[torch.device] = None"},{name:"offload_buffers",val:": bool = False"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"}],parametersDescription:[{anchor:"accelerate.disk_offload.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014; The model to offload.",name:"model"},{anchor:"accelerate.disk_offload.offload_dir",description:`<strong>offload_dir</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
The folder in which to offload the model weights (or where the model weights are already offloaded).`,name:"offload_dir"},{anchor:"accelerate.disk_offload.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) &#x2014;
The device on which the forward pass of the model will be executed (should be a GPU). Will default to the
model&#x2019;s first parameter device.`,name:"execution_device"},{anchor:"accelerate.disk_offload.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to offload the buffers with the model parameters.`,name:"offload_buffers"},{anchor:"accelerate.disk_offload.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of classes whose instances should load all their weights (even in the submodules) at the beginning
of the forward. This should only be used for classes that have submodules which are registered but not
called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward,
<code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"}]}),s(2),n(k);var $=e(k,2);o($,{title:"dispatch_model",local:"accelerate.dispatch_model",headingTag:"h3"});var y=e($,2),Ze=t(y);a(Ze,{name:"accelerate.dispatch_model",anchor:"accelerate.dispatch_model",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/big_modeling.py#L315",parameters:[{name:"model",val:": Module"},{name:"device_map",val:": dict"},{name:"main_device",val:": typing.Optional[torch.device] = None"},{name:"state_dict",val:": typing.Optional[dict[str, torch.Tensor]] = None"},{name:"offload_dir",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"offload_index",val:": typing.Optional[dict[str, str]] = None"},{name:"offload_buffers",val:": bool = False"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"force_hooks",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.dispatch_model.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014;
The model to dispatch.`,name:"model"},{anchor:"accelerate.dispatch_model.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[str, int, torch.device]]</code>) &#x2014;
A dictionary mapping module names in the models <code>state_dict</code> to the device they should go to. Note that
<code>&quot;disk&quot;</code> is accepted even if it&#x2019;s not a proper value for <code>torch.device</code>.`,name:"device_map"},{anchor:"accelerate.dispatch_model.main_device",description:`<strong>main_device</strong> (<code>str</code>, <code>int</code> or <code>torch.device</code>, <em>optional</em>) &#x2014;
The main execution device. Will default to the first device in the <code>device_map</code> different from <code>&quot;cpu&quot;</code> or
<code>&quot;disk&quot;</code>.`,name:"main_device"},{anchor:"accelerate.dispatch_model.state_dict",description:`<strong>state_dict</strong> (<code>Dict[str, torch.Tensor]</code>, <em>optional</em>) &#x2014;
The state dict of the part of the model that will be kept on CPU.`,name:"state_dict"},{anchor:"accelerate.dispatch_model.offload_dir",description:`<strong>offload_dir</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
The folder in which to offload the model weights (or where the model weights are already offloaded).`,name:"offload_dir"},{anchor:"accelerate.dispatch_model.offload_index",description:`<strong>offload_index</strong> (<code>Dict</code>, <em>optional</em>) &#x2014;
A dictionary from weight name to their information (<code>dtype</code>/ <code>shape</code> or safetensors filename). Will default
to the index saved in <code>save_folder</code>.`,name:"offload_index"},{anchor:"accelerate.dispatch_model.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to offload the buffers with the model parameters.`,name:"offload_buffers"},{anchor:"accelerate.dispatch_model.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.dispatch_model.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of classes whose instances should load all their weights (even in the submodules) at the beginning
of the forward. This should only be used for classes that have submodules which are registered but not
called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward,
<code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.dispatch_model.force_hooks",description:`<strong>force_hooks</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to force device hooks to be attached to the model even if all layers are dispatched to a
single device.`,name:"force_hooks"}]}),s(2),n(y);var K=e(y,2);o(K,{title:"load_checkpoint_and_dispatch",local:"accelerate.load_checkpoint_and_dispatch",headingTag:"h3"});var w=e(K,2),ee=t(w);a(ee,{name:"accelerate.load_checkpoint_and_dispatch",anchor:"accelerate.load_checkpoint_and_dispatch",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/big_modeling.py#L520",parameters:[{name:"model",val:": Module"},{name:"checkpoint",val:": typing.Union[str, os.PathLike]"},{name:"device_map",val:": typing.Union[str, dict[str, typing.Union[int, str, torch.device]], NoneType] = None"},{name:"max_memory",val:": typing.Optional[dict[typing.Union[int, str], typing.Union[int, str]]] = None"},{name:"no_split_module_classes",val:": typing.Optional[list[str]] = None"},{name:"offload_folder",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"offload_buffers",val:": bool = False"},{name:"dtype",val:": typing.Union[str, torch.dtype, NoneType] = None"},{name:"offload_state_dict",val:": typing.Optional[bool] = None"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"force_hooks",val:": bool = False"},{name:"strict",val:": bool = False"},{name:"full_state_dict",val:": bool = True"},{name:"broadcast_from_rank0",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.load_checkpoint_and_dispatch.model",description:"<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014; The model in which we want to load a checkpoint.",name:"model"},{anchor:"accelerate.load_checkpoint_and_dispatch.checkpoint",description:`<strong>checkpoint</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
The folder checkpoint to load. It can be:</p>
<ul>
<li>a path to a file containing a whole model state dict</li>
<li>a path to a <code>.json</code> file containing the index to a sharded checkpoint</li>
<li>a path to a folder containing a unique <code>.index.json</code> file and the shards of a checkpoint.</li>
</ul>`,name:"checkpoint"},{anchor:"accelerate.load_checkpoint_and_dispatch.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[int, str, torch.device]]</code>, <em>optional</em>) &#x2014;
A map that specifies where each submodule should go. It doesn&#x2019;t need to be refined to each parameter/buffer
name, once a given module name is inside, every submodule of it will be sent to the same device.</p>
<p>To have Accelerate compute the most optimized <code>device_map</code> automatically, set <code>device_map=&quot;auto&quot;</code>. For more
information about each option see <a href="../concept_guides/big_model_inference#designing-a-device-map">here</a>.
Defaults to None, which means <a href="/docs/accelerate/pr_4143/en/package_reference/big_modeling#accelerate.dispatch_model">dispatch_model()</a> will not be called.`,name:"device_map"},{anchor:"accelerate.load_checkpoint_and_dispatch.max_memory",description:`<strong>max_memory</strong> (<code>Dict</code>, <em>optional</em>) &#x2014;
A dictionary device identifier to maximum memory. Will default to the maximum memory available for each GPU
and the available CPU RAM if unset.`,name:"max_memory"},{anchor:"accelerate.load_checkpoint_and_dispatch.no_split_module_classes",description:`<strong>no_split_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of layer class names that should never be split across device (for instance any layer that has a
residual connection).`,name:"no_split_module_classes"},{anchor:"accelerate.load_checkpoint_and_dispatch.offload_folder",description:`<strong>offload_folder</strong> (<code>str</code> or <code>os.PathLike</code>, <em>optional</em>) &#x2014;
If the <code>device_map</code> contains any value <code>&quot;disk&quot;</code>, the folder where we will offload weights.`,name:"offload_folder"},{anchor:"accelerate.load_checkpoint_and_dispatch.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as
well as the parameters.`,name:"offload_buffers"},{anchor:"accelerate.load_checkpoint_and_dispatch.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) &#x2014;
If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.load_checkpoint_and_dispatch.offload_state_dict",description:`<strong>offload_state_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If <code>True</code>, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if
the weight of the CPU state dict + the biggest shard does not fit. Will default to <code>True</code> if the device map
picked contains <code>&quot;disk&quot;</code> values.`,name:"offload_state_dict"},{anchor:"accelerate.load_checkpoint_and_dispatch.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.load_checkpoint_and_dispatch.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of classes whose instances should load all their weights (even in the submodules) at the beginning
of the forward. This should only be used for classes that have submodules which are registered but not
called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward,
<code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.load_checkpoint_and_dispatch.force_hooks",description:`<strong>force_hooks</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to force device hooks to be attached to the model even if all layers are dispatched to a
single device.`,name:"force_hooks"},{anchor:"accelerate.load_checkpoint_and_dispatch.strict",description:`<strong>strict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model&#x2019;s
state_dict.`,name:"strict"},{anchor:"accelerate.load_checkpoint_and_dispatch.full_state_dict",description:`<strong>full_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014; if this is set to <code>True</code>, all the tensors in the
loaded state_dict will be gathered. No ShardedTensor and DTensor will be in the loaded state_dict.`,name:"full_state_dict"},{anchor:"accelerate.load_checkpoint_and_dispatch.broadcast_from_rank0",description:`<strong>broadcast_from_rank0</strong> (<code>False</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014; when the option is <code>True</code>, a distributed
<code>ProcessGroup</code> must be initialized. rank0 should receive a full state_dict and will broadcast the tensors
in the state_dict one by one to other ranks. Other ranks will receive the tensors and shard (if applicable)
according to the local shards in the model.`,name:"broadcast_from_rank0"}]});var Be=e(ee,4);_(Be,{anchor:"accelerate.load_checkpoint_and_dispatch.example",children:(d,l)=>{var c=f(),r=e(m(c),2);p(r,{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBpbml0X2VtcHR5X3dlaWdodHMlMkMlMjBsb2FkX2NoZWNrcG9pbnRfYW5kX2Rpc3BhdGNoJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGhmX2h1Yl9kb3dubG9hZCUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvQ29uZmlnJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEElMjMlMjBEb3dubG9hZCUyMHRoZSUyMFdlaWdodHMlMEFjaGVja3BvaW50JTIwJTNEJTIwJTIyRWxldXRoZXJBSSUyRmdwdC1qLTZCJTIyJTBBd2VpZ2h0c19sb2NhdGlvbiUyMCUzRCUyMGhmX2h1Yl9kb3dubG9hZChjaGVja3BvaW50JTJDJTIwJTIycHl0b3JjaF9tb2RlbC5iaW4lMjIpJTBBJTBBJTIzJTIwQ3JlYXRlJTIwYSUyMG1vZGVsJTIwYW5kJTIwaW5pdGlhbGl6ZSUyMGl0JTIwd2l0aCUyMGVtcHR5JTIwd2VpZ2h0cyUwQWNvbmZpZyUyMCUzRCUyMEF1dG9Db25maWcuZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBd2l0aCUyMGluaXRfZW1wdHlfd2VpZ2h0cygpJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX2NvbmZpZyhjb25maWcpJTBBJTBBJTIzJTIwTG9hZCUyMHRoZSUyMGNoZWNrcG9pbnQlMjBhbmQlMjBkaXNwYXRjaCUyMGl0JTIwdG8lMjB0aGUlMjByaWdodCUyMGRldmljZXMlMEFtb2RlbCUyMCUzRCUyMGxvYWRfY2hlY2twb2ludF9hbmRfZGlzcGF0Y2goJTBBJTIwJTIwJTIwJTIwbW9kZWwlMkMlMjB3ZWlnaHRzX2xvY2F0aW9uJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBub19zcGxpdF9tb2R1bGVfY2xhc3NlcyUzRCU1QiUyMkdQVEpCbG9jayUyMiU1RCUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> init_empty_weights, load_checkpoint_and_dispatch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> hf_hub_download
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoConfig, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Download the Weights</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;EleutherAI/gpt-j-6B&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>weights_location = hf_hub_download(checkpoint, <span class="hljs-string">&quot;pytorch_model.bin&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Create a model and initialize it with empty weights</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>config = AutoConfig.from_pretrained(checkpoint)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> init_empty_weights():
<span class="hljs-meta">... </span>    model = AutoModelForCausalLM.from_config(config)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Load the checkpoint and dispatch it to the right devices</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = load_checkpoint_and_dispatch(
<span class="hljs-meta">... </span>    model, weights_location, device_map=<span class="hljs-string">&quot;auto&quot;</span>, no_split_module_classes=[<span class="hljs-string">&quot;GPTJBlock&quot;</span>]
<span class="hljs-meta">... </span>)`,lang:"python",wrap:!1}),i(d,c)},$$slots:{default:!0}}),n(w);var oe=e(w,2);o(oe,{title:"load_checkpoint_in_model",local:"accelerate.load_checkpoint_in_model",headingTag:"h3"});var x=e(oe,2),ae=t(x);a(ae,{name:"accelerate.load_checkpoint_in_model",anchor:"accelerate.load_checkpoint_in_model",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/utils/modeling.py#L1805",parameters:[{name:"model",val:": Module"},{name:"checkpoint",val:": typing.Union[str, os.PathLike]"},{name:"device_map",val:": typing.Optional[dict[str, typing.Union[int, str, torch.device]]] = None"},{name:"offload_folder",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"dtype",val:": typing.Union[str, torch.dtype, NoneType] = None"},{name:"offload_state_dict",val:": bool = False"},{name:"offload_buffers",val:": bool = False"},{name:"keep_in_fp32_modules",val:": typing.Optional[list[str]] = None"},{name:"offload_8bit_bnb",val:": bool = False"},{name:"strict",val:": bool = False"},{name:"full_state_dict",val:": bool = True"},{name:"broadcast_from_rank0",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.load_checkpoint_in_model.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014;
The model in which we want to load a checkpoint.`,name:"model"},{anchor:"accelerate.load_checkpoint_in_model.checkpoint",description:`<strong>checkpoint</strong> (<code>str</code> or <code>os.PathLike</code>) &#x2014;
The folder checkpoint to load. It can be:
<ul>
<li>a path to a file containing a whole model state dict</li>
<li>a path to a <code>.json</code> file containing the index to a sharded checkpoint</li>
<li>a path to a folder containing a unique <code>.index.json</code> file and the shards of a checkpoint.</li>
<li>a path to a folder containing a unique pytorch_model.bin or a model.safetensors file.</li>
</ul>`,name:"checkpoint"},{anchor:"accelerate.load_checkpoint_in_model.device_map",description:`<strong>device_map</strong> (<code>Dict[str, Union[int, str, torch.device]]</code>, <em>optional</em>) &#x2014;
A map that specifies where each submodule should go. It doesn&#x2019;t need to be refined to each parameter/buffer
name, once a given module name is inside, every submodule of it will be sent to the same device.`,name:"device_map"},{anchor:"accelerate.load_checkpoint_in_model.offload_folder",description:`<strong>offload_folder</strong> (<code>str</code> or <code>os.PathLike</code>, <em>optional</em>) &#x2014;
If the <code>device_map</code> contains any value <code>&quot;disk&quot;</code>, the folder where we will offload weights.`,name:"offload_folder"},{anchor:"accelerate.load_checkpoint_in_model.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) &#x2014;
If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.load_checkpoint_in_model.offload_state_dict",description:`<strong>offload_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
If <code>True</code>, will temporarily offload the CPU state dict on the hard drive to avoid getting out of CPU RAM if
the weight of the CPU state dict + the biggest shard does not fit.`,name:"offload_state_dict"},{anchor:"accelerate.load_checkpoint_in_model.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to include the buffers in the weights offloaded to disk.`,name:"offload_buffers"},{anchor:"accelerate.load_checkpoint_in_model.keep_in_fp32_modules(List[str],",description:`<strong>keep_in_fp32_modules(<code>List[str]</code>,</strong> <em>optional</em>) &#x2014;
A list of the modules that we keep in <code>torch.float32</code> dtype.`,name:"keep_in_fp32_modules(List[str],"},{anchor:"accelerate.load_checkpoint_in_model.offload_8bit_bnb",description:`<strong>offload_8bit_bnb</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to enable offload of 8-bit modules on cpu/disk.`,name:"offload_8bit_bnb"},{anchor:"accelerate.load_checkpoint_in_model.strict",description:`<strong>strict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to strictly enforce that the keys in the checkpoint state_dict match the keys of the model&#x2019;s
state_dict.`,name:"strict"},{anchor:"accelerate.load_checkpoint_in_model.full_state_dict",description:`<strong>full_state_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014; if this is set to <code>True</code>, all the tensors in the
loaded state_dict will be gathered. No ShardedTensor and DTensor will be in the loaded state_dict.`,name:"full_state_dict"},{anchor:"accelerate.load_checkpoint_in_model.broadcast_from_rank0",description:`<strong>broadcast_from_rank0</strong> (<code>False</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014; when the option is <code>True</code>, a distributed
<code>ProcessGroup</code> must be initialized. rank0 should receive a full state_dict and will broadcast the tensors
in the state_dict one by one to other ranks. Other ranks will receive the tensors and shard (if applicable)
according to the local shards in the model.`,name:"broadcast_from_rank0"}]});var Ce=e(ae,4);u(Ce,{warning:!0,children:(d,l)=>{var c=po();i(d,c)},$$slots:{default:!0}}),n(x);var te=e(x,2);o(te,{title:"infer_auto_device_map",local:"accelerate.infer_auto_device_map",headingTag:"h3"});var M=e(te,2),ne=t(M);a(ne,{name:"accelerate.infer_auto_device_map",anchor:"accelerate.infer_auto_device_map",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/utils/modeling.py#L1295",parameters:[{name:"model",val:": Module"},{name:"max_memory",val:": typing.Optional[dict[typing.Union[int, str], typing.Union[int, str]]] = None"},{name:"no_split_module_classes",val:": typing.Optional[list[str]] = None"},{name:"dtype",val:": typing.Union[str, torch.dtype, NoneType] = None"},{name:"special_dtypes",val:": typing.Optional[dict[str, typing.Union[str, torch.dtype]]] = None"},{name:"verbose",val:": bool = False"},{name:"clean_result",val:": bool = True"},{name:"offload_buffers",val:": bool = False"},{name:"fallback_allocation",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.infer_auto_device_map.model",description:`<strong>model</strong> (<code>torch.nn.Module</code>) &#x2014;
The model to analyze.`,name:"model"},{anchor:"accelerate.infer_auto_device_map.max_memory",description:`<strong>max_memory</strong> (<code>Dict</code>, <em>optional</em>) &#x2014;
A dictionary device identifier to maximum memory. Will default to the maximum memory available if unset.
Example: <code>max_memory={0: &quot;1GB&quot;}</code>.`,name:"max_memory"},{anchor:"accelerate.infer_auto_device_map.no_split_module_classes",description:`<strong>no_split_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of layer class names that should never be split across device (for instance any layer that has a
residual connection).`,name:"no_split_module_classes"},{anchor:"accelerate.infer_auto_device_map.dtype",description:`<strong>dtype</strong> (<code>str</code> or <code>torch.dtype</code>, <em>optional</em>) &#x2014;
If provided, the weights will be converted to that type when loaded.`,name:"dtype"},{anchor:"accelerate.infer_auto_device_map.special_dtypes",description:`<strong>special_dtypes</strong> (<code>Dict[str, Union[str, torch.device]]</code>, <em>optional</em>) &#x2014;
If provided, special dtypes to consider for some specific weights (will override dtype used as default for
all weights).`,name:"special_dtypes"},{anchor:"accelerate.infer_auto_device_map.verbose",description:`<strong>verbose</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to provide debugging statements as the function builds the device_map.`,name:"verbose"},{anchor:"accelerate.infer_auto_device_map.clean_result",description:`<strong>clean_result</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Clean the resulting device_map by grouping all submodules that go on the same device together.`,name:"clean_result"},{anchor:"accelerate.infer_auto_device_map.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
In the layers that are offloaded on the CPU or the hard drive, whether or not to offload the buffers as
well as the parameters.`,name:"offload_buffers"},{anchor:"accelerate.infer_auto_device_map.fallback_allocation",description:`<strong>fallback_allocation</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
When regular allocation fails, try to allocate a module that fits in the size limit using BFS.`,name:"fallback_allocation"}]});var Ie=e(ne,6);u(Ie,{children:(d,l)=>{var c=mo();i(d,c)},$$slots:{default:!0}}),n(M);var de=e(M,2);o(de,{title:"Hooks",local:"hooks",headingTag:"h2"});var ce=e(de,2);o(ce,{title:"ModelHook",local:"accelerate.hooks.ModelHook",headingTag:"h3"});var T=e(ce,2),se=t(T);a(se,{name:"class accelerate.hooks.ModelHook",anchor:"accelerate.hooks.ModelHook",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L58",parameters:[]});var U=e(se,8),Ae=t(U);a(Ae,{name:"detach_hook",anchor:"accelerate.hooks.ModelHook.detach_hook",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L106",parameters:[{name:"module",val:""}],parametersDescription:[{anchor:"accelerate.hooks.ModelHook.detach_hook.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014; The module detached from this hook.",name:"module"}]}),s(2),n(U);var j=e(U,2),He=t(j);a(He,{name:"init_hook",anchor:"accelerate.hooks.ModelHook.init_hook",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L70",parameters:[{name:"module",val:""}],parametersDescription:[{anchor:"accelerate.hooks.ModelHook.init_hook.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014; The module attached to this hook.",name:"module"}]}),s(2),n(j);var J=e(j,2),De=t(J);a(De,{name:"post_forward",anchor:"accelerate.hooks.ModelHook.post_forward",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L93",parameters:[{name:"module",val:""},{name:"output",val:""}],parametersDescription:[{anchor:"accelerate.hooks.ModelHook.post_forward.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014; The module whose forward pass been executed just before this event.",name:"module"},{anchor:"accelerate.hooks.ModelHook.post_forward.output",description:"<strong>output</strong> (<code>Any</code>) &#x2014; The output of the module.",name:"output"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The processed <code>output</code>.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Any</code></p>
`}),s(2),n(J);var le=e(J,2),Re=t(le);a(Re,{name:"pre_forward",anchor:"accelerate.hooks.ModelHook.pre_forward",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L79",parameters:[{name:"module",val:""},{name:"*args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"accelerate.hooks.ModelHook.pre_forward.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014; The module whose forward pass will be executed just after this event.",name:"module"},{anchor:"accelerate.hooks.ModelHook.pre_forward.args",description:"<strong>args</strong> (<code>Tuple[Any]</code>) &#x2014; The positional arguments passed to the module.",name:"args"},{anchor:"accelerate.hooks.ModelHook.pre_forward.kwargs",description:"<strong>kwargs</strong> (<code>Dict[Str, Any]</code>) &#x2014; The keyword arguments passed to the module.",name:"kwargs"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A tuple with the treated <code>args</code> and <code>kwargs</code>.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple[Tuple[Any], Dict[Str, Any]]</code></p>
`}),s(2),n(le),n(T);var ie=e(T,2);o(ie,{title:"AlignDevicesHook",local:"accelerate.hooks.AlignDevicesHook",headingTag:"h3"});var G=e(ie,2),Xe=t(G);a(Xe,{name:"class accelerate.hooks.AlignDevicesHook",anchor:"accelerate.hooks.AlignDevicesHook",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L242",parameters:[{name:"execution_device",val:": typing.Union[int, str, torch.device, NoneType] = None"},{name:"offload",val:": bool = False"},{name:"io_same_device",val:": bool = False"},{name:"weights_map",val:": typing.Optional[collections.abc.Mapping] = None"},{name:"offload_buffers",val:": bool = False"},{name:"place_submodules",val:": bool = False"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"tied_params_map",val:": typing.Optional[dict[int, dict[torch.device, torch.Tensor]]] = None"}],parametersDescription:[{anchor:"accelerate.hooks.AlignDevicesHook.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) &#x2014;
The device on which inputs and model weights should be placed before the forward pass.`,name:"execution_device"},{anchor:"accelerate.hooks.AlignDevicesHook.offload",description:`<strong>offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the weights should be offloaded after the forward pass.`,name:"offload"},{anchor:"accelerate.hooks.AlignDevicesHook.io_same_device",description:`<strong>io_same_device</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the output should be placed on the same device as the input was.`,name:"io_same_device"},{anchor:"accelerate.hooks.AlignDevicesHook.weights_map",description:`<strong>weights_map</strong> (<code>Mapping[str, torch.Tensor]</code>, <em>optional</em>) &#x2014;
When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.`,name:"weights_map"},{anchor:"accelerate.hooks.AlignDevicesHook.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to include the associated module&#x2019;s buffers when offloading.`,name:"offload_buffers"},{anchor:"accelerate.hooks.AlignDevicesHook.place_submodules",description:`<strong>place_submodules</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to place the submodules on <code>execution_device</code> during the <code>init_hook</code> event.`,name:"place_submodules"}]}),s(2),n(G);var re=e(G,2);o(re,{title:"SequentialHook",local:"accelerate.hooks.SequentialHook",headingTag:"h3"});var N=e(re,2),Le=t(N);a(Le,{name:"class accelerate.hooks.SequentialHook",anchor:"accelerate.hooks.SequentialHook",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L116",parameters:[{name:"*hooks",val:""}]}),s(2),n(N);var he=e(N,2);o(he,{title:"LayerwiseCastingHook",local:"accelerate.hooks.LayerwiseCastingHook",headingTag:"h3"});var F=e(he,2),ze=t(F);a(ze,{name:"class accelerate.hooks.LayerwiseCastingHook",anchor:"accelerate.hooks.LayerwiseCastingHook",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L784",parameters:[{name:"storage_dtype",val:": dtype"},{name:"compute_dtype",val:": dtype"},{name:"non_blocking",val:": bool"}]}),s(2),n(F);var pe=e(F,2);o(pe,{title:"Adding Hooks",local:"adding-hooks",headingTag:"h2"});var me=e(pe,2);o(me,{title:"add_hook_to_module",local:"accelerate.hooks.add_hook_to_module",headingTag:"h3"});var W=e(me,2),_e=t(W);a(_e,{name:"accelerate.hooks.add_hook_to_module",anchor:"accelerate.hooks.add_hook_to_module",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L147",parameters:[{name:"module",val:": Module"},{name:"hook",val:": ModelHook"},{name:"append",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.hooks.add_hook_to_module.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014;
The module to attach a hook to.`,name:"module"},{anchor:"accelerate.hooks.add_hook_to_module.hook",description:`<strong>hook</strong> (<code>ModelHook</code>) &#x2014;
The hook to attach.`,name:"hook"},{anchor:"accelerate.hooks.add_hook_to_module.append",description:`<strong>append</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the hook should be chained with an existing one (if module already contains a hook) or not.`,name:"append"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The same module, with the hook attached (the module is modified in place, so the result can
be discarded).</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.nn.Module</code></p>
`});var Ve=e(_e,4);u(Ve,{warning:!0,children:(d,l)=>{var c=_o();i(d,c)},$$slots:{default:!0}}),n(W);var ue=e(W,2);o(ue,{title:"attach_execution_device_hook",local:"accelerate.hooks.attach_execution_device_hook",headingTag:"h3"});var Z=e(ue,2),Pe=t(Z);a(Pe,{name:"accelerate.hooks.attach_execution_device_hook",anchor:"accelerate.hooks.attach_execution_device_hook",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L443",parameters:[{name:"module",val:": Module"},{name:"execution_device",val:": typing.Union[int, str, torch.device]"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"tied_params_map",val:": typing.Optional[dict[int, dict[torch.device, torch.Tensor]]] = None"}],parametersDescription:[{anchor:"accelerate.hooks.attach_execution_device_hook.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014;
The module where we want to attach the hooks.`,name:"module"},{anchor:"accelerate.hooks.attach_execution_device_hook.execution_device",description:`<strong>execution_device</strong> (<code>int</code>, <code>str</code> or <code>torch.device</code>) &#x2014;
The device on which inputs and model weights should be placed before the forward pass.`,name:"execution_device"},{anchor:"accelerate.hooks.attach_execution_device_hook.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.hooks.attach_execution_device_hook.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of classes whose instances should load all their weights (even in the submodules) at the beginning
of the forward. This should only be used for classes that have submodules which are registered but not
called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward,
<code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.hooks.attach_execution_device_hook.tied_params_map",description:`<strong>tied_params_map</strong> (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], <em>optional</em>, defaults to <code>None</code>) &#x2014;
A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution
device, this parameter is useful to reuse the first available pointer of a shared weight for all others,
instead of duplicating memory.`,name:"tied_params_map"}]}),s(2),n(Z);var fe=e(Z,2);o(fe,{title:"attach_align_device_hook",local:"accelerate.hooks.attach_align_device_hook",headingTag:"h3"});var B=e(fe,2),Qe=t(B);a(Qe,{name:"accelerate.hooks.attach_align_device_hook",anchor:"accelerate.hooks.attach_align_device_hook",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L491",parameters:[{name:"module",val:": Module"},{name:"execution_device",val:": typing.Optional[torch.device] = None"},{name:"offload",val:": bool = False"},{name:"weights_map",val:": typing.Optional[collections.abc.Mapping] = None"},{name:"offload_buffers",val:": bool = False"},{name:"module_name",val:": str = ''"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"tied_params_map",val:": typing.Optional[dict[int, dict[torch.device, torch.Tensor]]] = None"}],parametersDescription:[{anchor:"accelerate.hooks.attach_align_device_hook.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014;
The module where we want to attach the hooks.`,name:"module"},{anchor:"accelerate.hooks.attach_align_device_hook.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) &#x2014;
The device on which inputs and model weights should be placed before the forward pass.`,name:"execution_device"},{anchor:"accelerate.hooks.attach_align_device_hook.offload",description:`<strong>offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the weights should be offloaded after the forward pass.`,name:"offload"},{anchor:"accelerate.hooks.attach_align_device_hook.weights_map",description:`<strong>weights_map</strong> (<code>Mapping[str, torch.Tensor]</code>, <em>optional</em>) &#x2014;
When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.`,name:"weights_map"},{anchor:"accelerate.hooks.attach_align_device_hook.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to include the associated module&#x2019;s buffers when offloading.`,name:"offload_buffers"},{anchor:"accelerate.hooks.attach_align_device_hook.module_name",description:`<strong>module_name</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&quot;</code>) &#x2014;
The name of the module.`,name:"module_name"},{anchor:"accelerate.hooks.attach_align_device_hook.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.hooks.attach_align_device_hook.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of classes whose instances should load all their weights (even in the submodules) at the beginning
of the forward. This should only be used for classes that have submodules which are registered but not
called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward,
<code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.hooks.attach_align_device_hook.tied_params_map",description:`<strong>tied_params_map</strong> (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], <em>optional</em>, defaults to <code>None</code>) &#x2014;
A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution
device, this parameter is useful to reuse the first available pointer of a shared weight for all others,
instead of duplicating memory.`,name:"tied_params_map"}]}),s(2),n(B);var ge=e(B,2);o(ge,{title:"attach_align_device_hook_on_blocks",local:"accelerate.hooks.attach_align_device_hook_on_blocks",headingTag:"h3"});var C=e(ge,2),Se=t(C);a(Se,{name:"accelerate.hooks.attach_align_device_hook_on_blocks",anchor:"accelerate.hooks.attach_align_device_hook_on_blocks",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L586",parameters:[{name:"module",val:": Module"},{name:"execution_device",val:": typing.Union[torch.device, dict[str, torch.device], NoneType] = None"},{name:"offload",val:": typing.Union[bool, dict[str, bool]] = False"},{name:"weights_map",val:": typing.Optional[collections.abc.Mapping] = None"},{name:"offload_buffers",val:": bool = False"},{name:"module_name",val:": str = ''"},{name:"skip_keys",val:": typing.Union[str, list[str], NoneType] = None"},{name:"preload_module_classes",val:": typing.Optional[list[str]] = None"},{name:"tied_params_map",val:": typing.Optional[dict[int, dict[torch.device, torch.Tensor]]] = None"}],parametersDescription:[{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014;
The module where we want to attach the hooks.`,name:"module"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code> or <code>Dict[str, torch.device]</code>, <em>optional</em>) &#x2014;
The device on which inputs and model weights should be placed before the forward pass. It can be one device
for the whole module, or a dictionary mapping module name to device.`,name:"execution_device"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.offload",description:`<strong>offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the weights should be offloaded after the forward pass. It can be one boolean for the whole
module, or a dictionary mapping module name to boolean.`,name:"offload"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.weights_map",description:`<strong>weights_map</strong> (<code>Mapping[str, torch.Tensor]</code>, <em>optional</em>) &#x2014;
When the model weights are offloaded, a (potentially lazy) map from param names to the tensor values.`,name:"weights_map"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.offload_buffers",description:`<strong>offload_buffers</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to include the associated module&#x2019;s buffers when offloading.`,name:"offload_buffers"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.module_name",description:`<strong>module_name</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&quot;</code>) &#x2014;
The name of the module.`,name:"module_name"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.skip_keys",description:`<strong>skip_keys</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
A list of keys to ignore when moving inputs or outputs between devices.`,name:"skip_keys"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.preload_module_classes",description:`<strong>preload_module_classes</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
A list of classes whose instances should load all their weights (even in the submodules) at the beginning
of the forward. This should only be used for classes that have submodules which are registered but not
called directly during the forward, for instance if a <code>dense</code> linear layer is registered, but at forward,
<code>dense.weight</code> and <code>dense.bias</code> are used in some operations instead of calling <code>dense</code> directly.`,name:"preload_module_classes"},{anchor:"accelerate.hooks.attach_align_device_hook_on_blocks.tied_params_map",description:`<strong>tied_params_map</strong> (Optional[Dict[int, Dict[torch.device, torch.Tensor]]], <em>optional</em>, defaults to <code>None</code>) &#x2014;
A map of data pointers to dictionaries of devices to already dispatched tied weights. For a given execution
device, this parameter is useful to reuse the first available pointer of a shared weight for all others,
instead of duplicating memory.`,name:"tied_params_map"}]}),s(2),n(C);var ve=e(C,2);o(ve,{title:"attach_layerwise_casting_hooks",local:"accelerate.big_modeling.attach_layerwise_casting_hooks",headingTag:"h3"});var I=e(ve,2),be=t(I);a(be,{name:"accelerate.big_modeling.attach_layerwise_casting_hooks",anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/big_modeling.py#L661",parameters:[{name:"module",val:": Module"},{name:"storage_dtype",val:": dtype"},{name:"compute_dtype",val:": dtype"},{name:"skip_modules_pattern",val:": typing.Union[str, tuple[str, ...], NoneType] = None"},{name:"skip_modules_classes",val:": typing.Optional[tuple[type[torch.nn.Module], ...]] = None"},{name:"non_blocking",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014;
The module whose leaf modules will be cast to a high precision dtype for computation, and to a low
precision dtype for storage.`,name:"module"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.storage_dtype",description:`<strong>storage_dtype</strong> (<code>torch.dtype</code>) &#x2014;
The dtype to cast the module to before/after the forward pass for storage.`,name:"storage_dtype"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.compute_dtype",description:`<strong>compute_dtype</strong> (<code>torch.dtype</code>) &#x2014;
The dtype to cast the module to during the forward pass for computation.`,name:"compute_dtype"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.skip_modules_pattern",description:`<strong>skip_modules_pattern</strong> (<code>tuple[str, ...]</code>, defaults to <code>None</code>) &#x2014;
A list of patterns to match the names of the modules to skip during the layerwise casting process. If set
to <code>None</code> alongside <code>skip_modules_classes</code> being <code>None</code>, the layerwise casting is applied directly to the
module instead of its internal submodules.`,name:"skip_modules_pattern"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.skip_modules_classes",description:`<strong>skip_modules_classes</strong> (<code>tuple[type[torch.nn.Module], ...]</code>, defaults to <code>None</code>) &#x2014;
A list of module classes to skip during the layerwise casting process.`,name:"skip_modules_classes"},{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.non_blocking",description:`<strong>non_blocking</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
If <code>True</code>, the weight casting operations are non-blocking.`,name:"non_blocking"}]});var ke=e(be,4);_(ke,{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.example",children:(d,l)=>{var c=f(),r=e(m(c),2);p(r,{code:"ZnJvbSUyMGFjY2VsZXJhdGUuaG9va3MlMjBpbXBvcnQlMjBhdHRhY2hfbGF5ZXJ3aXNlX2Nhc3RpbmdfaG9va3MlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEFpbXBvcnQlMjB0b3JjaCUwQSUwQSUyMyUyME1vZGVsJTBBY2hlY2twb2ludCUyMCUzRCUyMCUyMkVsZXV0aGVyQUklMkZncHQtai02QiUyMiUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKGNoZWNrcG9pbnQpJTBBJTBBJTIzJTIwQXR0YWNoJTIwaG9va3MlMjBhbmQlMjBwZXJmb3JtJTIwaW5mZXJlbmNlJTBBYXR0YWNoX2xheWVyd2lzZV9jYXN0aW5nX2hvb2tzKG1vZGVsJTJDJTIwc3RvcmFnZV9kdHlwZSUzRHRvcmNoLmZsb2F0OF9lNG0zZm4lMkMlMjBjb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYpJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsKC4uLik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> accelerate.hooks <span class="hljs-keyword">import</span> attach_layerwise_casting_hooks
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>checkpoint = <span class="hljs-string">&quot;EleutherAI/gpt-j-6B&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(checkpoint)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Attach hooks and perform inference</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>attach_layerwise_casting_hooks(model, storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    model(...)`,lang:"python",wrap:!1}),i(d,c)},$$slots:{default:!0}});var Ee=e(ke,4);_(Ee,{anchor:"accelerate.big_modeling.attach_layerwise_casting_hooks.example-2",children:(d,l)=>{p(d,{code:"YXR0YWNoX2xheWVyd2lzZV9jYXN0aW5nX2hvb2tzKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTJDJTIwc3RvcmFnZV9kdHlwZSUzRHRvcmNoLmZsb2F0OF9lNG0zZm4lMkMlMjBjb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYlMkMlMjBza2lwX21vZHVsZXNfcGF0dGVybiUzRCU1QiUyMm5vcm0lMjIlNUQlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>attach_layerwise_casting_hooks(
<span class="hljs-meta">... </span>    model, storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16, skip_modules_pattern=[<span class="hljs-string">&quot;norm&quot;</span>]
<span class="hljs-meta">... </span>)`,lang:"py",wrap:!1})},$$slots:{default:!0}}),n(I);var ye=e(I,2);o(ye,{title:"Removing Hooks",local:"removing-hooks",headingTag:"h2"});var we=e(ye,2);o(we,{title:"remove_hook_from_module",local:"accelerate.hooks.remove_hook_from_module",headingTag:"h3"});var A=e(we,2),qe=t(A);a(qe,{name:"accelerate.hooks.remove_hook_from_module",anchor:"accelerate.hooks.remove_hook_from_module",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L205",parameters:[{name:"module",val:": Module"},{name:"recurse",val:" = False"}],parametersDescription:[{anchor:"accelerate.hooks.remove_hook_from_module.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014; The module to attach a hook to.",name:"module"},{anchor:"accelerate.hooks.remove_hook_from_module.recurse",description:"<strong>recurse</strong> (<code>bool</code>, <strong>optional</strong>) &#x2014; Whether to remove the hooks recursively",name:"recurse"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The same module, with the hook detached (the module is modified in place, so the result can
be discarded).</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.nn.Module</code></p>
`}),s(2),n(A);var xe=e(A,2);o(xe,{title:"remove_hook_from_submodules",local:"accelerate.hooks.remove_hook_from_submodules",headingTag:"h3"});var H=e(xe,2),Ye=t(H);a(Ye,{name:"accelerate.hooks.remove_hook_from_submodules",anchor:"accelerate.hooks.remove_hook_from_submodules",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/hooks.py#L574",parameters:[{name:"module",val:": Module"}],parametersDescription:[{anchor:"accelerate.hooks.remove_hook_from_submodules.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014; The module on which to remove all hooks.",name:"module"}]}),s(2),n(H);var Me=e(H,2);o(Me,{title:"Utilities",local:"utilities",headingTag:"h2"});var Te=e(Me,2);o(Te,{title:"has_offloaded_params",local:"accelerate.utils.has_offloaded_params",headingTag:"h3"});var D=e(Te,2),Oe=t(D);a(Oe,{name:"accelerate.utils.has_offloaded_params",anchor:"accelerate.utils.has_offloaded_params",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/utils/modeling.py#L2150",parameters:[{name:"module",val:": Module"}],parametersDescription:[{anchor:"accelerate.utils.has_offloaded_params.module",description:"<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014; The module to check for an offload hook.",name:"module"}],returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>True</code> if the module has an offload hook and offloading is enabled, <code>False</code> otherwise.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>bool</p>
`}),s(2),n(D);var Ue=e(D,2);o(Ue,{title:"align_module_device",local:"accelerate.utils.align_module_device",headingTag:"h3"});var R=e(Ue,2),$e=t(R);a($e,{name:"accelerate.utils.align_module_device",anchor:"accelerate.utils.align_module_device",source:"https://github.com/huggingface/accelerate/blob/vr_4143/src/accelerate/utils/modeling.py#L2166",parameters:[{name:"module",val:": Module"},{name:"execution_device",val:": typing.Optional[torch.device] = None"}],parametersDescription:[{anchor:"accelerate.utils.align_module_device.module",description:`<strong>module</strong> (<code>torch.nn.Module</code>) &#x2014;
Module with parameters to align.`,name:"module"},{anchor:"accelerate.utils.align_module_device.execution_device",description:`<strong>execution_device</strong> (<code>torch.device</code>, <em>optional</em>) &#x2014;
If provided, overrides the module&#x2019;s execution device within the context. Otherwise, use hook execution
device or pass`,name:"execution_device"}]}),s(2),n(R);var Ke=e(R,2);to(Ke,{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/package_reference/big_modeling.md"}),s(2),i(je,X),lo()}export{yo as component};