accelerate/pr_4097/en/_app/immutable/nodes/34.9659ef8a.js

import{s as Nt,o as Wt,n as Ot}from"../chunks/scheduler.b9285784.js";import{S as Et,i as Bt,e as c,s as o,c as i,h as Gt,a as l,d as a,b as r,f as x,g as p,j as _,k as w,l as s,m as n,n as h,t as m,o as u,p as g}from"../chunks/index.26bc89a1.js";import{T as qt}from"../chunks/Tip.e4eba3d6.js";import{C as Ut,H as N,E as Vt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.6d2489e0.js";import{D as k}from"../chunks/Docstring.05f39797.js";function Rt(le){let d,L=`<code>BatchSampler</code>s with varying batch sizes are not enabled by default. To enable this behaviour, set <code>even_batches</code>
equal to <code>False</code>`;return{c(){d=c("p"),d.innerHTML=L},l(f){d=l(f,"P",{"data-svelte-h":!0}),_(d)!=="svelte-11uxp6"&&(d.innerHTML=L)},m(f,I){n(f,d,I)},p:Ot,d(f){f&&a(d)}}}function jt(le){let d,L=`<code>BatchSampler</code>s with varying batch sizes are not enabled by default. To enable this behaviour, set <code>even_batches</code>
equal to <code>False</code>`;return{c(){d=c("p"),d.innerHTML=L},l(f){d=l(f,"P",{"data-svelte-h":!0}),_(d)!=="svelte-11uxp6"&&(d.innerHTML=L)},m(f,I){n(f,d,I)},p:Ot,d(f){f&&a(d)}}}function Xt(le){let d,L,f,I,W,ke,E,Ae,B,vt=`The internal classes Accelerate uses to prepare objects for distributed training
when calling <a href="/docs/accelerate/pr_4097/en/package_reference/accelerator#accelerate.Accelerator.prepare">prepare()</a>.`,Me,G,Ie,v,U,Ye,de,$t="Wraps a PyTorch <code>DataLoader</code> to generate batches for one of the processes only.",Ze,ie,yt=`Depending on the value of the <code>drop_last</code> attribute of the <code>dataloader</code> passed, it will either stop the iteration
at the first batch that would be too small / not present on all processes or loop with indices from the beginning.`,et,C,Ce,A,V,tt,pe,xt=`Creates a <code>torch.utils.data.DataLoader</code> that will efficiently skip the first <code>num_batches</code>. Should not be used if
the original dataloader is a <code>StatefulDataLoader</code>.`,He,R,Pe,T,j,at,he,wt=`Wraps a PyTorch <code>BatchSampler</code> to generate batches for one of the processes only. Instances of this class will
always yield a number of batches that is a round multiple of <code>num_processes</code> and that all have the same size.
Depending on the value of the <code>drop_last</code> attribute of the batch sampler passed, it will either stop the iteration
at the first batch that would be too small / not present on all processes or loop with indices from the beginning.`,ot,H,Fe,X,qe,M,J,rt,me,Tt=`Wraps a PyTorch <code>IterableDataset</code> to generate samples for one of the processes only. Instances of this class will
always yield a number of samples that is a round multiple of the actual batch size (depending of the value of
<code>split_batches</code>, this is either <code>batch_size</code> or <code>batch_size x num_processes</code>). Depending on the value of the
<code>drop_last</code> attribute of the batch sampler passed, it will either stop the iteration at the first batch that would
be too small or loop with indices from the beginning.`,Oe,K,Ne,$,Q,st,ue,zt="Subclass of <code>DataLoaderAdapter</code> that will deal with device placement and current distributed setup.",nt,ge,Dt="<strong>Available attributes:</strong>",ct,_e,Lt=`<li><p><strong>total_batch_size</strong> (<code>int</code>) — Total batch size of the dataloader across all processes.
Equal to the original batch size when <code>split_batches=True</code>; otherwise the original batch size * the total
number of processes</p></li> <li><p><strong>total_dataset_length</strong> (<code>int</code>) — Total length of the inner dataset across all processes.</p></li>`,We,Y,Ee,y,Z,lt,fe,St=`Subclass of <code>DataLoaderAdapter</code> that will iterate and preprocess on process 0 only, then dispatch on each process
their part of the batch.`,dt,be,kt="<strong>Available attributes:</strong>",it,ve,At=`<li><p><strong>total_batch_size</strong> (<code>int</code>) — Total batch size of the dataloader across all processes.
Equal to the original batch size when <code>split_batches=True</code>; otherwise the original batch size * the total
number of processes</p></li> <li><p><strong>total_dataset_length</strong> (<code>int</code>) — Total length of the inner dataset across all processes.</p></li>`,Be,ee,Ge,b,te,pt,$e,Mt="Internal wrapper around a torch optimizer.",ht,ye,It=`Conditionally will perform <code>step</code> and <code>zero_grad</code> if gradients should be synchronized when performing gradient
accumulation.`,mt,P,ae,ut,xe,Ct="Sets the optimizer to “eval” mode. Useful for optimizers like <code>schedule_free</code>",gt,F,oe,_t,we,Ht="Sets the optimizer to “train” mode. Useful for optimizers like <code>schedule_free</code>",Ue,re,Ve,z,se,ft,Te,Pt=`A wrapper around a learning rate scheduler that will only step when the optimizer(s) have a training step. Useful
to avoid making a scheduler step too fast when gradients went overflow and there was no training step (in mixed
precision training)`,bt,ze,Ft=`When performing gradient accumulation scheduler lengths should not be changed accordingly, Accelerate will always
step the scheduler to account for it.`,Re,ne,je,Se,Xe;return W=new Ut({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),E=new N({props:{title:"DataLoaders, Optimizers, and Schedulers",local:"dataloaders-optimizers-and-schedulers",headingTag:"h1"}}),G=new N({props:{title:"DataLoader utilities",local:"accelerate.data_loader.prepare_data_loader",headingTag:"h2"}}),U=new k({props:{name:"accelerate.data_loader.prepare_data_loader",anchor:"accelerate.data_loader.prepare_data_loader",parameters:[{name:"dataloader",val:": DataLoader"},{name:"device",val:": typing.Optional[torch.device] = None"},{name:"num_processes",val:": typing.Optional[int] = None"},{name:"process_index",val:": typing.Optional[int] = None"},{name:"split_batches",val:": bool = False"},{name:"put_on_device",val:": bool = False"},{name:"rng_types",val:": typing.Optional[list[typing.Union[str, accelerate.utils.dataclasses.RNGType]]] = None"},{name:"dispatch_batches",val:": typing.Optional[bool] = None"},{name:"even_batches",val:": bool = True"},{name:"slice_fn_for_dispatch",val:": typing.Optional[typing.Callable] = None"},{name:"use_seedable_sampler",val:": bool = False"},{name:"data_seed",val:": typing.Optional[int] = None"},{name:"non_blocking",val:": bool = False"},{name:"use_stateful_dataloader",val:": bool = False"},{name:"torch_device_mesh",val:" = None"}],parametersDescription:[{anchor:"accelerate.data_loader.prepare_data_loader.dataloader",description:`<strong>dataloader</strong> (<code>torch.utils.data.dataloader.DataLoader</code>) &#x2014;
The data loader to split across several devices.`,name:"dataloader"},{anchor:"accelerate.data_loader.prepare_data_loader.device",description:`<strong>device</strong> (<code>torch.device</code>) &#x2014;
The target device for the returned <code>DataLoader</code>.`,name:"device"},{anchor:"accelerate.data_loader.prepare_data_loader.num_processes",description:`<strong>num_processes</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of processes running concurrently. Will default to the value given by <a href="/docs/accelerate/pr_4097/en/package_reference/state#accelerate.PartialState">PartialState</a>.`,name:"num_processes"},{anchor:"accelerate.data_loader.prepare_data_loader.process_index",description:`<strong>process_index</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The index of the current process. Will default to the value given by <a href="/docs/accelerate/pr_4097/en/package_reference/state#accelerate.PartialState">PartialState</a>.`,name:"process_index"},{anchor:"accelerate.data_loader.prepare_data_loader.split_batches",description:`<strong>split_batches</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the resulting <code>DataLoader</code> should split the batches of the original data loader across devices or
yield full batches (in which case it will yield batches starting at the <code>process_index</code>-th and advancing of
<code>num_processes</code> batches at each iteration).</p>
<p>Another way to see this is that the observed batch size will be the same as the initial <code>dataloader</code> if
this option is set to <code>True</code>, the batch size of the initial <code>dataloader</code> multiplied by <code>num_processes</code>
otherwise.</p>
<p>Setting this option to <code>True</code> requires that the batch size of the <code>dataloader</code> is a round multiple of
<code>batch_size</code>.`,name:"split_batches"},{anchor:"accelerate.data_loader.prepare_data_loader.put_on_device",description:`<strong>put_on_device</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to put the batches on <code>device</code> (only works if the batches are nested list, tuples or
dictionaries of tensors).`,name:"put_on_device"},{anchor:"accelerate.data_loader.prepare_data_loader.rng_types",description:`<strong>rng_types</strong> (list of <code>str</code> or <a href="/docs/accelerate/pr_4097/en/package_reference/utilities#accelerate.utils.RNGType">RNGType</a>) &#x2014;
The list of random number generators to synchronize at the beginning of each iteration. Should be one or
several of:</p>
<ul>
<li><code>&quot;torch&quot;</code>: the base torch random number generator</li>
<li><code>&quot;cuda&quot;</code>: the CUDA random number generator (GPU only)</li>
<li><code>&quot;xla&quot;</code>: the XLA random number generator (TPU only)</li>
<li><code>&quot;generator&quot;</code>: the <code>torch.Generator</code> of the sampler (or batch sampler if there is no sampler in your
dataloader) or of the iterable dataset (if it exists) if the underlying dataset is of that type.</li>
</ul>`,name:"rng_types"},{anchor:"accelerate.data_loader.prepare_data_loader.dispatch_batches",description:`<strong>dispatch_batches</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, the dataloader prepared is only iterated through on the main process and then the batches
are split and broadcast to each process. Will default to <code>True</code> when the underlying dataset is an
<code>IterableDataset</code>, <code>False</code> otherwise.`,name:"dispatch_batches"},{anchor:"accelerate.data_loader.prepare_data_loader.even_batches",description:`<strong>even_batches</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
If set to <code>True</code>, in cases where the total batch size across all processes does not exactly divide the
dataset, samples at the start of the dataset will be duplicated so the batch can be divided equally among
all workers.`,name:"even_batches"},{anchor:"accelerate.data_loader.prepare_data_loader.slice_fn_for_dispatch",description:"<strong>slice_fn_for_dispatch</strong> (<code>Callable</code>, <em>optional</em><code>) -- If passed, this function will be used to slice tensors across </code>num_processes<code>. Will default to [slice_tensors()](/docs/accelerate/pr_4097/en/package_reference/utilities#accelerate.utils.slice_tensors). This argument is used only when </code>dispatch_batches<code>is set to</code>True` and will be\nignored otherwise.",name:"slice_fn_for_dispatch"},{anchor:"accelerate.data_loader.prepare_data_loader.use_seedable_sampler",description:`<strong>use_seedable_sampler</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to use the <code>SeedableRandomSampler</code> instead of a <code>RandomSampler</code> for better
reproducibility. Comes at a cost of potentially different performances due to different shuffling
algorithms but ensures results will be the <em>exact</em> same. Should be paired with <code>set_seed()</code> at every
<code>self.set_epoch</code>`,name:"use_seedable_sampler"},{anchor:"accelerate.data_loader.prepare_data_loader.data_seed",description:`<strong>data_seed</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The seed to use for the underlying generator when using <code>use_seedable_sampler</code>. If <code>None</code>, the generator
will use the current default seed from torch.`,name:"data_seed"},{anchor:"accelerate.data_loader.prepare_data_loader.non_blocking",description:`<strong>non_blocking</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
If set to <code>True</code>, dataloader will utilize non-blocking host-to-device transfers. If the dataloader has
<code>pin_memory</code> set to <code>True</code>, this will help to increase overlap between data transfer and computations.`,name:"non_blocking"},{anchor:"accelerate.data_loader.prepare_data_loader.use_stateful_dataloader",description:`<strong>use_stateful_dataloader</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
&#x201C;If set to true, the dataloader prepared by the Accelerator will be backed by &#x201D;
&#x201D;<a href="https://github.com/pytorch/data/tree/main/torchdata/stateful_dataloader" rel="nofollow">torchdata.StatefulDataLoader</a>.
This requires <code>torchdata</code> version 0.8.0 or higher that supports StatefulDataLoader to be installed.&#x201D;`,name:"use_stateful_dataloader"},{anchor:"accelerate.data_loader.prepare_data_loader.torch_device_mesh",description:`<strong>torch_device_mesh</strong> (<code>torch.distributed.DeviceMesh</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
PyTorch device mesh.`,name:"torch_device_mesh"}],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/data_loader.py#L1016",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A new data loader that will yield the portion of the batches</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>torch.utils.data.dataloader.DataLoader</code></p>
`}}),C=new qt({props:{warning:!0,$$slots:{default:[Rt]},$$scope:{ctx:le}}}),V=new k({props:{name:"accelerate.skip_first_batches",anchor:"accelerate.skip_first_batches",parameters:[{name:"dataloader",val:""},{name:"num_batches",val:" = 0"}],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/data_loader.py#L1395"}}),R=new N({props:{title:"BatchSamplerShard",local:"accelerate.data_loader.BatchSamplerShard",headingTag:"h2"}}),j=new k({props:{name:"class accelerate.data_loader.BatchSamplerShard",anchor:"accelerate.data_loader.BatchSamplerShard",parameters:[{name:"batch_sampler",val:": BatchSampler"},{name:"num_processes",val:": int = 1"},{name:"process_index",val:": int = 0"},{name:"split_batches",val:": bool = False"},{name:"even_batches",val:": bool = True"}],parametersDescription:[{anchor:"accelerate.data_loader.BatchSamplerShard.batch_sampler",description:`<strong>batch_sampler</strong> (<code>torch.utils.data.sampler.BatchSampler</code>) &#x2014;
The batch sampler to split in several shards.`,name:"batch_sampler"},{anchor:"accelerate.data_loader.BatchSamplerShard.num_processes",description:`<strong>num_processes</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of processes running concurrently.`,name:"num_processes"},{anchor:"accelerate.data_loader.BatchSamplerShard.process_index",description:`<strong>process_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) &#x2014;
The index of the current process.`,name:"process_index"},{anchor:"accelerate.data_loader.BatchSamplerShard.split_batches",description:`<strong>split_batches</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the shards should be created by splitting a batch to give a piece of it on each process, or by
yielding different full batches on each process.</p>
<p>On two processes with a sampler of <code>[[0, 1, 2, 3], [4, 5, 6, 7]]</code>, this will result in:</p>
<ul>
<li>the sampler on process 0 to yield <code>[0, 1, 2, 3]</code> and the sampler on process 1 to yield <code>[4, 5, 6, 7]</code> if
this argument is set to <code>False</code>.</li>
<li>the sampler on process 0 to yield <code>[0, 1]</code> then <code>[4, 5]</code> and the sampler on process 1 to yield <code>[2, 3]</code>
then <code>[6, 7]</code> if this argument is set to <code>True</code>.</li>
</ul>`,name:"split_batches"},{anchor:"accelerate.data_loader.BatchSamplerShard.even_batches",description:`<strong>even_batches</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to loop back at the beginning of the sampler when the number of samples is not a round
multiple of (original batch size / number of processes).`,name:"even_batches"}],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/data_loader.py#L110"}}),H=new qt({props:{warning:!0,$$slots:{default:[jt]},$$scope:{ctx:le}}}),X=new N({props:{title:"IterableDatasetShard",local:"accelerate.data_loader.IterableDatasetShard",headingTag:"h2"}}),J=new k({props:{name:"class accelerate.data_loader.IterableDatasetShard",anchor:"accelerate.data_loader.IterableDatasetShard",parameters:[{name:"dataset",val:": IterableDataset"},{name:"batch_size",val:": int = 1"},{name:"drop_last",val:": bool = False"},{name:"num_processes",val:": int = 1"},{name:"process_index",val:": int = 0"},{name:"split_batches",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.data_loader.IterableDatasetShard.dataset",description:`<strong>dataset</strong> (<code>torch.utils.data.dataset.IterableDataset</code>) &#x2014;
The batch sampler to split in several shards.`,name:"dataset"},{anchor:"accelerate.data_loader.IterableDatasetShard.batch_size",description:`<strong>batch_size</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The size of the batches per shard (if <code>split_batches=False</code>) or the size of the batches (if
<code>split_batches=True</code>).`,name:"batch_size"},{anchor:"accelerate.data_loader.IterableDatasetShard.drop_last",description:`<strong>drop_last</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to drop the last incomplete batch or complete the last batches by using the samples from the
beginning.`,name:"drop_last"},{anchor:"accelerate.data_loader.IterableDatasetShard.num_processes",description:`<strong>num_processes</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of processes running concurrently.`,name:"num_processes"},{anchor:"accelerate.data_loader.IterableDatasetShard.process_index",description:`<strong>process_index</strong> (<code>int</code>, <em>optional</em>, defaults to 0) &#x2014;
The index of the current process.`,name:"process_index"},{anchor:"accelerate.data_loader.IterableDatasetShard.split_batches",description:`<strong>split_batches</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the shards should be created by splitting a batch to give a piece of it on each process, or by
yielding different full batches on each process.</p>
<p>On two processes with an iterable dataset yielding of <code>[0, 1, 2, 3, 4, 5, 6, 7]</code>, this will result in:</p>
<ul>
<li>the shard on process 0 to yield <code>[0, 1, 2, 3]</code> and the shard on process 1 to yield <code>[4, 5, 6, 7]</code> if this
argument is set to <code>False</code>.</li>
<li>the shard on process 0 to yield <code>[0, 1, 4, 5]</code> and the sampler on process 1 to yield <code>[2, 3, 6, 7]</code> if
this argument is set to <code>True</code>.</li>
</ul>`,name:"split_batches"}],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/data_loader.py#L274"}}),K=new N({props:{title:"DataLoaderShard",local:"accelerate.data_loader.DataLoaderShard",headingTag:"h2"}}),Q=new k({props:{name:"class accelerate.data_loader.DataLoaderShard",anchor:"accelerate.data_loader.DataLoaderShard",parameters:[{name:"dataset",val:""},{name:"device",val:" = None"},{name:"rng_types",val:" = None"},{name:"synchronized_generator",val:" = None"},{name:"skip_batches",val:" = 0"},{name:"use_stateful_dataloader",val:" = False"},{name:"_drop_last",val:": bool = False"},{name:"_non_blocking",val:": bool = False"},{name:"torch_device_mesh",val:" = None"},{name:"iteration",val:" = 0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"accelerate.data_loader.DataLoaderShard.dataset",description:`<strong>dataset</strong> (<code>torch.utils.data.dataset.Dataset</code>) &#x2014;
The dataset to use to build this dataloader.`,name:"dataset"},{anchor:"accelerate.data_loader.DataLoaderShard.device",description:`<strong>device</strong> (<code>torch.device</code>, <em>optional</em>) &#x2014;
If passed, the device to put all batches on.`,name:"device"},{anchor:"accelerate.data_loader.DataLoaderShard.rng_types",description:`<strong>rng_types</strong> (list of <code>str</code> or <a href="/docs/accelerate/pr_4097/en/package_reference/utilities#accelerate.utils.RNGType">RNGType</a>) &#x2014;
The list of random number generators to synchronize at the beginning of each iteration. Should be one or
several of:</p>
<ul>
<li><code>&quot;torch&quot;</code>: the base torch random number generator</li>
<li><code>&quot;cuda&quot;</code>: the CUDA random number generator (GPU only)</li>
<li><code>&quot;xla&quot;</code>: the XLA random number generator (TPU only)</li>
<li><code>&quot;generator&quot;</code>: an optional <code>torch.Generator</code></li>
</ul>`,name:"rng_types"},{anchor:"accelerate.data_loader.DataLoaderShard.synchronized_generator",description:`<strong>synchronized_generator</strong> (<code>torch.Generator</code>, <em>optional</em>) &#x2014;
A random number generator to keep synchronized across processes.`,name:"synchronized_generator"},{anchor:"accelerate.data_loader.DataLoaderShard.skip_batches",description:`<strong>skip_batches</strong> (<code>int</code>, <em>optional</em>, defaults to 0) &#x2014;
The number of batches to skip at the beginning.`,name:"skip_batches"},{anchor:"accelerate.data_loader.DataLoaderShard.use_stateful_dataloader",description:`<strong>use_stateful_dataloader</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to have this class adapt <code>StatefulDataLoader</code> from <code>torchdata</code> instead of the regular <code>DataLoader</code>.`,name:"use_stateful_dataloader"},{anchor:"accelerate.data_loader.DataLoaderShard.*kwargs",description:`*<strong>*kwargs</strong> (additional keyword arguments, <em>optional</em>) &#x2014;
All other keyword arguments to pass to the regular <code>DataLoader</code> initialization.`,name:"*kwargs"}],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/data_loader.py#L510"}}),Y=new N({props:{title:"DataLoaderDispatcher",local:"accelerate.data_loader.DataLoaderDispatcher",headingTag:"h2"}}),Z=new k({props:{name:"class accelerate.data_loader.DataLoaderDispatcher",anchor:"accelerate.data_loader.DataLoaderDispatcher",parameters:[{name:"dataset",val:""},{name:"split_batches",val:": bool = False"},{name:"skip_batches",val:" = 0"},{name:"use_stateful_dataloader",val:" = False"},{name:"_drop_last",val:": bool = False"},{name:"_non_blocking",val:": bool = False"},{name:"slice_fn",val:" = None"},{name:"torch_device_mesh",val:" = None"},{name:"iteration",val:" = 0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"accelerate.data_loader.DataLoaderDispatcher.split_batches",description:`<strong>split_batches</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the resulting <code>DataLoader</code> should split the batches of the original data loader across devices or
yield full batches (in which case it will yield batches starting at the <code>process_index</code>-th and advancing of
<code>num_processes</code> batches at each iteration). Another way to see this is that the observed batch size will be
the same as the initial <code>dataloader</code> if this option is set to <code>True</code>, the batch size of the initial
<code>dataloader</code> multiplied by <code>num_processes</code> otherwise. Setting this option to <code>True</code> requires that the batch
size of the <code>dataloader</code> is a round multiple of <code>batch_size</code>.`,name:"split_batches"},{anchor:"accelerate.data_loader.DataLoaderDispatcher.skip_batches",description:`<strong>skip_batches</strong> (<code>int</code>, <em>optional</em>, defaults to 0) &#x2014;
The number of batches to skip at the beginning of an iteration.`,name:"skip_batches"},{anchor:"accelerate.data_loader.DataLoaderDispatcher.use_stateful_dataloader",description:`<strong>use_stateful_dataloader</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to have this class adapt <code>StatefulDataLoader</code> from <code>torchdata</code> instead of the regular <code>DataLoader</code>.`,name:"use_stateful_dataloader"}],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/data_loader.py#L723"}}),ee=new N({props:{title:"AcceleratedOptimizer",local:"accelerate.optimizer.AcceleratedOptimizer",headingTag:"h2"}}),te=new k({props:{name:"class accelerate.optimizer.AcceleratedOptimizer",anchor:"accelerate.optimizer.AcceleratedOptimizer",parameters:[{name:"optimizer",val:""},{name:"device_placement",val:" = True"},{name:"scaler",val:" = None"}],parametersDescription:[{anchor:"accelerate.optimizer.AcceleratedOptimizer.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.optimizer.Optimizer</code>) &#x2014;
The optimizer to wrap.`,name:"optimizer"},{anchor:"accelerate.optimizer.AcceleratedOptimizer.device_placement",description:`<strong>device_placement</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the optimizer should handle device placement. If so, it will place the state dictionary of
<code>optimizer</code> on the right device.`,name:"device_placement"},{anchor:"accelerate.optimizer.AcceleratedOptimizer.scaler",description:`<strong>scaler</strong> (<code>torch.amp.GradScaler</code> or <code>torch.cuda.amp.GradScaler</code>, <em>optional</em>) &#x2014;
The scaler to use in the step function if training with mixed precision.`,name:"scaler"}],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/optimizer.py#L38"}}),ae=new k({props:{name:"eval",anchor:"accelerate.optimizer.AcceleratedOptimizer.eval",parameters:[],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/optimizer.py#L138"}}),oe=new k({props:{name:"train",anchor:"accelerate.optimizer.AcceleratedOptimizer.train",parameters:[],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/optimizer.py#L124"}}),re=new N({props:{title:"AcceleratedScheduler",local:"accelerate.scheduler.AcceleratedScheduler",headingTag:"h2"}}),se=new k({props:{name:"class accelerate.scheduler.AcceleratedScheduler",anchor:"accelerate.scheduler.AcceleratedScheduler",parameters:[{name:"scheduler",val:""},{name:"optimizers",val:""},{name:"step_with_optimizer",val:": bool = True"},{name:"split_batches",val:": bool = False"}],parametersDescription:[{anchor:"accelerate.scheduler.AcceleratedScheduler.scheduler",description:`<strong>scheduler</strong> (<code>torch.optim.lr_scheduler._LRScheduler</code>) &#x2014;
The scheduler to wrap.`,name:"scheduler"},{anchor:"accelerate.scheduler.AcceleratedScheduler.optimizers",description:`<strong>optimizers</strong> (one or a list of <code>torch.optim.Optimizer</code>) &#x2014;
The optimizers used.`,name:"optimizers"},{anchor:"accelerate.scheduler.AcceleratedScheduler.step_with_optimizer",description:`<strong>step_with_optimizer</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the scheduler should be stepped at each optimizer step.`,name:"step_with_optimizer"},{anchor:"accelerate.scheduler.AcceleratedScheduler.split_batches",description:`<strong>split_batches</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the dataloaders split one batch across the different processes (so batch size is the same
regardless of the number of processes) or create batches on each process (so batch size is the original
batch size multiplied by the number of processes).`,name:"split_batches"}],source:"https://github.com/huggingface/accelerate/blob/vr_4097/src/accelerate/scheduler.py#L25"}}),ne=new Vt({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/package_reference/torch_wrappers.md"}}),{c(){d=c("meta"),L=o(),f=c("p"),I=o(),i(W.$$.fragment),ke=o(),i(E.$$.fragment),Ae=o(),B=c("p"),B.innerHTML=vt,Me=o(),i(G.$$.fragment),Ie=o(),v=c("div"),i(U.$$.fragment),Ye=o(),de=c("p"),de.innerHTML=$t,Ze=o(),ie=c("p"),ie.innerHTML=yt,et=o(),i(C.$$.fragment),Ce=o(),A=c("div"),i(V.$$.fragment),tt=o(),pe=c("p"),pe.innerHTML=xt,He=o(),i(R.$$.fragment),Pe=o(),T=c("div"),i(j.$$.fragment),at=o(),he=c("p"),he.innerHTML=wt,ot=o(),i(H.$$.fragment),Fe=o(),i(X.$$.fragment),qe=o(),M=c("div"),i(J.$$.fragment),rt=o(),me=c("p"),me.innerHTML=Tt,Oe=o(),i(K.$$.fragment),Ne=o(),$=c("div"),i(Q.$$.fragment),st=o(),ue=c("p"),ue.innerHTML=zt,nt=o(),ge=c("p"),ge.innerHTML=Dt,ct=o(),_e=c("ul"),_e.innerHTML=Lt,We=o(),i(Y.$$.fragment),Ee=o(),y=c("div"),i(Z.$$.fragment),lt=o(),fe=c("p"),fe.innerHTML=St,dt=o(),be=c("p"),be.innerHTML=kt,it=o(),ve=c("ul"),ve.innerHTML=At,Be=o(),i(ee.$$.fragment),Ge=o(),b=c("div"),i(te.$$.fragment),pt=o(),$e=c("p"),$e.textContent=Mt,ht=o(),ye=c("p"),ye.innerHTML=It,mt=o(),P=c("div"),i(ae.$$.fragment),ut=o(),xe=c("p"),xe.innerHTML=Ct,gt=o(),F=c("div"),i(oe.$$.fragment),_t=o(),we=c("p"),we.innerHTML=Ht,Ue=o(),i(re.$$.fragment),Ve=o(),z=c("div"),i(se.$$.fragment),ft=o(),Te=c("p"),Te.textContent=Pt,bt=o(),ze=c("p"),ze.textContent=Ft,Re=o(),i(ne.$$.fragment),je=o(),Se=c("p"),this.h()},l(e){const t=Gt("svelte-u9bgzb",document.head);d=l(t,"META",{name:!0,content:!0}),t.forEach(a),L=r(e),f=l(e,"P",{}),x(f).forEach(a),I=r(e),p(W.$$.fragment,e),ke=r(e),p(E.$$.fragment,e),Ae=r(e),B=l(e,"P",{"data-svelte-h":!0}),_(B)!=="svelte-kplnj1"&&(B.innerHTML=vt),Me=r(e),p(G.$$.fragment,e),Ie=r(e),v=l(e,"DIV",{class:!0});var D=x(v);p(U.$$.fragment,D),Ye=r(D),de=l(D,"P",{"data-svelte-h":!0}),_(de)!=="svelte-18lxf6z"&&(de.innerHTML=$t),Ze=r(D),ie=l(D,"P",{"data-svelte-h":!0}),_(ie)!=="svelte-15utu04"&&(ie.innerHTML=yt),et=r(D),p(C.$$.fragment,D),D.forEach(a),Ce=r(e),A=l(e,"DIV",{class:!0});var ce=x(A);p(V.$$.fragment,ce),tt=r(ce),pe=l(ce,"P",{"data-svelte-h":!0}),_(pe)!=="svelte-1j74yzt"&&(pe.innerHTML=xt),ce.forEach(a),He=r(e),p(R.$$.fragment,e),Pe=r(e),T=l(e,"DIV",{class:!0});var De=x(T);p(j.$$.fragment,De),at=r(De),he=l(De,"P",{"data-svelte-h":!0}),_(he)!=="svelte-1d4wgd"&&(he.innerHTML=wt),ot=r(De),p(H.$$.fragment,De),De.forEach(a),Fe=r(e),p(X.$$.fragment,e),qe=r(e),M=l(e,"DIV",{class:!0});var Je=x(M);p(J.$$.fragment,Je),rt=r(Je),me=l(Je,"P",{"data-svelte-h":!0}),_(me)!=="svelte-1t48as2"&&(me.innerHTML=Tt),Je.forEach(a),Oe=r(e),p(K.$$.fragment,e),Ne=r(e),$=l(e,"DIV",{class:!0});var q=x($);p(Q.$$.fragment,q),st=r(q),ue=l(q,"P",{"data-svelte-h":!0}),_(ue)!=="svelte-1gl2w6s"&&(ue.innerHTML=zt),nt=r(q),ge=l(q,"P",{"data-svelte-h":!0}),_(ge)!=="svelte-17v1xbv"&&(ge.innerHTML=Dt),ct=r(q),_e=l(q,"UL",{"data-svelte-h":!0}),_(_e)!=="svelte-ixpwl0"&&(_e.innerHTML=Lt),q.forEach(a),We=r(e),p(Y.$$.fragment,e),Ee=r(e),y=l(e,"DIV",{class:!0});var O=x(y);p(Z.$$.fragment,O),lt=r(O),fe=l(O,"P",{"data-svelte-h":!0}),_(fe)!=="svelte-1mjzhu7"&&(fe.innerHTML=St),dt=r(O),be=l(O,"P",{"data-svelte-h":!0}),_(be)!=="svelte-17v1xbv"&&(be.innerHTML=kt),it=r(O),ve=l(O,"UL",{"data-svelte-h":!0}),_(ve)!=="svelte-ixpwl0"&&(ve.innerHTML=At),O.forEach(a),Be=r(e),p(ee.$$.fragment,e),Ge=r(e),b=l(e,"DIV",{class:!0});var S=x(b);p(te.$$.fragment,S),pt=r(S),$e=l(S,"P",{"data-svelte-h":!0}),_($e)!=="svelte-1kknwnf"&&($e.textContent=Mt),ht=r(S),ye=l(S,"P",{"data-svelte-h":!0}),_(ye)!=="svelte-vsq67w"&&(ye.innerHTML=It),mt=r(S),P=l(S,"DIV",{class:!0});var Ke=x(P);p(ae.$$.fragment,Ke),ut=r(Ke),xe=l(Ke,"P",{"data-svelte-h":!0}),_(xe)!=="svelte-2k0jq3"&&(xe.innerHTML=Ct),Ke.forEach(a),gt=r(S),F=l(S,"DIV",{class:!0});var Qe=x(F);p(oe.$$.fragment,Qe),_t=r(Qe),we=l(Qe,"P",{"data-svelte-h":!0}),_(we)!=="svelte-2n5ee9"&&(we.innerHTML=Ht),Qe.forEach(a),S.forEach(a),Ue=r(e),p(re.$$.fragment,e),Ve=r(e),z=l(e,"DIV",{class:!0});var Le=x(z);p(se.$$.fragment,Le),ft=r(Le),Te=l(Le,"P",{"data-svelte-h":!0}),_(Te)!=="svelte-ebdman"&&(Te.textContent=Pt),bt=r(Le),ze=l(Le,"P",{"data-svelte-h":!0}),_(ze)!=="svelte-1g111ye"&&(ze.textContent=Ft),Le.forEach(a),Re=r(e),p(ne.$$.fragment,e),je=r(e),Se=l(e,"P",{}),x(Se).forEach(a),this.h()},h(){w(d,"name","hf:doc:metadata"),w(d,"content",Jt),w(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(P,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),w(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){s(document.head,d),n(e,L,t),n(e,f,t),n(e,I,t),h(W,e,t),n(e,ke,t),h(E,e,t),n(e,Ae,t),n(e,B,t),n(e,Me,t),h(G,e,t),n(e,Ie,t),n(e,v,t),h(U,v,null),s(v,Ye),s(v,de),s(v,Ze),s(v,ie),s(v,et),h(C,v,null),n(e,Ce,t),n(e,A,t),h(V,A,null),s(A,tt),s(A,pe),n(e,He,t),h(R,e,t),n(e,Pe,t),n(e,T,t),h(j,T,null),s(T,at),s(T,he),s(T,ot),h(H,T,null),n(e,Fe,t),h(X,e,t),n(e,qe,t),n(e,M,t),h(J,M,null),s(M,rt),s(M,me),n(e,Oe,t),h(K,e,t),n(e,Ne,t),n(e,$,t),h(Q,$,null),s($,st),s($,ue),s($,nt),s($,ge),s($,ct),s($,_e),n(e,We,t),h(Y,e,t),n(e,Ee,t),n(e,y,t),h(Z,y,null),s(y,lt),s(y,fe),s(y,dt),s(y,be),s(y,it),s(y,ve),n(e,Be,t),h(ee,e,t),n(e,Ge,t),n(e,b,t),h(te,b,null),s(b,pt),s(b,$e),s(b,ht),s(b,ye),s(b,mt),s(b,P),h(ae,P,null),s(P,ut),s(P,xe),s(b,gt),s(b,F),h(oe,F,null),s(F,_t),s(F,we),n(e,Ue,t),h(re,e,t),n(e,Ve,t),n(e,z,t),h(se,z,null),s(z,ft),s(z,Te),s(z,bt),s(z,ze),n(e,Re,t),h(ne,e,t),n(e,je,t),n(e,Se,t),Xe=!0},p(e,[t]){const D={};t&2&&(D.$$scope={dirty:t,ctx:e}),C.$set(D);const ce={};t&2&&(ce.$$scope={dirty:t,ctx:e}),H.$set(ce)},i(e){Xe||(m(W.$$.fragment,e),m(E.$$.fragment,e),m(G.$$.fragment,e),m(U.$$.fragment,e),m(C.$$.fragment,e),m(V.$$.fragment,e),m(R.$$.fragment,e),m(j.$$.fragment,e),m(H.$$.fragment,e),m(X.$$.fragment,e),m(J.$$.fragment,e),m(K.$$.fragment,e),m(Q.$$.fragment,e),m(Y.$$.fragment,e),m(Z.$$.fragment,e),m(ee.$$.fragment,e),m(te.$$.fragment,e),m(ae.$$.fragment,e),m(oe.$$.fragment,e),m(re.$$.fragment,e),m(se.$$.fragment,e),m(ne.$$.fragment,e),Xe=!0)},o(e){u(W.$$.fragment,e),u(E.$$.fragment,e),u(G.$$.fragment,e),u(U.$$.fragment,e),u(C.$$.fragment,e),u(V.$$.fragment,e),u(R.$$.fragment,e),u(j.$$.fragment,e),u(H.$$.fragment,e),u(X.$$.fragment,e),u(J.$$.fragment,e),u(K.$$.fragment,e),u(Q.$$.fragment,e),u(Y.$$.fragment,e),u(Z.$$.fragment,e),u(ee.$$.fragment,e),u(te.$$.fragment,e),u(ae.$$.fragment,e),u(oe.$$.fragment,e),u(re.$$.fragment,e),u(se.$$.fragment,e),u(ne.$$.fragment,e),Xe=!1},d(e){e&&(a(L),a(f),a(I),a(ke),a(Ae),a(B),a(Me),a(Ie),a(v),a(Ce),a(A),a(He),a(Pe),a(T),a(Fe),a(qe),a(M),a(Oe),a(Ne),a($),a(We),a(Ee),a(y),a(Be),a(Ge),a(b),a(Ue),a(Ve),a(z),a(Re),a(je),a(Se)),a(d),g(W,e),g(E,e),g(G,e),g(U),g(C),g(V),g(R,e),g(j),g(H),g(X,e),g(J),g(K,e),g(Q),g(Y,e),g(Z),g(ee,e),g(te),g(ae),g(oe),g(re,e),g(se),g(ne,e)}}}const Jt='{"title":"DataLoaders, Optimizers, and Schedulers","local":"dataloaders-optimizers-and-schedulers","sections":[{"title":"DataLoader utilities","local":"accelerate.data_loader.prepare_data_loader","sections":[],"depth":2},{"title":"BatchSamplerShard","local":"accelerate.data_loader.BatchSamplerShard","sections":[],"depth":2},{"title":"IterableDatasetShard","local":"accelerate.data_loader.IterableDatasetShard","sections":[],"depth":2},{"title":"DataLoaderShard","local":"accelerate.data_loader.DataLoaderShard","sections":[],"depth":2},{"title":"DataLoaderDispatcher","local":"accelerate.data_loader.DataLoaderDispatcher","sections":[],"depth":2},{"title":"AcceleratedOptimizer","local":"accelerate.optimizer.AcceleratedOptimizer","sections":[],"depth":2},{"title":"AcceleratedScheduler","local":"accelerate.scheduler.AcceleratedScheduler","sections":[],"depth":2}],"depth":1}';function Kt(le){return Wt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class aa extends Et{constructor(d){super(),Bt(this,d,Kt,Xt,Nt,{})}}export{aa as component};