accelerate/pr_4119/en/_app/immutable/nodes/16.C2LbdvRT.js

import"../chunks/DsnmJJEf.js";import{i as f,h as g,C as _,H as y,E as b,s as w}from"../chunks/ClLAY5C0.js";import{p as v,o as D,s as a,f as T,a as t,b as k,c as o,n as L}from"../chunks/BhevU81Y.js";import{T as h}from"../chunks/L24edx8-.js";const S='{"title":"Accelerate’s internal mechanisms","local":"accelerates-internal-mechanisms","sections":[],"depth":1}';var A=o('<meta name="hf:doc:metadata"/>'),z=o(`<p>Synchronization of the main torch (or CUDA or XLA) random number generator will affect any other potential random
artifacts you could have in your dataset (like random data augmentation) in the sense that all processes will get
the same random numbers from the torch random modules (so will apply the same random data augmentation if it’s
controlled by torch).</p>`),P=o("<p>The randomization part of your custom sampler, batch sampler or iterable dataset should be done using a local <code>torch.Generator</code> object (in PyTorch >= 1.6), see the traditional <code>RandomSampler</code>, as an example.</p>"),x=o(`<p></p> <!> <!> <p>Internally, Accelerate works by first analyzing the environment in which the script is launched to determine which
kind of distributed setup is used, how many different processes there are and which one the current script is in. All
that information is stored in the <code>~AcceleratorState</code>.</p> <p>This class is initialized the first time you instantiate an <a href="/docs/accelerate/pr_4119/en/package_reference/accelerator#accelerate.Accelerator">~Accelerator</a> as well as performing any
specific initialization your distributed setup needs. Its state is then uniquely shared through all instances of <a href="/docs/accelerate/pr_4119/en/package_reference/state#accelerate.state.AcceleratorState">AcceleratorState</a>. (The same can also be done with the <a href="/docs/accelerate/pr_4119/en/package_reference/state#accelerate.PartialState">PartialState</a>, a more barebones version it inherits)</p> <p>Then, when calling <a href="/docs/accelerate/pr_4119/en/package_reference/accelerator#accelerate.Accelerator.prepare">prepare()</a>, the library:</p> <ul><li>wraps your model(s) in the container adapted for the distributed setup,</li> <li>wraps your optimizer(s) in an <a href="/docs/accelerate/pr_4119/en/package_reference/torch_wrappers#accelerate.optimizer.AcceleratedOptimizer">AcceleratedOptimizer</a>,</li> <li>wraps your scheduler(s) in an <a href="/docs/accelerate/pr_4119/en/package_reference/torch_wrappers#accelerate.scheduler.AcceleratedScheduler">AcceleratedScheduler</a></li> <li>creates a new version of your dataloader(s) in a <a href="/docs/accelerate/pr_4119/en/package_reference/torch_wrappers#accelerate.data_loader.DataLoaderShard">DataLoaderShard</a> or <a href="/docs/accelerate/pr_4119/en/package_reference/torch_wrappers#accelerate.data_loader.DataLoaderDispatcher">DataLoaderDispatcher</a></li></ul> <p>While the model(s), optimizer(s), and scheduler(s) are just put in simple wrappers, the dataloader(s) are re-created. This is mostly
because PyTorch does not let the user change the <code>batch_sampler</code> of a dataloader once it’s been created and the
library handles the sharding of your data between processes by changing that <code>batch_sampler</code> to yield every other <code>num_processes</code> batches (if enabled).</p> <p>The <a href="/docs/accelerate/pr_4119/en/package_reference/torch_wrappers#accelerate.data_loader.DataLoaderShard">DataLoaderShard</a> subclasses <code>DataLoader</code> to add the following functionality:</p> <ul><li>it synchronizes the appropriate random number generator of all processes at each new iteration, to ensure any
randomization (like shuffling) is done the exact same way across processes.</li> <li>it puts the batches on the proper device before yielding them (unless you have opted out of <code>device_placement=True</code>).</li></ul> <p>The <a href="/docs/accelerate/pr_4119/en/package_reference/torch_wrappers#accelerate.data_loader.DataLoaderDispatcher">DataLoaderDispatcher</a> subclasses differs from the <a href="/docs/accelerate/pr_4119/en/package_reference/torch_wrappers#accelerate.data_loader.DataLoaderShard">DataLoaderShard</a> in that when iterating through the <code>DataLoader</code>, the data is all starting from process 0 and <em>then</em> split and sent off to each process rather than it happening at the dataset level.</p> <p>The random number generator synchronization will by default synchronize:</p> <ul><li>the <code>generator</code> attribute of a given sampler (like the PyTorch <code>RandomSampler</code>) for PyTorch >= 1.6</li> <li>the main random number generator in PyTorch &#60;=1.5.1</li></ul> <p>You can choose which random number generator(s) to synchronize with the <code>rng_types</code> argument of the main <a href="/docs/accelerate/pr_4119/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a>. In PyTorch >= 1.6, it is recommended to rely on a local <code>generator</code> to avoid
setting the same seed in the main random number generator in all processes.</p> <!> <!> <p>If you have <a href="https://github.com/pytorch/data/tree/main" rel="nofollow"><code>torchdata&gt;=0.8.0</code></a> installed, and you have passed <code>use_stateful_dataloader=True</code> into your <a href="/docs/accelerate/pr_4119/en/package_reference/utilities#accelerate.DataLoaderConfiguration">DataLoaderConfiguration</a>, these classes will directly inherit from <code>StatefulDataLoader</code> instead, and maintain a <code>state_dict</code>.</p> <p>For more details about the internals, see the <a href="../package_reference/torch_wrappers">Internals page</a>.</p> <!> <p></p>`,1);function M(p,m){v(m,!1),D(()=>{new URLSearchParams(window.location.search).get("fw")}),f();var n=x();g("jz1xet",e=>{var r=A();w(r,"content",S),t(e,r)});var s=a(T(n),2);_(s,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var i=a(s,2);y(i,{title:"Accelerate’s internal mechanisms",local:"accelerates-internal-mechanisms",headingTag:"h1"});var l=a(i,24);h(l,{warning:!0,children:(e,r)=>{var c=z();t(e,c)},$$slots:{default:!0}});var d=a(l,2);h(d,{children:(e,r)=>{var c=P();t(e,c)},$$slots:{default:!0}});var u=a(d,6);b(u,{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/concept_guides/internal_mechanism.md"}),L(2),t(p,n),k()}export{M as component};