accelerate/pr_4037/en/_app/immutable/nodes/50.d805fa2d.js

import{s as Ce,n as Ge,o as be}from"../chunks/scheduler.b9285784.js";import{S as Ae,i as ve,e as i,s,c as r,h as $e,a as o,d as l,b as n,f as Ue,g as p,j as c,k as ge,l as Be,m as a,n as M,t as w,o as d,p as J}from"../chunks/index.26bc89a1.js";import{C as Ze,H as x,E as Ve}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.7ea56988.js";import{C as pe}from"../chunks/CodeBlock.91c8cb69.js";function ze(Me){let T,S,X,W,m,N,y,Y,h,we=`Local SGD is a technique for distributed training where gradients are not synchronized every step. Thus, each process updates its own version of the model weights and after a given number of steps these weights are synchronized by averaging across all processes. This improves communication efficiency and can lead to substantial training speed up especially when a computer lacks a faster interconnect such as NVLink.
Unlike gradient accumulation (where improving communication efficiency requires increasing the effective batch size), Local SGD does not require changing a batch size or a learning rate / schedule. However, if necessary, Local SGD can be combined with gradient accumulation as well.`,F,u,de="In this tutorial you will see how to quickly setup  Local SGD Accelerate. Compared to a standard Accelerate setup, this requires only two extra lines of code.",E,j,Je="This example will use a very simplistic PyTorch training loop that performs gradient accumulation every two batches:",L,I,H,f,Q,U,Te="First the code shown earlier will be converted to use Accelerate  with neither a LocalSGD or a gradient accumulation helper:",D,g,K,C,P,G,me=`All that is left now is to let Accelerate handle model parameter synchronization <strong>and</strong> the gradient accumulation for us. For simplicity let us assume we need to synchronize every 8 steps. This is
achieved by adding one <code>with LocalSGD</code> statement and one call <code>local_sgd.step()</code> after every optimizer step:`,q,b,O,A,ye="Under the hood, the Local SGD code <strong>disables</strong> automatic gradient synchronization (but accumulation still works as expected!). Instead it averages model parameters every <code>local_sgd_steps</code> steps (as well as at the end of the training loop).",ee,v,te,$,he='The current implementation works only with basic multi-GPU (or multi-CPU) training without, e.g., <a href="https://github.com/deepspeedai/DeepSpeed" rel="nofollow">DeepSpeed.</a>.',le,B,ae,Z,ue=`Although we are not aware of the true origins of this simple approach, the idea of local SGD is quite old and goes
back to at least:`,se,V,je=`Zhang, J., De Sa, C., Mitliagkas, I., &amp; Ré, C. (2016). <a href="https://huggingface.co/papers/1606.07365" rel="nofollow">Parallel SGD: When does averaging help?. arXiv preprint
arXiv:1606.07365.</a>`,ne,z,Ie="We credit the term Local SGD to the following paper (but there might be earlier references we are not aware of).",ie,R,fe=`Stich, Sebastian Urban. <a href="https://huggingface.co/papers/1805.09767" rel="nofollow">“Local SGD Converges Fast and Communicates Little.” ICLR 2019-International Conference on
Learning Representations. No. CONF. 2019.</a>`,oe,k,ce,_,re;return m=new Ze({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),y=new x({props:{title:"Using Local SGD with Accelerate",local:"using-local-sgd-with-accelerate",headingTag:"h1"}}),I=new pe({props:{code:"ZGV2aWNlJTIwJTNEJTIwJTIyY3VkYSUyMiUwQW1vZGVsLnRvKGRldmljZSklMEElMEFncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlMjAlM0QlMjAyJTBBJTBBZm9yJTIwaW5kZXglMkMlMjBiYXRjaCUyMGluJTIwZW51bWVyYXRlKHRyYWluaW5nX2RhdGFsb2FkZXIpJTNBJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTJDJTIwdGFyZ2V0cyUyMCUzRCUyMGJhdGNoJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTIwJTNEJTIwaW5wdXRzLnRvKGRldmljZSklMEElMjAlMjAlMjAlMjB0YXJnZXRzJTIwJTNEJTIwdGFyZ2V0cy50byhkZXZpY2UpJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0cyklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mdW5jdGlvbihvdXRwdXRzJTJDJTIwdGFyZ2V0cyklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zcyUyMCUyRiUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUwQSUyMCUyMCUyMCUyMGxvc3MuYmFja3dhcmQoKSUwQSUyMCUyMCUyMCUyMGlmJTIwKGluZGV4JTIwJTJCJTIwMSklMjAlMjUlMjBncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlMjAlM0QlM0QlMjAwJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCk=",highlighted:`device = <span class="hljs-string">&quot;cuda&quot;</span>
model.to(device)

gradient_accumulation_steps = <span class="hljs-number">2</span>

<span class="hljs-keyword">for</span> index, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(training_dataloader):
    inputs, targets = batch
    inputs = inputs.to(device)
    targets = targets.to(device)
    outputs = model(inputs)
    loss = loss_function(outputs, targets)
    loss = loss / gradient_accumulation_steps
    loss.backward()
    <span class="hljs-keyword">if</span> (index + <span class="hljs-number">1</span>) % gradient_accumulation_steps == <span class="hljs-number">0</span>:
        optimizer.step()
        scheduler.step()
        optimizer.zero_grad()`,lang:"python",wrap:!1}}),f=new x({props:{title:"Converting it to Accelerate",local:"converting-it-to-accelerate",headingTag:"h2"}}),g=new pe({props:{code:"JTJCJTIwZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUyQiUyMGFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoKSUwQSUwQSUyQiUyMG1vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwdHJhaW5pbmdfZGF0YWxvYWRlciUyQyUyMHNjaGVkdWxlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUoJTBBJTJCJTIwJTIwJTIwJTIwJTIwbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjB0cmFpbmluZ19kYXRhbG9hZGVyJTJDJTIwc2NoZWR1bGVyJTBBJTJCJTIwKSUwQSUwQSUyMCUyMGZvciUyMGluZGV4JTJDJTIwYmF0Y2glMjBpbiUyMGVudW1lcmF0ZSh0cmFpbmluZ19kYXRhbG9hZGVyKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyUyQyUyMHRhcmdldHMlMjAlM0QlMjBiYXRjaCUwQS0lMjAlMjAlMjAlMjAlMjBpbnB1dHMlMjAlM0QlMjBpbnB1dHMudG8oZGV2aWNlKSUwQS0lMjAlMjAlMjAlMjAlMjB0YXJnZXRzJTIwJTNEJTIwdGFyZ2V0cy50byhkZXZpY2UpJTBBJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mdW5jdGlvbihvdXRwdXRzJTJDJTIwdGFyZ2V0cyklMEElMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zcyUyMCUyRiUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUwQSUyQiUyMCUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjAoaW5kZXglMkIxKSUyMCUyNSUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUyMCUzRCUzRCUyMDAlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2NoZWR1bGVyLnN0ZXAoKQ==",highlighted:`<span class="hljs-addition">+ from accelerate import Accelerator</span>
<span class="hljs-addition">+ accelerator = Accelerator()</span>

<span class="hljs-addition">+ model, optimizer, training_dataloader, scheduler = accelerator.prepare(</span>
<span class="hljs-addition">+     model, optimizer, training_dataloader, scheduler</span>
<span class="hljs-addition">+ )</span>

  for index, batch in enumerate(training_dataloader):
      inputs, targets = batch
<span class="hljs-deletion">-     inputs = inputs.to(device)</span>
<span class="hljs-deletion">-     targets = targets.to(device)</span>
      outputs = model(inputs)
      loss = loss_function(outputs, targets)
      loss = loss / gradient_accumulation_steps
<span class="hljs-addition">+     accelerator.backward(loss)</span>
      if (index+1) % gradient_accumulation_steps == 0:
          optimizer.step()
          scheduler.step()`,lang:"diff",wrap:!1}}),C=new x({props:{title:"Letting Accelerate handle model synchronization",local:"letting-accelerate-handle-model-synchronization",headingTag:"h2"}}),b=new pe({props:{code:"JTJCbG9jYWxfc2dkX3N0ZXBzJTNEOCUwQSUwQSUyQndpdGglMjBMb2NhbFNHRChhY2NlbGVyYXRvciUzRGFjY2VsZXJhdG9yJTJDJTIwbW9kZWwlM0Rtb2RlbCUyQyUyMGxvY2FsX3NnZF9zdGVwcyUzRDglMkMlMjBlbmFibGVkJTNEVHJ1ZSklMjBhcyUyMGxvY2FsX3NnZCUzQSUwQSUyMCUyMCUyMCUyMGZvciUyMGJhdGNoJTIwaW4lMjB0cmFpbmluZ19kYXRhbG9hZGVyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2l0aCUyMGFjY2VsZXJhdG9yLmFjY3VtdWxhdGUobW9kZWwpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5wdXRzJTJDJTIwdGFyZ2V0cyUyMCUzRCUyMGJhdGNoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mdW5jdGlvbihvdXRwdXRzJTJDJTIwdGFyZ2V0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhY2NlbGVyYXRvci5iYWNrd2FyZChsb3NzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci5zdGVwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzY2hlZHVsZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyLnplcm9fZ3JhZCgpJTBBJTJCJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG9jYWxfc2dkLnN0ZXAoKQ==",highlighted:`<span class="hljs-addition">+local_sgd_steps=8</span>

<span class="hljs-addition">+with LocalSGD(accelerator=accelerator, model=model, local_sgd_steps=8, enabled=True) as local_sgd:</span>
    for batch in training_dataloader:
        with accelerator.accumulate(model):
            inputs, targets = batch
            outputs = model(inputs)
            loss = loss_function(outputs, targets)
            accelerator.backward(loss)
            optimizer.step()
            scheduler.step()
            optimizer.zero_grad()
<span class="hljs-addition">+           local_sgd.step()</span>`,lang:"diff",wrap:!1}}),v=new x({props:{title:"Limitations",local:"limitations",headingTag:"h2"}}),B=new x({props:{title:"References",local:"references",headingTag:"h2"}}),k=new Ve({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/local_sgd.md"}}),{c(){T=i("meta"),S=s(),X=i("p"),W=s(),r(m.$$.fragment),N=s(),r(y.$$.fragment),Y=s(),h=i("p"),h.textContent=we,F=s(),u=i("p"),u.textContent=de,E=s(),j=i("p"),j.textContent=Je,L=s(),r(I.$$.fragment),H=s(),r(f.$$.fragment),Q=s(),U=i("p"),U.textContent=Te,D=s(),r(g.$$.fragment),K=s(),r(C.$$.fragment),P=s(),G=i("p"),G.innerHTML=me,q=s(),r(b.$$.fragment),O=s(),A=i("p"),A.innerHTML=ye,ee=s(),r(v.$$.fragment),te=s(),$=i("p"),$.innerHTML=he,le=s(),r(B.$$.fragment),ae=s(),Z=i("p"),Z.textContent=ue,se=s(),V=i("p"),V.innerHTML=je,ne=s(),z=i("p"),z.textContent=Ie,ie=s(),R=i("p"),R.innerHTML=fe,oe=s(),r(k.$$.fragment),ce=s(),_=i("p"),this.h()},l(e){const t=$e("svelte-u9bgzb",document.head);T=o(t,"META",{name:!0,content:!0}),t.forEach(l),S=n(e),X=o(e,"P",{}),Ue(X).forEach(l),W=n(e),p(m.$$.fragment,e),N=n(e),p(y.$$.fragment,e),Y=n(e),h=o(e,"P",{"data-svelte-h":!0}),c(h)!=="svelte-1gzsxbq"&&(h.textContent=we),F=n(e),u=o(e,"P",{"data-svelte-h":!0}),c(u)!=="svelte-1d63qys"&&(u.textContent=de),E=n(e),j=o(e,"P",{"data-svelte-h":!0}),c(j)!=="svelte-n9bxrd"&&(j.textContent=Je),L=n(e),p(I.$$.fragment,e),H=n(e),p(f.$$.fragment,e),Q=n(e),U=o(e,"P",{"data-svelte-h":!0}),c(U)!=="svelte-19w42d7"&&(U.textContent=Te),D=n(e),p(g.$$.fragment,e),K=n(e),p(C.$$.fragment,e),P=n(e),G=o(e,"P",{"data-svelte-h":!0}),c(G)!=="svelte-192byqc"&&(G.innerHTML=me),q=n(e),p(b.$$.fragment,e),O=n(e),A=o(e,"P",{"data-svelte-h":!0}),c(A)!=="svelte-avyf7q"&&(A.innerHTML=ye),ee=n(e),p(v.$$.fragment,e),te=n(e),$=o(e,"P",{"data-svelte-h":!0}),c($)!=="svelte-10m76zr"&&($.innerHTML=he),le=n(e),p(B.$$.fragment,e),ae=n(e),Z=o(e,"P",{"data-svelte-h":!0}),c(Z)!=="svelte-1uio6eh"&&(Z.textContent=ue),se=n(e),V=o(e,"P",{"data-svelte-h":!0}),c(V)!=="svelte-xebpa"&&(V.innerHTML=je),ne=n(e),z=o(e,"P",{"data-svelte-h":!0}),c(z)!=="svelte-5vkth6"&&(z.textContent=Ie),ie=n(e),R=o(e,"P",{"data-svelte-h":!0}),c(R)!=="svelte-1xdvzf3"&&(R.innerHTML=fe),oe=n(e),p(k.$$.fragment,e),ce=n(e),_=o(e,"P",{}),Ue(_).forEach(l),this.h()},h(){ge(T,"name","hf:doc:metadata"),ge(T,"content",Re)},m(e,t){Be(document.head,T),a(e,S,t),a(e,X,t),a(e,W,t),M(m,e,t),a(e,N,t),M(y,e,t),a(e,Y,t),a(e,h,t),a(e,F,t),a(e,u,t),a(e,E,t),a(e,j,t),a(e,L,t),M(I,e,t),a(e,H,t),M(f,e,t),a(e,Q,t),a(e,U,t),a(e,D,t),M(g,e,t),a(e,K,t),M(C,e,t),a(e,P,t),a(e,G,t),a(e,q,t),M(b,e,t),a(e,O,t),a(e,A,t),a(e,ee,t),M(v,e,t),a(e,te,t),a(e,$,t),a(e,le,t),M(B,e,t),a(e,ae,t),a(e,Z,t),a(e,se,t),a(e,V,t),a(e,ne,t),a(e,z,t),a(e,ie,t),a(e,R,t),a(e,oe,t),M(k,e,t),a(e,ce,t),a(e,_,t),re=!0},p:Ge,i(e){re||(w(m.$$.fragment,e),w(y.$$.fragment,e),w(I.$$.fragment,e),w(f.$$.fragment,e),w(g.$$.fragment,e),w(C.$$.fragment,e),w(b.$$.fragment,e),w(v.$$.fragment,e),w(B.$$.fragment,e),w(k.$$.fragment,e),re=!0)},o(e){d(m.$$.fragment,e),d(y.$$.fragment,e),d(I.$$.fragment,e),d(f.$$.fragment,e),d(g.$$.fragment,e),d(C.$$.fragment,e),d(b.$$.fragment,e),d(v.$$.fragment,e),d(B.$$.fragment,e),d(k.$$.fragment,e),re=!1},d(e){e&&(l(S),l(X),l(W),l(N),l(Y),l(h),l(F),l(u),l(E),l(j),l(L),l(H),l(Q),l(U),l(D),l(K),l(P),l(G),l(q),l(O),l(A),l(ee),l(te),l($),l(le),l(ae),l(Z),l(se),l(V),l(ne),l(z),l(ie),l(R),l(oe),l(ce),l(_)),l(T),J(m,e),J(y,e),J(I,e),J(f,e),J(g,e),J(C,e),J(b,e),J(v,e),J(B,e),J(k,e)}}}const Re='{"title":"Using Local SGD with Accelerate","local":"using-local-sgd-with-accelerate","sections":[{"title":"Converting it to Accelerate","local":"converting-it-to-accelerate","sections":[],"depth":2},{"title":"Letting Accelerate handle model synchronization","local":"letting-accelerate-handle-model-synchronization","sections":[],"depth":2},{"title":"Limitations","local":"limitations","sections":[],"depth":2},{"title":"References","local":"references","sections":[],"depth":2}],"depth":1}';function ke(Me){return be(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class We extends Ae{constructor(T){super(),ve(this,T,ke,ze,Ce,{})}}export{We as component};