accelerate/pr_4049/en/_app/immutable/nodes/12.ff181576.js

import{s as Re,o as Qe,n as Ne}from"../chunks/scheduler.b9285784.js";import{S as Se,i as Ve,e as y,s as a,c as r,h as Ee,a as f,d as s,b as n,f as Ye,g as p,j as M,k as Fe,l as qe,m as l,n as c,t as d,o as m,p as h}from"../chunks/index.26bc89a1.js";import{T as He}from"../chunks/Tip.e4eba3d6.js";import{C as Le,H as L,E as Pe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.c0f56888.js";import{C as E}from"../chunks/CodeBlock.2b639c7d.js";function Ke(q){let i,u=`Some of these are utilized with the <a href="/docs/accelerate/pr_4049/en/package_reference/accelerator#accelerate.Accelerator.main_process_first">main_process_first()</a> context manager, which utilizes <a href="/docs/accelerate/pr_4049/en/package_reference/accelerator#accelerate.Accelerator.wait_for_everyone">wait_for_everyone()</a> to
run a particular set of code on the main process beforehand before triggering and launching the other processes`;return{c(){i=y("p"),i.innerHTML=u},l(o){i=f(o,"P",{"data-svelte-h":!0}),M(i)!=="svelte-1kqh5mg"&&(i.innerHTML=u)},m(o,w){l(o,i,w)},p:Ne,d(o){o&&s(i)}}}function De(q){let i,u=`<code>load_dataset</code> will perform a lock under the hood to stop multiple downloads from happening at once, but if you are downloading something
not using this library you should use this method.`;return{c(){i=y("p"),i.innerHTML=u},l(o){i=f(o,"P",{"data-svelte-h":!0}),M(i)!=="svelte-909il1"&&(i.innerHTML=u)},m(o,w){l(o,i,w)},p:Ne,d(o){o&&s(i)}}}function Oe(q){let i,u,o,w,j,K,U,D,T,be=`When you run your usual script, instructions are executed in order. Using Accelerate to deploy your script on several
GPUs at the same time introduces a complication: while each process executes all instructions in order, some may be
faster than others.`,O,J,Ce=`You might need to wait for all processes to have reached a certain point before executing a given instruction. For
instance, you shouldn’t save a model before being sure every process is done with training, and you wouldn’t want to
continue training before all the model weights have been loaded in. To do this, just write the following line in your code:`,ee,_,te,b,Ie=`This instruction will block all the processes that arrive first until all the other processes have reached that
point (if you run your script on just one GPU or CPU, this won’t do anything).`,se,C,ke="A few example cases of when to use this utility are listed below:",le,g,ae,I,ne,k,ve="When downloading a dataset, you should download it first on the main process and then load the cached dataset afterward",ie,$,oe,v,re,G,Ge="Under the hood this is the same as calling:",pe,Z,ce,B,de,x,Ze=`When saving the <code>state_dict</code> of the model, since you would normally save one file on just the main process
you should specify that:`,me,X,he,A,ye,W,Be=`When loading in the <code>state_dict</code> to a model, optimizer, or scheduler, you should wait
for all workers to have the weights loaded in before moving on to training`,fe,z,Me,Y,ue,F,xe=`Applying a <code>map()</code> operation on multiple workers, such as tokenizing should be done on the
main process first, and then propagated to each one.`,we,H,ge,N,$e,R,Xe=`To have a check that works with a flag set by a particular process, the <code>set_trigger</code> and <code>check_trigger</code> API should be used. Useful examples
for doing so can include situations such as using early stopping and monitoring the loss (as each loss slightly differs on each process).`,je,Q,Ae='Call <a href="/docs/accelerate/pr_4049/en/package_reference/accelerator#accelerate.Accelerator.set_trigger">Accelerator.set_trigger()</a> when your condition has been met, and <a href="/docs/accelerate/pr_4049/en/package_reference/accelerator#accelerate.Accelerator.check_trigger">Accelerator.check_trigger()</a> when checking if that condition has been met in any process:',Ue,S,Te,V,Je,P,_e;return j=new Le({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new L({props:{title:"Executing and deferring jobs",local:"executing-and-deferring-jobs",headingTag:"h1"}}),_=new E({props:{code:"YWNjZWxlcmF0b3Iud2FpdF9mb3JfZXZlcnlvbmUoKQ==",highlighted:'accelerator.wait<span class="hljs-constructor">_for_everyone()</span>',lang:"",wrap:!1}}),g=new He({props:{$$slots:{default:[Ke]},$$scope:{ctx:q}}}),I=new L({props:{title:"Downloading a Dataset",local:"downloading-a-dataset",headingTag:"h2"}}),$=new He({props:{$$slots:{default:[De]},$$scope:{ctx:q}}}),v=new E({props:{code:"d2l0aCUyMGFjY2VsZXJhdG9yLm1haW5fcHJvY2Vzc19maXJzdCgpJTNBJTBBJTIwJTIwJTIwJTIwZGF0YXNldHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyZ2x1ZSUyMiUyQyUyMCUyMm1ycGMlMjIp",highlighted:`<span class="hljs-keyword">with</span> accelerator.main_process_first():
    datasets = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mrpc&quot;</span>)`,lang:"python",wrap:!1}}),Z=new E({props:{code:"JTIzJTIwRmlyc3QlMjBkbyUyMHNvbWV0aGluZyUyMG9uJTIwdGhlJTIwbWFpbiUyMHByb2Nlc3MlMEFpZiUyMGFjY2VsZXJhdG9yLmlzX21haW5fcHJvY2VzcyUzQSUwQSUyMCUyMCUyMCUyMGRhdGFzZXRzJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmdsdWUlMjIlMkMlMjAlMjJtcnBjJTIyKSUwQWVsc2UlM0ElMEElMjAlMjAlMjAlMjBhY2NlbGVyYXRvci53YWl0X2Zvcl9ldmVyeW9uZSgpJTBBJTBBJTIzJTIwQW5kJTIwdGhlbiUyMHNlbmQlMjBpdCUyMHRvJTIwdGhlJTIwcmVzdCUyMG9mJTIwdGhlbSUwQWlmJTIwbm90JTIwYWNjZWxlcmF0b3IuaXNfbWFpbl9wcm9jZXNzJTNBJTBBJTIwJTIwJTIwJTIwZGF0YXNldHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyZ2x1ZSUyMiUyQyUyMCUyMm1ycGMlMjIpJTBBZWxzZSUzQSUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLndhaXRfZm9yX2V2ZXJ5b25lKCk=",highlighted:`<span class="hljs-comment"># First do something on the main process</span>
<span class="hljs-keyword">if</span> accelerator.is_main_process:
    datasets = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mrpc&quot;</span>)
<span class="hljs-keyword">else</span>:
    accelerator.wait_for_everyone()

<span class="hljs-comment"># And then send it to the rest of them</span>
<span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> accelerator.is_main_process:
    datasets = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mrpc&quot;</span>)
<span class="hljs-keyword">else</span>:
    accelerator.wait_for_everyone()`,lang:"python",wrap:!1}}),B=new L({props:{title:"Saving the state_dict",local:"saving-the-statedict",headingTag:"h2"}}),X=new E({props:{code:"aWYlMjBhY2NlbGVyYXRvci5pc19tYWluX3Byb2Nlc3MlM0ElMEElMjAlMjAlMjAlMjBtb2RlbCUyMCUzRCUyMGFjY2VsZXJhdG9yLnVud3JhcF9tb2RlbChtb2RlbCklMEElMjAlMjAlMjAlMjB0b3JjaC5zYXZlKG1vZGVsLnN0YXRlX2RpY3QoKSUyQyUyMCUyMndlaWdodHMucHRoJTIyKQ==",highlighted:`<span class="hljs-keyword">if</span> accelerator.is_main_process:
    model = accelerator.unwrap_model(model)
    torch.save(model.state_dict(), <span class="hljs-string">&quot;weights.pth&quot;</span>)`,lang:"python",wrap:!1}}),A=new L({props:{title:"Loading in the state_dict",local:"loading-in-the-statedict",headingTag:"h2"}}),z=new E({props:{code:"d2l0aCUyMGFjY2VsZXJhdG9yLm1haW5fcHJvY2Vzc19maXJzdCgpJTNBJTBBJTIwJTIwJTIwJTIwc3RhdGUlMjAlM0QlMjB0b3JjaC5sb2FkKCUyMndlaWdodHMucHRoJTIyKSUwQSUyMCUyMCUyMCUyMG1vZGVsLmxvYWRfc3RhdGVfZGljdChzdGF0ZSk=",highlighted:`<span class="hljs-keyword">with</span> accelerator.main_process_first():
    state = torch.load(<span class="hljs-string">&quot;weights.pth&quot;</span>)
    model.load_state_dict(state)`,lang:"python",wrap:!1}}),Y=new L({props:{title:"Applying a multi-worker CPU operation",local:"applying-a-multi-worker-cpu-operation",headingTag:"h2"}}),H=new E({props:{code:"ZGF0YXNldHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyZ2x1ZSUyMiUyQyUyMCUyMm1ycGMlMjIpJTBBJTBBd2l0aCUyMGFjY2VsZXJhdG9yLm1haW5fcHJvY2Vzc19maXJzdCgpJTNBJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVkX2RhdGFzZXRzJTIwJTNEJTIwZGF0YXNldHMubWFwKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRva2VuaXplX2Z1bmN0aW9uJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYmF0Y2hlZCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZW1vdmVfY29sdW1ucyUzRCU1QiUyMmlkeCUyMiUyQyUyMCUyMnNlbnRlbmNlMSUyMiUyQyUyMCUyMnNlbnRlbmNlMiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCk=",highlighted:`datasets = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mrpc&quot;</span>)

<span class="hljs-keyword">with</span> accelerator.main_process_first():
    tokenized_datasets = datasets.<span class="hljs-built_in">map</span>(
        tokenize_function,
        batched=<span class="hljs-literal">True</span>,
        remove_columns=[<span class="hljs-string">&quot;idx&quot;</span>, <span class="hljs-string">&quot;sentence1&quot;</span>, <span class="hljs-string">&quot;sentence2&quot;</span>],
    )`,lang:"python",wrap:!1}}),N=new L({props:{title:"Applying checks such as Early Stopping",local:"applying-checks-such-as-early-stopping",headingTag:"h2"}}),S=new E({props:{code:"Zm9yJTIwKHglMkN5KSUyMGluJTIwZGF0YV9sb2FkZXIlM0ElMEElMjAlMjAlMjAlMjBsb2dpdHMlMjAlM0QlMjBtb2RlbCh4KSUwQSUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBsb3NzX2Z1bmMobG9naXRzJTJDJTIweSklMEElMjAlMjAlMjAlMjAlMjMlMjBBc3N1bWUlMjAlNjBzaG91bGRfZG9fZWFybHlfc3RvcHBpbmclNjAlMjBpcyUyMGElMjBjdXN0b20lMjBkZWZpbmVkJTIwZnVuY3Rpb24lMjB0aGF0JTIwcmV0dXJucyUyMGElMjBjb25kaXRpb25hbCUwQSUyMCUyMCUyMCUyMGlmJTIwc2hvdWxkX2RvX2Vhcmx5X3N0b3BwaW5nKGxvc3MpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3Iuc2V0X3RyaWdnZXIoKSUwQSUwQSUyMCUyMCUyMCUyMCUyMyUyMExhdGVyJTIwaW4lMjB0aGUlMjB0cmFpbmluZyUyMHNjcmlwdCUyMHdoZW4lMjB3ZSUyMG5lZWQlMjB0byUyMGNoZWNrJTIwZm9yJTIwdGhlJTIwYnJlYWtwb2ludCUwQSUyMCUyMCUyMCUyMGlmJTIwYWNjZWxlcmF0b3IuY2hlY2tfdHJpZ2dlcigpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwYnJlYWs=",highlighted:`<span class="hljs-keyword">for</span> (x,y) <span class="hljs-keyword">in</span> data_loader:
    logits = model(x)
    loss = loss_func(logits, y)
    <span class="hljs-comment"># Assume \`should_do_early_stopping\` is a custom defined function that returns a conditional</span>
    <span class="hljs-keyword">if</span> should_do_early_stopping(loss):
        accelerator.set_trigger()

    <span class="hljs-comment"># Later in the training script when we need to check for the breakpoint</span>
    <span class="hljs-keyword">if</span> accelerator.check_trigger():
        <span class="hljs-keyword">break</span>`,lang:"python",wrap:!1}}),V=new Pe({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/concept_guides/deferring_execution.md"}}),{c(){i=y("meta"),u=a(),o=y("p"),w=a(),r(j.$$.fragment),K=a(),r(U.$$.fragment),D=a(),T=y("p"),T.textContent=be,O=a(),J=y("p"),J.textContent=Ce,ee=a(),r(_.$$.fragment),te=a(),b=y("p"),b.textContent=Ie,se=a(),C=y("p"),C.textContent=ke,le=a(),r(g.$$.fragment),ae=a(),r(I.$$.fragment),ne=a(),k=y("p"),k.textContent=ve,ie=a(),r($.$$.fragment),oe=a(),r(v.$$.fragment),re=a(),G=y("p"),G.textContent=Ge,pe=a(),r(Z.$$.fragment),ce=a(),r(B.$$.fragment),de=a(),x=y("p"),x.innerHTML=Ze,me=a(),r(X.$$.fragment),he=a(),r(A.$$.fragment),ye=a(),W=y("p"),W.innerHTML=Be,fe=a(),r(z.$$.fragment),Me=a(),r(Y.$$.fragment),ue=a(),F=y("p"),F.innerHTML=xe,we=a(),r(H.$$.fragment),ge=a(),r(N.$$.fragment),$e=a(),R=y("p"),R.innerHTML=Xe,je=a(),Q=y("p"),Q.innerHTML=Ae,Ue=a(),r(S.$$.fragment),Te=a(),r(V.$$.fragment),Je=a(),P=y("p"),this.h()},l(e){const t=Ee("svelte-u9bgzb",document.head);i=f(t,"META",{name:!0,content:!0}),t.forEach(s),u=n(e),o=f(e,"P",{}),Ye(o).forEach(s),w=n(e),p(j.$$.fragment,e),K=n(e),p(U.$$.fragment,e),D=n(e),T=f(e,"P",{"data-svelte-h":!0}),M(T)!=="svelte-kzhlld"&&(T.textContent=be),O=n(e),J=f(e,"P",{"data-svelte-h":!0}),M(J)!=="svelte-15mf0bm"&&(J.textContent=Ce),ee=n(e),p(_.$$.fragment,e),te=n(e),b=f(e,"P",{"data-svelte-h":!0}),M(b)!=="svelte-19k5yf4"&&(b.textContent=Ie),se=n(e),C=f(e,"P",{"data-svelte-h":!0}),M(C)!=="svelte-8u19qz"&&(C.textContent=ke),le=n(e),p(g.$$.fragment,e),ae=n(e),p(I.$$.fragment,e),ne=n(e),k=f(e,"P",{"data-svelte-h":!0}),M(k)!=="svelte-g4t7z"&&(k.textContent=ve),ie=n(e),p($.$$.fragment,e),oe=n(e),p(v.$$.fragment,e),re=n(e),G=f(e,"P",{"data-svelte-h":!0}),M(G)!=="svelte-mg8wl6"&&(G.textContent=Ge),pe=n(e),p(Z.$$.fragment,e),ce=n(e),p(B.$$.fragment,e),de=n(e),x=f(e,"P",{"data-svelte-h":!0}),M(x)!=="svelte-gpwub8"&&(x.innerHTML=Ze),me=n(e),p(X.$$.fragment,e),he=n(e),p(A.$$.fragment,e),ye=n(e),W=f(e,"P",{"data-svelte-h":!0}),M(W)!=="svelte-yeso4z"&&(W.innerHTML=Be),fe=n(e),p(z.$$.fragment,e),Me=n(e),p(Y.$$.fragment,e),ue=n(e),F=f(e,"P",{"data-svelte-h":!0}),M(F)!=="svelte-n7gycq"&&(F.innerHTML=xe),we=n(e),p(H.$$.fragment,e),ge=n(e),p(N.$$.fragment,e),$e=n(e),R=f(e,"P",{"data-svelte-h":!0}),M(R)!=="svelte-j7zjkt"&&(R.innerHTML=Xe),je=n(e),Q=f(e,"P",{"data-svelte-h":!0}),M(Q)!=="svelte-1oyld50"&&(Q.innerHTML=Ae),Ue=n(e),p(S.$$.fragment,e),Te=n(e),p(V.$$.fragment,e),Je=n(e),P=f(e,"P",{}),Ye(P).forEach(s),this.h()},h(){Fe(i,"name","hf:doc:metadata"),Fe(i,"content",et)},m(e,t){qe(document.head,i),l(e,u,t),l(e,o,t),l(e,w,t),c(j,e,t),l(e,K,t),c(U,e,t),l(e,D,t),l(e,T,t),l(e,O,t),l(e,J,t),l(e,ee,t),c(_,e,t),l(e,te,t),l(e,b,t),l(e,se,t),l(e,C,t),l(e,le,t),c(g,e,t),l(e,ae,t),c(I,e,t),l(e,ne,t),l(e,k,t),l(e,ie,t),c($,e,t),l(e,oe,t),c(v,e,t),l(e,re,t),l(e,G,t),l(e,pe,t),c(Z,e,t),l(e,ce,t),c(B,e,t),l(e,de,t),l(e,x,t),l(e,me,t),c(X,e,t),l(e,he,t),c(A,e,t),l(e,ye,t),l(e,W,t),l(e,fe,t),c(z,e,t),l(e,Me,t),c(Y,e,t),l(e,ue,t),l(e,F,t),l(e,we,t),c(H,e,t),l(e,ge,t),c(N,e,t),l(e,$e,t),l(e,R,t),l(e,je,t),l(e,Q,t),l(e,Ue,t),c(S,e,t),l(e,Te,t),c(V,e,t),l(e,Je,t),l(e,P,t),_e=!0},p(e,[t]){const We={};t&2&&(We.$$scope={dirty:t,ctx:e}),g.$set(We);const ze={};t&2&&(ze.$$scope={dirty:t,ctx:e}),$.$set(ze)},i(e){_e||(d(j.$$.fragment,e),d(U.$$.fragment,e),d(_.$$.fragment,e),d(g.$$.fragment,e),d(I.$$.fragment,e),d($.$$.fragment,e),d(v.$$.fragment,e),d(Z.$$.fragment,e),d(B.$$.fragment,e),d(X.$$.fragment,e),d(A.$$.fragment,e),d(z.$$.fragment,e),d(Y.$$.fragment,e),d(H.$$.fragment,e),d(N.$$.fragment,e),d(S.$$.fragment,e),d(V.$$.fragment,e),_e=!0)},o(e){m(j.$$.fragment,e),m(U.$$.fragment,e),m(_.$$.fragment,e),m(g.$$.fragment,e),m(I.$$.fragment,e),m($.$$.fragment,e),m(v.$$.fragment,e),m(Z.$$.fragment,e),m(B.$$.fragment,e),m(X.$$.fragment,e),m(A.$$.fragment,e),m(z.$$.fragment,e),m(Y.$$.fragment,e),m(H.$$.fragment,e),m(N.$$.fragment,e),m(S.$$.fragment,e),m(V.$$.fragment,e),_e=!1},d(e){e&&(s(u),s(o),s(w),s(K),s(D),s(T),s(O),s(J),s(ee),s(te),s(b),s(se),s(C),s(le),s(ae),s(ne),s(k),s(ie),s(oe),s(re),s(G),s(pe),s(ce),s(de),s(x),s(me),s(he),s(ye),s(W),s(fe),s(Me),s(ue),s(F),s(we),s(ge),s($e),s(R),s(je),s(Q),s(Ue),s(Te),s(Je),s(P)),s(i),h(j,e),h(U,e),h(_,e),h(g,e),h(I,e),h($,e),h(v,e),h(Z,e),h(B,e),h(X,e),h(A,e),h(z,e),h(Y,e),h(H,e),h(N,e),h(S,e),h(V,e)}}}const et='{"title":"Executing and deferring jobs","local":"executing-and-deferring-jobs","sections":[{"title":"Downloading a Dataset","local":"downloading-a-dataset","sections":[],"depth":2},{"title":"Saving the state_dict","local":"saving-the-statedict","sections":[],"depth":2},{"title":"Loading in the state_dict","local":"loading-in-the-statedict","sections":[],"depth":2},{"title":"Applying a multi-worker CPU operation","local":"applying-a-multi-worker-cpu-operation","sections":[],"depth":2},{"title":"Applying checks such as Early Stopping","local":"applying-checks-such-as-early-stopping","sections":[],"depth":2}],"depth":1}';function tt(q){return Qe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ot extends Se{constructor(i){super(),Ve(this,i,tt,Oe,Re,{})}}export{ot as component};