accelerate/pr_4085/en/_app/immutable/nodes/4.9f4ceebf.js

import{s as al,o as sl,n as Ce}from"../chunks/scheduler.b9285784.js";import{S as il,i as cl,e as o,s as a,c as u,h as ol,a as r,d as l,b as s,f as ll,g as m,j as p,k as nl,l as rl,m as n,n as f,t as d,o as h,p as y,q as pl,r as ul}from"../chunks/index.26bc89a1.js";import{T as Je}from"../chunks/Tip.e4eba3d6.js";import{C as ml,H as Ue,E as fl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.0ecbc028.js";import{C as $}from"../chunks/CodeBlock.a2b9dbf6.js";function dl(J){let i,w=`It’s recommended you run <code>accelerate config</code> before using <code>accelerate launch</code> to configure your environment to your liking.
Otherwise Accelerate will use very basic defaults depending on your system setup.`;return{c(){i=o("p"),i.innerHTML=w},l(c){i=r(c,"P",{"data-svelte-h":!0}),p(i)!=="svelte-u7llw1"&&(i.innerHTML=w)},m(c,M){n(c,i,M)},p:Ce,d(c){c&&l(i)}}}function hl(J){let i,w="If you are familiar with launching scripts in PyTorch yourself such as with <code>torchrun</code>, you can still do this. It is not required to use <code>accelerate launch</code>.";return{c(){i=o("p"),i.innerHTML=w},l(c){i=r(c,"P",{"data-svelte-h":!0}),p(i)!=="svelte-l2vpv8"&&(i.innerHTML=w)},m(c,M){n(c,i,M)},p:Ce,d(c){c&&l(i)}}}function yl(J){let i,w="Even if you are not using Accelerate in your code, you can still use the launcher for starting your scripts!";return{c(){i=o("p"),i.textContent=w},l(c){i=r(c,"P",{"data-svelte-h":!0}),p(i)!=="svelte-fshzln"&&(i.textContent=w)},m(c,M){n(c,i,M)},p:Ce,d(c){c&&l(i)}}}function wl(J){let i,w="You can run your code on CPU as well! This is helpful for debugging and testing purposes on toy models and datasets.",c,M,g;return M=new $({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tY3B1JTIwJTdCc2NyaXB0X25hbWUucHklN0QlMjAlN0ItLWFyZzElN0QlMjAlN0ItLWFyZzIlN0Q=",highlighted:"accelerate launch --cpu {script_name.py} {--arg1} {--arg2}",lang:"bash",wrap:!1}}),{c(){i=o("p"),i.textContent=w,c=a(),u(M.$$.fragment)},l(T){i=r(T,"P",{"data-svelte-h":!0}),p(i)!=="svelte-zpf42g"&&(i.textContent=w),c=s(T),m(M.$$.fragment,T)},m(T,U){n(T,i,U),n(T,c,U),f(M,T,U),g=!0},p:Ce,i(T){g||(d(M.$$.fragment,T),g=!0)},o(T){h(M.$$.fragment,T),g=!1},d(T){T&&(l(i),l(c)),y(M,T)}}}function Ml(J){let i;return{c(){i=pl("It is required that the command be run on all nodes for everything to start, not just running it from the main node. You can use something like SLURM or a different process executor to wrap around this requirement and call everything from a single command.")},l(w){i=ul(w,"It is required that the command be run on all nodes for everything to start, not just running it from the main node. You can use something like SLURM or a different process executor to wrap around this requirement and call everything from a single command.")},m(w,c){n(w,i,c)},d(w){w&&l(i)}}}function Tl(J){let i,w="It is recommended to use the intranet IP of your main node over the public IP for better latency. This is the <code>192.168.x.x</code> or the <code>172.x.x.x</code> address you see when you run <code>hostname -I</code> on the main node.";return{c(){i=o("p"),i.innerHTML=w},l(c){i=r(c,"P",{"data-svelte-h":!0}),p(i)!=="svelte-psz6nh"&&(i.innerHTML=w)},m(c,M){n(c,i,M)},p:Ce,d(c){c&&l(i)}}}function $l(J){let i,w,c,M,g,T,U,be,x,Ct=`In the previous tutorial, you were introduced to how to modify your current training script to use Accelerate.
The final version of that code is shown below:`,Ie,Z,je,W,bt="But how do you run this code and have it utilize the special hardware available to it?",ve,G,It="First, you should rewrite the above code into a function, and make it callable as a script. For example:",_e,L,xe,B,jt="Next, you need to launch it with <code>accelerate launch</code>.",Ze,C,We,N,Ge,S,vt=`Accelerate has a special CLI command to help you launch your code in your system through <code>accelerate launch</code>.
This command wraps around all of the different commands needed to launch your script on various platforms, without you having to remember what each of them is.`,Le,b,Be,Y,_t="You can launch your script quickly by using:",Ne,H,Se,k,xt="Just put <code>accelerate launch</code> at the start of your command, and pass in additional arguments and parameters to your script afterward like normal!",Ye,A,Zt=`Since this runs the various torch spawn methods, all of the expected environment variables can be modified here as well.
For example, here is how to use <code>accelerate launch</code> with a single GPU:`,He,V,ke,F,Wt=`You can also use <code>accelerate launch</code> without performing <code>accelerate config</code> first, but you may need to manually pass in the right configuration parameters.
In this case, Accelerate will make some hyperparameter decisions for you, e.g., if GPUs are available, it will use all of them by default without the mixed precision.
Here is how you would use all GPUs and train with mixed precision disabled:`,Ae,R,Ve,z,Gt="Or by specifying a number of GPUs to use:",Fe,X,Re,E,Lt=`To get more specific you should pass in the needed parameters yourself. For instance, here is how you
would also launch that same script on two GPUs using mixed precision while avoiding all of the warnings:`,ze,Q,Xe,P,Bt="For a complete list of parameters you can pass in, run:",Ee,D,Qe,I,Pe,q,Nt="For a visualization of this difference, that earlier <code>accelerate launch</code> on multi-gpu would look something like so with <code>torchrun</code>:",De,K,qe,O,St=`You can also launch your script utilizing the launch CLI as a python module itself, enabling the ability to pass in other python-specific
launching behaviors. To do so, use <code>accelerate.commands.launch</code> instead of <code>accelerate launch</code>:`,Ke,ee,Oe,te,Yt=`If you want to execute the script with any other python flags, you can pass them in as well similar to <code>-m</code>, such as
the below example enabling unbuffered stdout and stderr:`,et,le,tt,j,lt,ne,nt,ae,Ht="Why is it useful to the point you should <strong>always</strong> run <code>accelerate config</code>?",at,se,kt=`Remember that earlier call to <code>accelerate launch</code> as well as <code>torchrun</code>?
Post configuration, to run that script with the needed parts you just need to use <code>accelerate launch</code> outright, without passing anything else in:`,st,ie,it,ce,ct,oe,At=`As briefly mentioned earlier, <code>accelerate launch</code> should be mostly used through combining set configurations
made with the <code>accelerate config</code> command. These configs are saved to a <code>default_config.yaml</code> file in your cache folder for Accelerate.
This cache folder is located at (with decreasing order of priority):`,ot,re,Vt=`<li>The content of your environment variable <code>HF_HOME</code> suffixed with <code>accelerate</code>.</li> <li>If it does not exist, the content of your environment variable <code>XDG_CACHE_HOME</code> suffixed with
<code>huggingface/accelerate</code>.</li> <li>If this does not exist either, the folder <code>~/.cache/huggingface/accelerate</code>.</li>`,rt,pe,Ft=`To have multiple configurations, the flag <code>--config_file</code> can be passed to the <code>accelerate launch</code> command paired
with the location of the custom yaml.`,pt,ue,Rt="An example yaml may look something like the following for two GPUs on a single machine using <code>fp16</code> for mixed precision:",ut,me,mt,fe,zt="Launching a script from the location of that custom yaml file looks like the following:",ft,de,dt,he,ht,ye,Xt='Multi-node training with Accelerate is similar to <a href="https://pytorch.org/tutorials/intermediate/ddp_series_multinode.html" rel="nofollow">multi-node training with torchrun</a>. The simplest way to launch a multi-node training run is to do the following:',yt,we,Et="<li>Copy your codebase and data to all nodes. (or place them on a shared filesystem)</li> <li>Setup your python packages on all nodes.</li> <li>Run <code>accelerate config</code> on the main single node first. After specifying the number of nodes, you will be asked to specify the rank of each node (this will be 0 for the main/master node), along with the IP address and port for the main process. This is required for the worker nodes to communicate with the main process. Afterwards, you can copy or send this config file across all of your nodes, changing the <code>machine_rank</code> to 1, 2,3, etc. to avoid having to run the command (or just follow their directions directly for launching with <code>torchrun</code> as well)</li>",wt,Me,Qt="Once you have done this, you can start your multi-node training run by running <code>accelerate launch</code> (or <code>torchrun</code>) on all nodes.",Mt,v,Tt,_,$t,Te,Pt='To get a better idea about multi-node training, check out our example for <a href="https://huggingface.co/blog/ram-efficient-pytorch-fsdp" rel="nofollow">multi-node training with FSDP</a>.',Jt,$e,gt,ge,Ut;return g=new ml({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new Ue({props:{title:"Launching Accelerate scripts",local:"launching-accelerate-scripts",headingTag:"h1"}}),Z=new $({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoKSUwQSUwQW1vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwdHJhaW5pbmdfZGF0YWxvYWRlciUyQyUyMHNjaGVkdWxlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUoJTBBJTIwJTIwJTIwJTIwbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjB0cmFpbmluZ19kYXRhbG9hZGVyJTJDJTIwc2NoZWR1bGVyJTBBKSUwQSUwQWZvciUyMGJhdGNoJTIwaW4lMjB0cmFpbmluZ19kYXRhbG9hZGVyJTNBJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnplcm9fZ3JhZCgpJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTJDJTIwdGFyZ2V0cyUyMCUzRCUyMGJhdGNoJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0cyklMEElMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mdW5jdGlvbihvdXRwdXRzJTJDJTIwdGFyZ2V0cyklMEElMjAlMjAlMjAlMjBhY2NlbGVyYXRvci5iYWNrd2FyZChsb3NzKSUwQSUyMCUyMCUyMCUyMG9wdGltaXplci5zdGVwKCklMEElMjAlMjAlMjAlMjBzY2hlZHVsZXIuc3RlcCgp",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator()

model, optimizer, training_dataloader, scheduler = accelerator.prepare(
    model, optimizer, training_dataloader, scheduler
)

<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> training_dataloader:
    optimizer.zero_grad()
    inputs, targets = batch
    outputs = model(inputs)
    loss = loss_function(outputs, targets)
    accelerator.backward(loss)
    optimizer.step()
    scheduler.step()`,lang:"python",wrap:!1}}),L=new $({props:{code:"JTIwJTIwZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUyMCUyMCUwQSUyQiUyMGRlZiUyMG1haW4oKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwdHJhaW5pbmdfZGF0YWxvYWRlciUyQyUyMHNjaGVkdWxlciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjB0cmFpbmluZ19kYXRhbG9hZGVyJTJDJTIwc2NoZWR1bGVyJTBBJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMGJhdGNoJTIwaW4lMjB0cmFpbmluZ19kYXRhbG9hZGVyJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyLnplcm9fZ3JhZCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5wdXRzJTJDJTIwdGFyZ2V0cyUyMCUzRCUyMGJhdGNoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mdW5jdGlvbihvdXRwdXRzJTJDJTIwdGFyZ2V0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhY2NlbGVyYXRvci5iYWNrd2FyZChsb3NzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci5zdGVwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzY2hlZHVsZXIuc3RlcCgpJTBBJTBBJTJCJTIwaWYlMjBfX25hbWVfXyUyMCUzRCUzRCUyMCUyMl9fbWFpbl9fJTIyJTNBJTBBJTJCJTIwJTIwJTIwJTIwJTIwbWFpbigp",highlighted:`  from accelerate import Accelerator
  
<span class="hljs-addition">+ def main():</span>
      accelerator = Accelerator()

      model, optimizer, training_dataloader, scheduler = accelerator.prepare(
          model, optimizer, training_dataloader, scheduler
      )

      for batch in training_dataloader:
          optimizer.zero_grad()
          inputs, targets = batch
          outputs = model(inputs)
          loss = loss_function(outputs, targets)
          accelerator.backward(loss)
          optimizer.step()
          scheduler.step()

<span class="hljs-addition">+ if __name__ == &quot;__main__&quot;:</span>
<span class="hljs-addition">+     main()</span>`,lang:"diff",wrap:!1}}),C=new Je({props:{warning:!0,$$slots:{default:[dl]},$$scope:{ctx:J}}}),N=new Ue({props:{title:"Using accelerate launch",local:"using-accelerate-launch",headingTag:"h2"}}),b=new Je({props:{$$slots:{default:[hl]},$$scope:{ctx:J}}}),H=new $({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMCU3QnNjcmlwdF9uYW1lLnB5JTdEJTIwLS1hcmcxJTIwLS1hcmcyJTIwLi4u",highlighted:"accelerate launch {script_name.py} --arg1 --arg2 ...",lang:"bash",wrap:!1}}),V=new $({props:{code:"JTIzJTIwZm9yJTIwY3VkYSUyMGRldmljZSUzQSUwQUNVREFfVklTSUJMRV9ERVZJQ0VTJTNEJTIyMCUyMiUyMGFjY2VsZXJhdGUlMjBsYXVuY2glMjAlN0JzY3JpcHRfbmFtZS5weSU3RCUyMC0tYXJnMSUyMC0tYXJnMiUyMC4uLiUwQSUyMyUyMGZvciUyMHhwdSUyMGRldmljZSUzQSUwQVpFX0FGRklOSVRZX01BU0slM0QlMjIwJTIyJTIwYWNjZWxlcmF0ZSUyMGxhdW5jaCUyMCU3QnNjcmlwdF9uYW1lLnB5JTdEJTIwLS1hcmcxJTIwLS1hcmcyJTIwLi4u",highlighted:`<span class="hljs-comment"># for cuda device:</span>
CUDA_VISIBLE_DEVICES=<span class="hljs-string">&quot;0&quot;</span> accelerate launch {script_name.py} --arg1 --arg2 ...
<span class="hljs-comment"># for xpu device:</span>
ZE_AFFINITY_MASK=<span class="hljs-string">&quot;0&quot;</span> accelerate launch {script_name.py} --arg1 --arg2 ...`,lang:"bash",wrap:!1}}),R=new $({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbXVsdGlfZ3B1JTIwJTdCc2NyaXB0X25hbWUucHklN0QlMjAlN0ItLWFyZzElN0QlMjAlN0ItLWFyZzIlN0QlMjAuLi4=",highlighted:"accelerate launch --multi_gpu {script_name.py} {--arg1} {--arg2} ...",lang:"bash",wrap:!1}}),X=new $({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbnVtX3Byb2Nlc3NlcyUzRDIlMjAlN0JzY3JpcHRfbmFtZS5weSU3RCUyMCU3Qi0tYXJnMSU3RCUyMCU3Qi0tYXJnMiU3RCUyMC4uLg==",highlighted:"accelerate launch --num_processes=2 {script_name.py} {--arg1} {--arg2} ...",lang:"bash",wrap:!1}}),Q=new $({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbXVsdGlfZ3B1JTIwLS1taXhlZF9wcmVjaXNpb24lM0RmcDE2JTIwLS1udW1fcHJvY2Vzc2VzJTNEMiUyMCU3QnNjcmlwdF9uYW1lLnB5JTdEJTIwJTdCLS1hcmcxJTdEJTIwJTdCLS1hcmcyJTdEJTIwLi4u",highlighted:"accelerate launch --multi_gpu --mixed_precision=fp16 --num_processes=2 {script_name.py} {--arg1} {--arg2} ...",lang:"bash",wrap:!1}}),D=new $({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC1o",highlighted:"accelerate launch -h",lang:"bash",wrap:!1}}),I=new Je({props:{$$slots:{default:[yl]},$$scope:{ctx:J}}}),K=new $({props:{code:"TUlYRURfUFJFQ0lTSU9OJTNEJTIyZnAxNiUyMiUyMHRvcmNocnVuJTIwLS1ucHJvY19wZXJfbm9kZSUzRDIlMjAtLW5ub2RlcyUzRDElMjAlN0JzY3JpcHRfbmFtZS5weSU3RCUyMCU3Qi0tYXJnMSU3RCUyMCU3Qi0tYXJnMiU3RCUyMC4uLg==",highlighted:'MIXED_PRECISION=<span class="hljs-string">&quot;fp16&quot;</span> torchrun --nproc_per_node=2 --nnodes=1 {script_name.py} {--arg1} {--arg2} ...',lang:"bash",wrap:!1}}),ee=new $({props:{code:"cHl0aG9uJTIwLW0lMjBhY2NlbGVyYXRlLmNvbW1hbmRzLmxhdW5jaCUyMC0tbnVtX3Byb2Nlc3NlcyUzRDIlMjAlN0JzY3JpcHRfbmFtZS5weSU3RCUyMCU3Qi0tYXJnMSU3RCUyMCU3Qi0tYXJnMiU3RA==",highlighted:"python -m accelerate.commands.launch --num_processes=2 {script_name.py} {--arg1} {--arg2}",lang:"bash",wrap:!1}}),le=new $({props:{code:"cHl0aG9uJTIwLXUlMjAtbSUyMGFjY2VsZXJhdGUuY29tbWFuZHMubGF1bmNoJTIwLS1udW1fcHJvY2Vzc2VzJTNEMiUyMCU3QnNjcmlwdF9uYW1lLnB5JTdEJTIwJTdCLS1hcmcxJTdEJTIwJTdCLS1hcmcyJTdE",highlighted:"python -u -m accelerate.commands.launch --num_processes=2 {script_name.py} {--arg1} {--arg2}",lang:"bash",wrap:!1}}),j=new Je({props:{$$slots:{default:[wl]},$$scope:{ctx:J}}}),ne=new Ue({props:{title:"Why you should always use accelerate config",local:"why-you-should-always-use-accelerate-config",headingTag:"h2"}}),ie=new $({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMCU3QnNjcmlwdF9uYW1lLnB5JTdEJTIwJTdCLS1hcmcxJTdEJTIwJTdCLS1hcmcyJTdEJTIwLi4u",highlighted:"accelerate launch {script_name.py} {--arg1} {--arg2} ...",lang:"bash",wrap:!1}}),ce=new Ue({props:{title:"Custom Configurations",local:"custom-configurations",headingTag:"h2"}}),me=new $({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTIwJTdCJTdEJTBBZGlzdHJpYnV0ZWRfdHlwZSUzQSUyME1VTFRJX0dQVSUwQWZzZHBfY29uZmlnJTNBJTIwJTdCJTdEJTBBbWFjaGluZV9yYW5rJTNBJTIwMCUwQW1haW5fcHJvY2Vzc19pcCUzQSUyMG51bGwlMEFtYWluX3Byb2Nlc3NfcG9ydCUzQSUyMG51bGwlMEFtYWluX3RyYWluaW5nX2Z1bmN0aW9uJTNBJTIwbWFpbiUwQW1peGVkX3ByZWNpc2lvbiUzQSUyMGZwMTYlMEFudW1fbWFjaGluZXMlM0ElMjAxJTBBbnVtX3Byb2Nlc3NlcyUzQSUyMDIlMEF1c2VfY3B1JTNBJTIwZmFsc2U=",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">deepspeed_config:</span> {}
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">MULTI_GPU</span>
<span class="hljs-attr">fsdp_config:</span> {}
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_process_ip:</span> <span class="hljs-literal">null</span>
<span class="hljs-attr">main_process_port:</span> <span class="hljs-literal">null</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">fp16</span>
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,lang:"yaml",wrap:!1}}),de=new $({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tY29uZmlnX2ZpbGUlMjAlN0JwYXRoJTJGdG8lMkZjb25maWclMkZteV9jb25maWdfZmlsZS55YW1sJTdEJTIwJTdCc2NyaXB0X25hbWUucHklN0QlMjAlN0ItLWFyZzElN0QlMjAlN0ItLWFyZzIlN0QlMjAuLi4=",highlighted:"accelerate launch --config_file {path/to/config/my_config_file.yaml} {script_name.py} {--arg1} {--arg2} ...",lang:"bash",wrap:!1}}),he=new Ue({props:{title:"Multi-node training",local:"multi-node-training",headingTag:"h2"}}),v=new Je({props:{$$slots:{default:[Ml]},$$scope:{ctx:J}}}),_=new Je({props:{$$slots:{default:[Tl]},$$scope:{ctx:J}}}),$e=new fl({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/basic_tutorials/launch.md"}}),{c(){i=o("meta"),w=a(),c=o("p"),M=a(),u(g.$$.fragment),T=a(),u(U.$$.fragment),be=a(),x=o("p"),x.textContent=Ct,Ie=a(),u(Z.$$.fragment),je=a(),W=o("p"),W.textContent=bt,ve=a(),G=o("p"),G.textContent=It,_e=a(),u(L.$$.fragment),xe=a(),B=o("p"),B.innerHTML=jt,Ze=a(),u(C.$$.fragment),We=a(),u(N.$$.fragment),Ge=a(),S=o("p"),S.innerHTML=vt,Le=a(),u(b.$$.fragment),Be=a(),Y=o("p"),Y.textContent=_t,Ne=a(),u(H.$$.fragment),Se=a(),k=o("p"),k.innerHTML=xt,Ye=a(),A=o("p"),A.innerHTML=Zt,He=a(),u(V.$$.fragment),ke=a(),F=o("p"),F.innerHTML=Wt,Ae=a(),u(R.$$.fragment),Ve=a(),z=o("p"),z.textContent=Gt,Fe=a(),u(X.$$.fragment),Re=a(),E=o("p"),E.textContent=Lt,ze=a(),u(Q.$$.fragment),Xe=a(),P=o("p"),P.textContent=Bt,Ee=a(),u(D.$$.fragment),Qe=a(),u(I.$$.fragment),Pe=a(),q=o("p"),q.innerHTML=Nt,De=a(),u(K.$$.fragment),qe=a(),O=o("p"),O.innerHTML=St,Ke=a(),u(ee.$$.fragment),Oe=a(),te=o("p"),te.innerHTML=Yt,et=a(),u(le.$$.fragment),tt=a(),u(j.$$.fragment),lt=a(),u(ne.$$.fragment),nt=a(),ae=o("p"),ae.innerHTML=Ht,at=a(),se=o("p"),se.innerHTML=kt,st=a(),u(ie.$$.fragment),it=a(),u(ce.$$.fragment),ct=a(),oe=o("p"),oe.innerHTML=At,ot=a(),re=o("ul"),re.innerHTML=Vt,rt=a(),pe=o("p"),pe.innerHTML=Ft,pt=a(),ue=o("p"),ue.innerHTML=Rt,ut=a(),u(me.$$.fragment),mt=a(),fe=o("p"),fe.textContent=zt,ft=a(),u(de.$$.fragment),dt=a(),u(he.$$.fragment),ht=a(),ye=o("p"),ye.innerHTML=Xt,yt=a(),we=o("ul"),we.innerHTML=Et,wt=a(),Me=o("p"),Me.innerHTML=Qt,Mt=a(),u(v.$$.fragment),Tt=a(),u(_.$$.fragment),$t=a(),Te=o("p"),Te.innerHTML=Pt,Jt=a(),u($e.$$.fragment),gt=a(),ge=o("p"),this.h()},l(e){const t=ol("svelte-u9bgzb",document.head);i=r(t,"META",{name:!0,content:!0}),t.forEach(l),w=s(e),c=r(e,"P",{}),ll(c).forEach(l),M=s(e),m(g.$$.fragment,e),T=s(e),m(U.$$.fragment,e),be=s(e),x=r(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-lkgw9a"&&(x.textContent=Ct),Ie=s(e),m(Z.$$.fragment,e),je=s(e),W=r(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-16box71"&&(W.textContent=bt),ve=s(e),G=r(e,"P",{"data-svelte-h":!0}),p(G)!=="svelte-aneby1"&&(G.textContent=It),_e=s(e),m(L.$$.fragment,e),xe=s(e),B=r(e,"P",{"data-svelte-h":!0}),p(B)!=="svelte-10zvu98"&&(B.innerHTML=jt),Ze=s(e),m(C.$$.fragment,e),We=s(e),m(N.$$.fragment,e),Ge=s(e),S=r(e,"P",{"data-svelte-h":!0}),p(S)!=="svelte-1vv9yh"&&(S.innerHTML=vt),Le=s(e),m(b.$$.fragment,e),Be=s(e),Y=r(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-1h0cdt"&&(Y.textContent=_t),Ne=s(e),m(H.$$.fragment,e),Se=s(e),k=r(e,"P",{"data-svelte-h":!0}),p(k)!=="svelte-1pkspsa"&&(k.innerHTML=xt),Ye=s(e),A=r(e,"P",{"data-svelte-h":!0}),p(A)!=="svelte-pi8kj9"&&(A.innerHTML=Zt),He=s(e),m(V.$$.fragment,e),ke=s(e),F=r(e,"P",{"data-svelte-h":!0}),p(F)!=="svelte-362fsb"&&(F.innerHTML=Wt),Ae=s(e),m(R.$$.fragment,e),Ve=s(e),z=r(e,"P",{"data-svelte-h":!0}),p(z)!=="svelte-1f9nwyt"&&(z.textContent=Gt),Fe=s(e),m(X.$$.fragment,e),Re=s(e),E=r(e,"P",{"data-svelte-h":!0}),p(E)!=="svelte-1sylg60"&&(E.textContent=Lt),ze=s(e),m(Q.$$.fragment,e),Xe=s(e),P=r(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-16vu16q"&&(P.textContent=Bt),Ee=s(e),m(D.$$.fragment,e),Qe=s(e),m(I.$$.fragment,e),Pe=s(e),q=r(e,"P",{"data-svelte-h":!0}),p(q)!=="svelte-vejsyy"&&(q.innerHTML=Nt),De=s(e),m(K.$$.fragment,e),qe=s(e),O=r(e,"P",{"data-svelte-h":!0}),p(O)!=="svelte-1bdn45"&&(O.innerHTML=St),Ke=s(e),m(ee.$$.fragment,e),Oe=s(e),te=r(e,"P",{"data-svelte-h":!0}),p(te)!=="svelte-1fkf60g"&&(te.innerHTML=Yt),et=s(e),m(le.$$.fragment,e),tt=s(e),m(j.$$.fragment,e),lt=s(e),m(ne.$$.fragment,e),nt=s(e),ae=r(e,"P",{"data-svelte-h":!0}),p(ae)!=="svelte-uc738h"&&(ae.innerHTML=Ht),at=s(e),se=r(e,"P",{"data-svelte-h":!0}),p(se)!=="svelte-1kcurae"&&(se.innerHTML=kt),st=s(e),m(ie.$$.fragment,e),it=s(e),m(ce.$$.fragment,e),ct=s(e),oe=r(e,"P",{"data-svelte-h":!0}),p(oe)!=="svelte-5vvoem"&&(oe.innerHTML=At),ot=s(e),re=r(e,"UL",{"data-svelte-h":!0}),p(re)!=="svelte-12bywag"&&(re.innerHTML=Vt),rt=s(e),pe=r(e,"P",{"data-svelte-h":!0}),p(pe)!=="svelte-1m3wa4r"&&(pe.innerHTML=Ft),pt=s(e),ue=r(e,"P",{"data-svelte-h":!0}),p(ue)!=="svelte-1aynacm"&&(ue.innerHTML=Rt),ut=s(e),m(me.$$.fragment,e),mt=s(e),fe=r(e,"P",{"data-svelte-h":!0}),p(fe)!=="svelte-s3hhms"&&(fe.textContent=zt),ft=s(e),m(de.$$.fragment,e),dt=s(e),m(he.$$.fragment,e),ht=s(e),ye=r(e,"P",{"data-svelte-h":!0}),p(ye)!=="svelte-11ne4r6"&&(ye.innerHTML=Xt),yt=s(e),we=r(e,"UL",{"data-svelte-h":!0}),p(we)!=="svelte-3qaail"&&(we.innerHTML=Et),wt=s(e),Me=r(e,"P",{"data-svelte-h":!0}),p(Me)!=="svelte-1dl49tf"&&(Me.innerHTML=Qt),Mt=s(e),m(v.$$.fragment,e),Tt=s(e),m(_.$$.fragment,e),$t=s(e),Te=r(e,"P",{"data-svelte-h":!0}),p(Te)!=="svelte-ajn4si"&&(Te.innerHTML=Pt),Jt=s(e),m($e.$$.fragment,e),gt=s(e),ge=r(e,"P",{}),ll(ge).forEach(l),this.h()},h(){nl(i,"name","hf:doc:metadata"),nl(i,"content",Jl)},m(e,t){rl(document.head,i),n(e,w,t),n(e,c,t),n(e,M,t),f(g,e,t),n(e,T,t),f(U,e,t),n(e,be,t),n(e,x,t),n(e,Ie,t),f(Z,e,t),n(e,je,t),n(e,W,t),n(e,ve,t),n(e,G,t),n(e,_e,t),f(L,e,t),n(e,xe,t),n(e,B,t),n(e,Ze,t),f(C,e,t),n(e,We,t),f(N,e,t),n(e,Ge,t),n(e,S,t),n(e,Le,t),f(b,e,t),n(e,Be,t),n(e,Y,t),n(e,Ne,t),f(H,e,t),n(e,Se,t),n(e,k,t),n(e,Ye,t),n(e,A,t),n(e,He,t),f(V,e,t),n(e,ke,t),n(e,F,t),n(e,Ae,t),f(R,e,t),n(e,Ve,t),n(e,z,t),n(e,Fe,t),f(X,e,t),n(e,Re,t),n(e,E,t),n(e,ze,t),f(Q,e,t),n(e,Xe,t),n(e,P,t),n(e,Ee,t),f(D,e,t),n(e,Qe,t),f(I,e,t),n(e,Pe,t),n(e,q,t),n(e,De,t),f(K,e,t),n(e,qe,t),n(e,O,t),n(e,Ke,t),f(ee,e,t),n(e,Oe,t),n(e,te,t),n(e,et,t),f(le,e,t),n(e,tt,t),f(j,e,t),n(e,lt,t),f(ne,e,t),n(e,nt,t),n(e,ae,t),n(e,at,t),n(e,se,t),n(e,st,t),f(ie,e,t),n(e,it,t),f(ce,e,t),n(e,ct,t),n(e,oe,t),n(e,ot,t),n(e,re,t),n(e,rt,t),n(e,pe,t),n(e,pt,t),n(e,ue,t),n(e,ut,t),f(me,e,t),n(e,mt,t),n(e,fe,t),n(e,ft,t),f(de,e,t),n(e,dt,t),f(he,e,t),n(e,ht,t),n(e,ye,t),n(e,yt,t),n(e,we,t),n(e,wt,t),n(e,Me,t),n(e,Mt,t),f(v,e,t),n(e,Tt,t),f(_,e,t),n(e,$t,t),n(e,Te,t),n(e,Jt,t),f($e,e,t),n(e,gt,t),n(e,ge,t),Ut=!0},p(e,[t]){const Dt={};t&2&&(Dt.$$scope={dirty:t,ctx:e}),C.$set(Dt);const qt={};t&2&&(qt.$$scope={dirty:t,ctx:e}),b.$set(qt);const Kt={};t&2&&(Kt.$$scope={dirty:t,ctx:e}),I.$set(Kt);const Ot={};t&2&&(Ot.$$scope={dirty:t,ctx:e}),j.$set(Ot);const el={};t&2&&(el.$$scope={dirty:t,ctx:e}),v.$set(el);const tl={};t&2&&(tl.$$scope={dirty:t,ctx:e}),_.$set(tl)},i(e){Ut||(d(g.$$.fragment,e),d(U.$$.fragment,e),d(Z.$$.fragment,e),d(L.$$.fragment,e),d(C.$$.fragment,e),d(N.$$.fragment,e),d(b.$$.fragment,e),d(H.$$.fragment,e),d(V.$$.fragment,e),d(R.$$.fragment,e),d(X.$$.fragment,e),d(Q.$$.fragment,e),d(D.$$.fragment,e),d(I.$$.fragment,e),d(K.$$.fragment,e),d(ee.$$.fragment,e),d(le.$$.fragment,e),d(j.$$.fragment,e),d(ne.$$.fragment,e),d(ie.$$.fragment,e),d(ce.$$.fragment,e),d(me.$$.fragment,e),d(de.$$.fragment,e),d(he.$$.fragment,e),d(v.$$.fragment,e),d(_.$$.fragment,e),d($e.$$.fragment,e),Ut=!0)},o(e){h(g.$$.fragment,e),h(U.$$.fragment,e),h(Z.$$.fragment,e),h(L.$$.fragment,e),h(C.$$.fragment,e),h(N.$$.fragment,e),h(b.$$.fragment,e),h(H.$$.fragment,e),h(V.$$.fragment,e),h(R.$$.fragment,e),h(X.$$.fragment,e),h(Q.$$.fragment,e),h(D.$$.fragment,e),h(I.$$.fragment,e),h(K.$$.fragment,e),h(ee.$$.fragment,e),h(le.$$.fragment,e),h(j.$$.fragment,e),h(ne.$$.fragment,e),h(ie.$$.fragment,e),h(ce.$$.fragment,e),h(me.$$.fragment,e),h(de.$$.fragment,e),h(he.$$.fragment,e),h(v.$$.fragment,e),h(_.$$.fragment,e),h($e.$$.fragment,e),Ut=!1},d(e){e&&(l(w),l(c),l(M),l(T),l(be),l(x),l(Ie),l(je),l(W),l(ve),l(G),l(_e),l(xe),l(B),l(Ze),l(We),l(Ge),l(S),l(Le),l(Be),l(Y),l(Ne),l(Se),l(k),l(Ye),l(A),l(He),l(ke),l(F),l(Ae),l(Ve),l(z),l(Fe),l(Re),l(E),l(ze),l(Xe),l(P),l(Ee),l(Qe),l(Pe),l(q),l(De),l(qe),l(O),l(Ke),l(Oe),l(te),l(et),l(tt),l(lt),l(nt),l(ae),l(at),l(se),l(st),l(it),l(ct),l(oe),l(ot),l(re),l(rt),l(pe),l(pt),l(ue),l(ut),l(mt),l(fe),l(ft),l(dt),l(ht),l(ye),l(yt),l(we),l(wt),l(Me),l(Mt),l(Tt),l($t),l(Te),l(Jt),l(gt),l(ge)),l(i),y(g,e),y(U,e),y(Z,e),y(L,e),y(C,e),y(N,e),y(b,e),y(H,e),y(V,e),y(R,e),y(X,e),y(Q,e),y(D,e),y(I,e),y(K,e),y(ee,e),y(le,e),y(j,e),y(ne,e),y(ie,e),y(ce,e),y(me,e),y(de,e),y(he,e),y(v,e),y(_,e),y($e,e)}}}const Jl='{"title":"Launching Accelerate scripts","local":"launching-accelerate-scripts","sections":[{"title":"Using accelerate launch","local":"using-accelerate-launch","sections":[],"depth":2},{"title":"Why you should always use accelerate config","local":"why-you-should-always-use-accelerate-config","sections":[],"depth":2},{"title":"Custom Configurations","local":"custom-configurations","sections":[],"depth":2},{"title":"Multi-node training","local":"multi-node-training","sections":[],"depth":2}],"depth":1}';function gl(J){return sl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class vl extends il{constructor(i){super(),cl(this,i,gl,$l,al,{})}}export{vl as component};