accelerate/pr_4039/en/_app/immutable/nodes/51.c8ec79a3.js

import{s as Yt,o as zt,n as Vt}from"../chunks/scheduler.b9285784.js";import{S as Lt,i as qt,e as i,s,c,h as Ot,a as r,d as a,b as n,f as It,g as m,j as p,k as Qt,l as Dt,m as l,n as h,t as d,o as f,p as u}from"../chunks/index.26bc89a1.js";import{T as Kt}from"../chunks/Tip.e4eba3d6.js";import{C as ea,H as _,E as ta}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.91c9ff84.js";import{C as $}from"../chunks/CodeBlock.ef23fd93.js";function aa(Ue){let M,Z='<strong>⚠️ Deprecated / Unmaintained:</strong> MS-AMP is no longer actively maintained by Microsoft. The <a href="https://github.com/Azure/MS-AMP" rel="nofollow">MS-AMP repository</a> has not received updates since 2023 and has known compatibility issues:',T,g,U="<li>Requires CUDA 11.x (does not support CUDA 12.x+)</li> <li>Requires older NCCL versions incompatible with recent PyTorch releases</li> <li>Does not support recent PyTorch versions (2.2+)</li>",J,y,B='<strong>We strongly recommend using <a href="#configuring-transformersengine"><code>TransformersEngine</code></a> or <a href="#configuring-torchao"><code>torchao</code></a> instead for all new and existing FP8 training workflows.</strong> Both are actively maintained and support modern CUDA/PyTorch versions. Native PyTorch FP8 support via <code>torchao</code> is particularly promising as a vendor-neutral solution.',b,j,C="The MS-AMP backend is retained in Accelerate for legacy compatibility but may be removed in a future release.";return{c(){M=i("p"),M.innerHTML=Z,T=s(),g=i("ul"),g.innerHTML=U,J=s(),y=i("p"),y.innerHTML=B,b=s(),j=i("p"),j.textContent=C},l(o){M=r(o,"P",{"data-svelte-h":!0}),p(M)!=="svelte-17ommj7"&&(M.innerHTML=Z),T=n(o),g=r(o,"UL",{"data-svelte-h":!0}),p(g)!=="svelte-fabj8h"&&(g.innerHTML=U),J=n(o),y=r(o,"P",{"data-svelte-h":!0}),p(y)!=="svelte-1v3pi2f"&&(y.innerHTML=B),b=n(o),j=r(o,"P",{"data-svelte-h":!0}),p(j)!=="svelte-1bl9wit"&&(j.textContent=C)},m(o,w){l(o,M,w),l(o,T,w),l(o,g,w),l(o,J,w),l(o,y,w),l(o,b,w),l(o,j,w)},p:Vt,d(o){o&&(a(M),a(T),a(g),a(J),a(y),a(b),a(j))}}}function la(Ue){let M,Z,T,g,U,J,y,B,b,j='Accelerate provides integrations to train on lower precision methods using specified supported hardware through the <code>TransformersEngine</code>, <code>MS-AMP</code>, and <code>torchao</code> packages. This documentation will help guide you through what hardware is supported, how to configure your <a href="/docs/accelerate/pr_4039/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> to leverage the low precision methods, and what you can expect when training.',C,o,w,F,ut='To explore more of the nitty-gritty in training in FP8 with PyTorch and Accelerate, check out the <a href="../concept_guides/low_precision_training">concept_guide</a> on why this can be difficult. But essentially rather than training in BF16, some (or all) aspects of training a model can be performed using 8 bits instead of 16. The challenge is doing so without degrading final performance.',je,x,Mt="This is only enabled on specific NVIDIA hardware, namely:",Je,k,yt="<li>Anything after the 3000 series consumer graphics cards (such as the 4090)</li> <li>Hopper-based GPU architectures (such as the <code>H100</code> and <code>H200</code>)</li>",$e,W,gt="What this will result in is some reduction in the memory used (as we’ve cut the needed memory in half for some parts of training) and an increase in throughput <em>should</em> be seen as well for larger models that can replace certain layers with FP8-enabled ones.",ve,R,_e,A,bt='Currently two actively maintained backends for FP8 are supported (<code>TransformersEngine</code> and <code>torchao</code>), each with different capabilities and configurations. A legacy <code>MS-AMP</code> backend also exists but is no longer recommended (see <a href="#configuring-ms-amp">below</a> for details).',Ze,E,wt='To use either, the same core API is used. Just pass <code>mixed_precision=&quot;fp8&quot;</code> to either the <a href="/docs/accelerate/pr_4039/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a>, during <code>accelerate config</code> when prompted about mixed precision, or as part of your <code>config.yaml</code> file in the <code>mixed_precision</code> key:',Be,X,Ce,P,Tt="To specify a backend (and customize other parts of the FP8 mixed precision setup), you can utilize one of the <code>RecipeKwargs</code> dataclasses such as <code>utils.AORecipeKwargs</code>, <code>utils.TERecipeKwargs</code>, or <code>utils.MSAMPRecipeKwargs</code>; you can also clarify it in your config <code>yaml</code>/during <code>accelerate launch</code>. We recommend using <code>TransformersEngine</code> or <code>torchao</code> for new projects:",Fe,G,xe,H,ke,N,We,v,Re,S,Ut="<code>MS-AMP</code> has a single configuration argument: the optimization level.",Ae,I,jt="Currently two levels of optimization are supported in the Accelerate integration, <code>&quot;O1&quot;</code> and <code>&quot;O2&quot;</code> (using the letter ‘o’, not zero).",Ee,Q,Jt="<li><code>&quot;O1&quot;</code> will cast the weight gradients and <code>all_reduce</code> communications to happen in 8-bit, while the rest are done in 16 bit. This reduces the general GPU memory usage and speeds up communication bandwidths.</li> <li><code>&quot;O2&quot;</code> will also cast first-order optimizer states into 8 bit, while the second order states are in FP16. (Currently just the <code>Adam</code> optimizer is supported). This tries its best to minimize final accuracy degradation and will save the highest potential memory.</li>",Xe,Y,$t="To specify an optimization level, pass it to the <code>FP8KwargsHandler</code> by setting the <code>optimization_level</code> argument:",Pe,z,Ge,V,vt="Or during <code>accelerate launch</code> via <code>--fp8_backend=msamp --fp8_opt_level=O2</code>",He,L,_t="Similarly this can be set in your <code>config.yaml</code>:",Ne,q,Se,O,Ie,D,Zt='TransformersEngine has many options for customizing how and what FP8 calculations are performed. A full list of supported arguments and what they mean are available in <a href="https://docs.nvidia.com/deeplearning/transformer-engine/user-guide/api/common.html" rel="nofollow">NVIDIA’s documentation</a>, however they are restated as part of <code>FP8KwargsHandler</code>’s docstring for your convenience.',Qe,K,Bt="Accelerate tries to set sensible defaults, but exploring and tweaking the various parameters yourself can lead to better performance potentially.",Ye,ee,Ct="To use it, specify <code>backend=&quot;te&quot;</code> and modify any of the arguments you want as part of your kwarg handler:",ze,te,Ve,ae,Ft="Or during <code>accelerate launch</code> via <code>--fp8_backend=te ...</code>. Use <code>accelerate launch --fp8_backend=te -h</code> to see relevent arguments.",Le,le,xt="Similarly this can be set in your <code>config.yaml</code>:",qe,se,Oe,ne,De,ie,kt='<code>torchao</code> is a <a href="https://github.com/pytorch/ao/tree/main/torchao/float8" rel="nofollow">PyTorch-driven</a> hackable FP8 backend, aiming to be more approchable than the prior two engines. One of the core differences with <code>ao</code> compared to the prior two is that for numerical stability, it’s found to be generally better off keeping the first <em>and</em> last layers in the model at the regular precision (be it FP32 or BF16), and then the other layers quantized down to FP8. As a result, a config for <code>ao</code> looks a bit differently:',Ke,re,Wt="<p>Note: this API is experimental and is subject to change</p>",et,pe,tt,oe,Rt="Or during <code>accelerate launch</code> via <code>--fp8_backend=ao ...</code>. Use <code>accelerate launch --fp8_backend=ao -h</code> to see relevent arguments.",at,ce,At="Similarly, this can be set in <code>config.yaml</code>:",lt,me,st,he,Et="To learn more about the specific parameters to be used, please see the official <code>torchao</code> repo.",nt,de,it,fe,Xt=`We have examples showcasing training with FP8 both with accelerate and its underlying implementation available in the accelerate repo.
Currently we support scripts showcasing:`,rt,ue,Pt="<li>Single GPU</li> <li>Distributed Data Parallelism (Multi-GPU)</li> <li>Fully Sharded Data Parallelism</li> <li>DeepSpeed ZeRO 1 through 3</li>",pt,Me,Gt='Find out more <a href="https://github.com/huggingface/accelerate/tree/main/benchmarks/fp8" rel="nofollow">here</a>',ot,ye,ct,ge,Ht="To learn more about training in FP8 please check out the following resources:",mt,be,Nt='<li><a href="../concept_guides/low_precision_training">Our concept guide</a> detailing into more about TransformersEngine, torchao, and MS-AMP</li> <li><a href="https://docs.nvidia.com/deeplearning/transformer-engine/user-guide/api/common.html" rel="nofollow">The <code>transformers-engine</code> documentation</a></li> <li><a href="https://github.com/pytorch/ao/tree/main/torchao/float8" rel="nofollow">The <code>torchao</code> documentation</a></li> <li><a href="https://azure.github.io/MS-AMP/docs/" rel="nofollow">The <code>MS-AMP</code> documentation</a> (⚠️ no longer maintained)</li>',ht,we,dt,Te,ft;return U=new ea({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),y=new _({props:{title:"Low Precision Training Methods",local:"low-precision-training-methods",headingTag:"h1"}}),o=new _({props:{title:"What training on FP8 means",local:"what-training-on-fp8-means",headingTag:"h2"}}),R=new _({props:{title:"Configuring the Accelerator",local:"configuring-the-accelerator",headingTag:"h2"}}),X=new $({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IobWl4ZWRfcHJlY2lzaW9uJTNEJTIyZnA4JTIyKQ==",highlighted:`from accelerate import Accelerator
<span class="hljs-attribute">accelerator</span> <span class="hljs-operator">=</span> Accelerator(mixed_precision<span class="hljs-operator">=</span><span class="hljs-string">&quot;fp8&quot;</span>)`,lang:"",wrap:!1}}),G=new $({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwVEVSZWNpcGVLd2FyZ3MlMkMlMjBBT1JlY2lwZUt3YXJncyUwQSUyMyUyMFVzZSUyMFRyYW5zZm9ybWVyc0VuZ2luZSUwQWt3YXJncyUyMCUzRCUyMCU1QlRFUmVjaXBlS3dhcmdzKCklNUQlMEElMjMlMjBPciUyMHRvJTIwdXNlJTIwdG9yY2hhbyUwQSUyMyUyMGt3YXJncyUyMCUzRCUyMCU1QkFPUmVjaXBlS3dhcmdzKCklNUQlMEFhY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKG1peGVkX3ByZWNpc2lvbiUzRCUyMmZwOCUyMiUyQyUyMGt3YXJnX2hhbmRsZXJzJTNEa3dhcmdzKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate import Accelerator
<span class="hljs-keyword">from</span> accelerate.utils import TERecipeKwargs, AORecipeKwargs
<span class="hljs-comment"># Use TransformersEngine</span>
kwargs = [TERecipeKwargs()]
<span class="hljs-comment"># Or to use torchao</span>
<span class="hljs-comment"># kwargs = [AORecipeKwargs()]</span>
accelerator = Accelerator(<span class="hljs-attribute">mixed_precision</span>=<span class="hljs-string">&quot;fp8&quot;</span>, <span class="hljs-attribute">kwarg_handlers</span>=kwargs)`,lang:"",wrap:!1}}),H=new $({props:{code:"bWl4ZWRfcHJlY2lzaW9uJTNBJTIwZnA4JTBBZnA4X2NvbmZpZyUzQSUwQSUyMCUyMGFtYXhfY29tcHV0ZV9hbGdvJTNBJTIwbWF4JTBBJTIwJTIwYW1heF9oaXN0b3J5X2xlbiUzQSUyMDEwMjQlMEElMjAlMjBiYWNrZW5kJTNBJTIwVEUlMEElMjAlMjBmcDhfZm9ybWF0JTNBJTIwSFlCUklEJTBBJTIwJTIwaW50ZXJ2YWwlM0ElMjAxJTBBJTIwJTIwbWFyZ2luJTNBJTIwMCUwQSUyMCUyMG92ZXJyaWRlX2xpbmVhcl9wcmVjaXNpb24lM0ElMjAoZmFsc2UlMkMlMjBmYWxzZSUyQyUyMGZhbHNlKSUwQSUyMCUyMHVzZV9hdXRvY2FzdF9kdXJpbmdfZXZhbCUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">fp8</span>
<span class="hljs-attr">fp8_config:</span>
  <span class="hljs-attr">amax_compute_algo:</span> <span class="hljs-string">max</span>
  <span class="hljs-attr">amax_history_len:</span> <span class="hljs-number">1024</span>
  <span class="hljs-attr">backend:</span> <span class="hljs-string">TE</span>
  <span class="hljs-attr">fp8_format:</span> <span class="hljs-string">HYBRID</span>
  <span class="hljs-attr">interval:</span> <span class="hljs-number">1</span>
  <span class="hljs-attr">margin:</span> <span class="hljs-number">0</span>
  <span class="hljs-attr">override_linear_precision:</span> <span class="hljs-string">(false,</span> <span class="hljs-literal">false</span><span class="hljs-string">,</span> <span class="hljs-literal">false</span><span class="hljs-string">)</span>
  <span class="hljs-attr">use_autocast_during_eval:</span> <span class="hljs-literal">false</span>`,lang:"",wrap:!1}}),N=new _({props:{title:"Configuring MS-AMP",local:"configuring-ms-amp",headingTag:"h2"}}),v=new Kt({props:{warning:!0,$$slots:{default:[aa]},$$scope:{ctx:Ue}}}),z=new $({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwRlA4UmVjaXBlS3dhcmdzJTBBa3dhcmdzJTIwJTNEJTIwJTVCRlA4UmVjaXBlS3dhcmdzKGJhY2tlbmQlM0QlMjJtc2FtcCUyMiUyQyUyMG9wdGltaXphdGlvbl9sZXZlbCUzRCUyMk8yJTIyKSU1RCUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IobWl4ZWRfcHJlY2lzaW9uJTNEJTIyZnA4JTIyJTJDJTIwa3dhcmdfaGFuZGxlcnMlM0Rrd2FyZ3Mp",highlighted:`<span class="hljs-keyword">from</span> accelerate import Accelerator
<span class="hljs-keyword">from</span> accelerate.utils import FP8RecipeKwargs
kwargs = [FP8RecipeKwargs(<span class="hljs-attribute">backend</span>=<span class="hljs-string">&quot;msamp&quot;</span>, <span class="hljs-attribute">optimization_level</span>=<span class="hljs-string">&quot;O2&quot;</span>)]
accelerator = Accelerator(<span class="hljs-attribute">mixed_precision</span>=<span class="hljs-string">&quot;fp8&quot;</span>, <span class="hljs-attribute">kwarg_handlers</span>=kwargs)`,lang:"",wrap:!1}}),q=new $({props:{code:"bWl4ZWRfcHJlY2lzaW9uJTNBJTIwZnA4JTBBZnA4X2NvbmZpZyUzQSUwQSUyMCUyMCUyMCUyMGJhY2tlbmQlM0ElMjBNU0FNUCUwQSUyMCUyMCUyMCUyMG9wdF9sZXZlbCUzQSUyME8y",highlighted:`<span class="hljs-symbol">mixed_precision:</span> fp8
<span class="hljs-symbol">fp8_config:</span>
<span class="hljs-symbol">    backend:</span> MSAMP
<span class="hljs-symbol">    opt_level:</span> O2`,lang:"",wrap:!1}}),O=new _({props:{title:"Configuring TransformersEngine",local:"configuring-transformersengine",headingTag:"h2"}}),te=new $({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwRlA4UmVjaXBlS3dhcmdzJTBBa3dhcmdzJTIwJTNEJTIwJTVCRlA4UmVjaXBlS3dhcmdzKGJhY2tlbmQlM0QlMjJ0ZSUyMiUyQyUyMC4uLiklNUQlMEFhY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKG1peGVkX3ByZWNpc2lvbiUzRCUyMmZwOCUyMiUyQyUyMGt3YXJnX2hhbmRsZXJzJTNEa3dhcmdzKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate import Accelerator
<span class="hljs-keyword">from</span> accelerate.utils import FP8RecipeKwargs
kwargs = [FP8RecipeKwargs(<span class="hljs-attribute">backend</span>=<span class="hljs-string">&quot;te&quot;</span>, <span class="hljs-built_in">..</span>.)]
accelerator = Accelerator(<span class="hljs-attribute">mixed_precision</span>=<span class="hljs-string">&quot;fp8&quot;</span>, <span class="hljs-attribute">kwarg_handlers</span>=kwargs)`,lang:"",wrap:!1}}),se=new $({props:{code:"bWl4ZWRfcHJlY2lzaW9uJTNBJTIwZnA4JTBBZnA4X2NvbmZpZyUzQSUwQSUyMCUyMCUyMCUyMGFtYXhfY29tcHV0ZV9hbGdvJTNBJTIwbWF4JTBBJTIwJTIwJTIwJTIwYW1heF9oaXN0b3J5X2xlbiUzQSUyMDEwMjQlMEElMjAlMjAlMjAlMjBiYWNrZW5kJTNBJTIwVEUlMEElMjAlMjAlMjAlMjBmcDhfZm9ybWF0JTNBJTIwSFlCUklEJTBBJTIwJTIwJTIwJTIwaW50ZXJ2YWwlM0ElMjAxJTBBJTIwJTIwJTIwJTIwbWFyZ2luJTNBJTIwMCUwQSUyMCUyMCUyMCUyMG92ZXJyaWRlX2xpbmVhcl9wcmVjaXNpb24lM0ElMjAoZmFsc2UlMkMlMjBmYWxzZSUyQyUyMGZhbHNlKSUwQSUyMCUyMCUyMCUyMHVzZV9hdXRvY2FzdF9kdXJpbmdfZXZhbCUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">fp8</span>
<span class="hljs-attr">fp8_config:</span>
    <span class="hljs-attr">amax_compute_algo:</span> <span class="hljs-string">max</span>
    <span class="hljs-attr">amax_history_len:</span> <span class="hljs-number">1024</span>
    <span class="hljs-attr">backend:</span> <span class="hljs-string">TE</span>
    <span class="hljs-attr">fp8_format:</span> <span class="hljs-string">HYBRID</span>
    <span class="hljs-attr">interval:</span> <span class="hljs-number">1</span>
    <span class="hljs-attr">margin:</span> <span class="hljs-number">0</span>
    <span class="hljs-attr">override_linear_precision:</span> <span class="hljs-string">(false,</span> <span class="hljs-literal">false</span><span class="hljs-string">,</span> <span class="hljs-literal">false</span><span class="hljs-string">)</span>
    <span class="hljs-attr">use_autocast_during_eval:</span> <span class="hljs-literal">false</span>`,lang:"",wrap:!1}}),ne=new _({props:{title:"Configuring torchao",local:"configuring-torchao",headingTag:"h2"}}),pe=new $({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwQU9SZWNpcGVLd2FyZ3MlMkMlMjBUb3JjaER5bmFtb1BsdWdpbiUyQyUyMEZ1bGx5U2hhcmRlZERhdGFQYXJhbGxlbFBsdWdpbiUwQWZyb20lMjB0b3JjaGFvLmZsb2F0OCUyMGltcG9ydCUyMEZsb2F0OExpbmVhckNvbmZpZyUwQSUwQWZzZHAyX3BsdWdpbiUyMCUzRCUyMEZ1bGx5U2hhcmRlZERhdGFQYXJhbGxlbFBsdWdpbiglMEElMjAlMjBmc2RwX3ZlcnNpb24lM0QyJTJDJTBBJTIwJTIwY3B1X3JhbV9lZmZpY2llbnRfbG9hZGluZyUzREZhbHNlJTJDJTIwJTIzJTIwQ1BVJTIwUkFNJTIwZWZmaWNpZW50JTIwbG9hZGluZyUyMENBTk5PVCUyMHdvcmslMjB3aXRoJTIwZnA4JTIwdG9yY2hhbyUwQSUyMCUyMGZzZHBfYXV0b193cmFwX3BvbGljeSUzRCUyMlRSQU5TRk9STUVSX0JBU0VEX1dSQVAlMjIlMkMlMEEpJTBBZHluYW1vX3BsdWdpbiUyMCUzRCUyMFRvcmNoRHluYW1vUGx1Z2luKCUwQSUyMCUyMGJhY2tlbmQlM0QlMjJpbmR1Y3RvciUyMiUyQyUwQSUyMCUyMHVzZV9yZWdpb25hbF9jb21waWxhdGlvbiUzRFRydWUlMkMlMEEpJTBBZnA4X2NvbmZpZyUyMCUzRCUyMEZsb2F0OExpbmVhckNvbmZpZyglMEElMjAlMjBlbmFibGVfZnNkcF9mbG9hdDhfYWxsX2dhdGhlciUzRFRydWUlMkMlMjAlMjMlMjBVc2UlMjBGUDglMjBhbGxfZ2F0aGVyJTIwaW4lMjBGU0RQMiUwQSUyMCUyMHBhZF9pbm5lcl9kaW0lM0RUcnVlJTJDJTBBKSUwQWt3YXJncyUyMCUzRCUyMCU1QkFPUmVjaXBlS3dhcmdzKCUwQSUyMCUyMGNvbmZpZyUzRGZwOF9jb25maWclMEEpJTVEJTBBYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvciglMEElMjAlMjBtaXhlZF9wcmVjaXNpb24lM0QlMjJmcDglMjIlMkMlMEElMjAlMjBmc2RwX3BsdWdpbiUzRGZzZHAyX3BsdWdpbiUyQyUwQSUyMCUyMGR5bmFtb19wbHVnaW4lM0RkeW5hbW9fcGx1Z2luJTJDJTBBJTIwJTIwa3dhcmdfaGFuZGxlcnMlM0Rrd2FyZ3MlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> accelerate import Accelerator
<span class="hljs-keyword">from</span> accelerate.utils import AORecipeKwargs, TorchDynamoPlugin, FullyShardedDataParallelPlugin
<span class="hljs-keyword">from</span> torchao.float8 import Float8LinearConfig

fsdp2_plugin = FullyShardedDataParallelPlugin(
  <span class="hljs-attribute">fsdp_version</span>=2,
  <span class="hljs-attribute">cpu_ram_efficient_loading</span>=<span class="hljs-literal">False</span>, # CPU RAM efficient loading CANNOT work with fp8 torchao
  <span class="hljs-attribute">fsdp_auto_wrap_policy</span>=<span class="hljs-string">&quot;TRANSFORMER_BASED_WRAP&quot;</span>,
)
dynamo_plugin = TorchDynamoPlugin(
  <span class="hljs-attribute">backend</span>=<span class="hljs-string">&quot;inductor&quot;</span>,
  <span class="hljs-attribute">use_regional_compilation</span>=<span class="hljs-literal">True</span>,
)
fp8_config = Float8LinearConfig(
  <span class="hljs-attribute">enable_fsdp_float8_all_gather</span>=<span class="hljs-literal">True</span>, # Use FP8 all_gather <span class="hljs-keyword">in</span> FSDP2
  <span class="hljs-attribute">pad_inner_dim</span>=<span class="hljs-literal">True</span>,
)
kwargs = [AORecipeKwargs(
  <span class="hljs-attribute">config</span>=fp8_config
)]
accelerator = Accelerator(
  <span class="hljs-attribute">mixed_precision</span>=<span class="hljs-string">&quot;fp8&quot;</span>,
  <span class="hljs-attribute">fsdp_plugin</span>=fsdp2_plugin,
  <span class="hljs-attribute">dynamo_plugin</span>=dynamo_plugin,
  <span class="hljs-attribute">kwarg_handlers</span>=kwargs,
)`,lang:"",wrap:!1}}),me=new $({props:{code:"bWl4ZWRfcHJlY2lzaW9uJTNBJTIwZnA4JTBBZnNkcF9jb25maWclM0ElMEElMjAlMjBmc2RwX2F1dG9fd3JhcF9wb2xpY3klM0ElMjBUUkFOU0ZPUk1FUl9CQVNFRF9XUkFQJTBBJTIwJTIwZnNkcF9jcHVfcmFtX2VmZmljaWVudF9sb2FkaW5nJTNBJTIwZmFsc2UlMEElMjAlMjBmc2RwX3ZlcnNpb24lM0ElMjAyJTBBZnA4X2NvbmZpZyUzQSUwQSUyMCUyMGJhY2tlbmQlM0ElMjBBTyUwQSUyMCUyMHBhZF9pbm5lcl9kaW0lM0ElMjB0cnVlJTBBJTIwJTIwZW5hYmxlX2ZzZHBfZmxvYXQ4X2FsbF9nYXRoZXIlM0ElMjB0cnVlJTBBZHluYW1vX2NvbmZpZyUzQSUwQSUyMCUyMGR5bmFtb19iYWNrZW5kJTNBJTIwSU5EVUNUT1IlMEElMjAlMjBkeW5hbW9fdXNlX3JlZ2lvbmFsX2NvbXBpbGF0aW9uJTNBJTIwdHJ1ZQ==",highlighted:`<span class="hljs-attr">mixed_precision:</span> <span class="hljs-string">fp8</span>
<span class="hljs-attr">fsdp_config:</span>
  <span class="hljs-attr">fsdp_auto_wrap_policy:</span> <span class="hljs-string">TRANSFORMER_BASED_WRAP</span>
  <span class="hljs-attr">fsdp_cpu_ram_efficient_loading:</span> <span class="hljs-literal">false</span>
  <span class="hljs-attr">fsdp_version:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">fp8_config:</span>
  <span class="hljs-attr">backend:</span> <span class="hljs-string">AO</span>
  <span class="hljs-attr">pad_inner_dim:</span> <span class="hljs-literal">true</span>
  <span class="hljs-attr">enable_fsdp_float8_all_gather:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">dynamo_config:</span>
  <span class="hljs-attr">dynamo_backend:</span> <span class="hljs-string">INDUCTOR</span>
  <span class="hljs-attr">dynamo_use_regional_compilation:</span> <span class="hljs-literal">true</span>`,lang:"",wrap:!1}}),de=new _({props:{title:"Example Zoo",local:"example-zoo",headingTag:"h2"}}),ye=new _({props:{title:"Further Reading",local:"further-reading",headingTag:"h2"}}),we=new ta({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/low_precision_training.md"}}),{c(){M=i("meta"),Z=s(),T=i("p"),g=s(),c(U.$$.fragment),J=s(),c(y.$$.fragment),B=s(),b=i("p"),b.innerHTML=j,C=s(),c(o.$$.fragment),w=s(),F=i("p"),F.innerHTML=ut,je=s(),x=i("p"),x.textContent=Mt,Je=s(),k=i("ul"),k.innerHTML=yt,$e=s(),W=i("p"),W.innerHTML=gt,ve=s(),c(R.$$.fragment),_e=s(),A=i("p"),A.innerHTML=bt,Ze=s(),E=i("p"),E.innerHTML=wt,Be=s(),c(X.$$.fragment),Ce=s(),P=i("p"),P.innerHTML=Tt,Fe=s(),c(G.$$.fragment),xe=s(),c(H.$$.fragment),ke=s(),c(N.$$.fragment),We=s(),c(v.$$.fragment),Re=s(),S=i("p"),S.innerHTML=Ut,Ae=s(),I=i("p"),I.innerHTML=jt,Ee=s(),Q=i("ul"),Q.innerHTML=Jt,Xe=s(),Y=i("p"),Y.innerHTML=$t,Pe=s(),c(z.$$.fragment),Ge=s(),V=i("p"),V.innerHTML=vt,He=s(),L=i("p"),L.innerHTML=_t,Ne=s(),c(q.$$.fragment),Se=s(),c(O.$$.fragment),Ie=s(),D=i("p"),D.innerHTML=Zt,Qe=s(),K=i("p"),K.textContent=Bt,Ye=s(),ee=i("p"),ee.innerHTML=Ct,ze=s(),c(te.$$.fragment),Ve=s(),ae=i("p"),ae.innerHTML=Ft,Le=s(),le=i("p"),le.innerHTML=xt,qe=s(),c(se.$$.fragment),Oe=s(),c(ne.$$.fragment),De=s(),ie=i("p"),ie.innerHTML=kt,Ke=s(),re=i("blockquote"),re.innerHTML=Wt,et=s(),c(pe.$$.fragment),tt=s(),oe=i("p"),oe.innerHTML=Rt,at=s(),ce=i("p"),ce.innerHTML=At,lt=s(),c(me.$$.fragment),st=s(),he=i("p"),he.innerHTML=Et,nt=s(),c(de.$$.fragment),it=s(),fe=i("p"),fe.textContent=Xt,rt=s(),ue=i("ul"),ue.innerHTML=Pt,pt=s(),Me=i("p"),Me.innerHTML=Gt,ot=s(),c(ye.$$.fragment),ct=s(),ge=i("p"),ge.textContent=Ht,mt=s(),be=i("ul"),be.innerHTML=Nt,ht=s(),c(we.$$.fragment),dt=s(),Te=i("p"),this.h()},l(e){const t=Ot("svelte-u9bgzb",document.head);M=r(t,"META",{name:!0,content:!0}),t.forEach(a),Z=n(e),T=r(e,"P",{}),It(T).forEach(a),g=n(e),m(U.$$.fragment,e),J=n(e),m(y.$$.fragment,e),B=n(e),b=r(e,"P",{"data-svelte-h":!0}),p(b)!=="svelte-bw9le8"&&(b.innerHTML=j),C=n(e),m(o.$$.fragment,e),w=n(e),F=r(e,"P",{"data-svelte-h":!0}),p(F)!=="svelte-wuhpuo"&&(F.innerHTML=ut),je=n(e),x=r(e,"P",{"data-svelte-h":!0}),p(x)!=="svelte-10cwb11"&&(x.textContent=Mt),Je=n(e),k=r(e,"UL",{"data-svelte-h":!0}),p(k)!=="svelte-5d1df8"&&(k.innerHTML=yt),$e=n(e),W=r(e,"P",{"data-svelte-h":!0}),p(W)!=="svelte-149dy0g"&&(W.innerHTML=gt),ve=n(e),m(R.$$.fragment,e),_e=n(e),A=r(e,"P",{"data-svelte-h":!0}),p(A)!=="svelte-mfpbqx"&&(A.innerHTML=bt),Ze=n(e),E=r(e,"P",{"data-svelte-h":!0}),p(E)!=="svelte-311lpy"&&(E.innerHTML=wt),Be=n(e),m(X.$$.fragment,e),Ce=n(e),P=r(e,"P",{"data-svelte-h":!0}),p(P)!=="svelte-1jfw9hc"&&(P.innerHTML=Tt),Fe=n(e),m(G.$$.fragment,e),xe=n(e),m(H.$$.fragment,e),ke=n(e),m(N.$$.fragment,e),We=n(e),m(v.$$.fragment,e),Re=n(e),S=r(e,"P",{"data-svelte-h":!0}),p(S)!=="svelte-1mf0zic"&&(S.innerHTML=Ut),Ae=n(e),I=r(e,"P",{"data-svelte-h":!0}),p(I)!=="svelte-11bftkh"&&(I.innerHTML=jt),Ee=n(e),Q=r(e,"UL",{"data-svelte-h":!0}),p(Q)!=="svelte-ha185h"&&(Q.innerHTML=Jt),Xe=n(e),Y=r(e,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-wx6vs8"&&(Y.innerHTML=$t),Pe=n(e),m(z.$$.fragment,e),Ge=n(e),V=r(e,"P",{"data-svelte-h":!0}),p(V)!=="svelte-vo4vij"&&(V.innerHTML=vt),He=n(e),L=r(e,"P",{"data-svelte-h":!0}),p(L)!=="svelte-1n0fir7"&&(L.innerHTML=_t),Ne=n(e),m(q.$$.fragment,e),Se=n(e),m(O.$$.fragment,e),Ie=n(e),D=r(e,"P",{"data-svelte-h":!0}),p(D)!=="svelte-1tlihxp"&&(D.innerHTML=Zt),Qe=n(e),K=r(e,"P",{"data-svelte-h":!0}),p(K)!=="svelte-wvn4fr"&&(K.textContent=Bt),Ye=n(e),ee=r(e,"P",{"data-svelte-h":!0}),p(ee)!=="svelte-8khoko"&&(ee.innerHTML=Ct),ze=n(e),m(te.$$.fragment,e),Ve=n(e),ae=r(e,"P",{"data-svelte-h":!0}),p(ae)!=="svelte-99qxte"&&(ae.innerHTML=Ft),Le=n(e),le=r(e,"P",{"data-svelte-h":!0}),p(le)!=="svelte-1n0fir7"&&(le.innerHTML=xt),qe=n(e),m(se.$$.fragment,e),Oe=n(e),m(ne.$$.fragment,e),De=n(e),ie=r(e,"P",{"data-svelte-h":!0}),p(ie)!=="svelte-13u4x7b"&&(ie.innerHTML=kt),Ke=n(e),re=r(e,"BLOCKQUOTE",{"data-svelte-h":!0}),p(re)!=="svelte-rg03zf"&&(re.innerHTML=Wt),et=n(e),m(pe.$$.fragment,e),tt=n(e),oe=r(e,"P",{"data-svelte-h":!0}),p(oe)!=="svelte-1qvuvpw"&&(oe.innerHTML=Rt),at=n(e),ce=r(e,"P",{"data-svelte-h":!0}),p(ce)!=="svelte-12gtxdw"&&(ce.innerHTML=At),lt=n(e),m(me.$$.fragment,e),st=n(e),he=r(e,"P",{"data-svelte-h":!0}),p(he)!=="svelte-1qqyqot"&&(he.innerHTML=Et),nt=n(e),m(de.$$.fragment,e),it=n(e),fe=r(e,"P",{"data-svelte-h":!0}),p(fe)!=="svelte-1ay0trc"&&(fe.textContent=Xt),rt=n(e),ue=r(e,"UL",{"data-svelte-h":!0}),p(ue)!=="svelte-1affbo7"&&(ue.innerHTML=Pt),pt=n(e),Me=r(e,"P",{"data-svelte-h":!0}),p(Me)!=="svelte-sau342"&&(Me.innerHTML=Gt),ot=n(e),m(ye.$$.fragment,e),ct=n(e),ge=r(e,"P",{"data-svelte-h":!0}),p(ge)!=="svelte-t5s4ol"&&(ge.textContent=Ht),mt=n(e),be=r(e,"UL",{"data-svelte-h":!0}),p(be)!=="svelte-1yn0lu3"&&(be.innerHTML=Nt),ht=n(e),m(we.$$.fragment,e),dt=n(e),Te=r(e,"P",{}),It(Te).forEach(a),this.h()},h(){Qt(M,"name","hf:doc:metadata"),Qt(M,"content",sa)},m(e,t){Dt(document.head,M),l(e,Z,t),l(e,T,t),l(e,g,t),h(U,e,t),l(e,J,t),h(y,e,t),l(e,B,t),l(e,b,t),l(e,C,t),h(o,e,t),l(e,w,t),l(e,F,t),l(e,je,t),l(e,x,t),l(e,Je,t),l(e,k,t),l(e,$e,t),l(e,W,t),l(e,ve,t),h(R,e,t),l(e,_e,t),l(e,A,t),l(e,Ze,t),l(e,E,t),l(e,Be,t),h(X,e,t),l(e,Ce,t),l(e,P,t),l(e,Fe,t),h(G,e,t),l(e,xe,t),h(H,e,t),l(e,ke,t),h(N,e,t),l(e,We,t),h(v,e,t),l(e,Re,t),l(e,S,t),l(e,Ae,t),l(e,I,t),l(e,Ee,t),l(e,Q,t),l(e,Xe,t),l(e,Y,t),l(e,Pe,t),h(z,e,t),l(e,Ge,t),l(e,V,t),l(e,He,t),l(e,L,t),l(e,Ne,t),h(q,e,t),l(e,Se,t),h(O,e,t),l(e,Ie,t),l(e,D,t),l(e,Qe,t),l(e,K,t),l(e,Ye,t),l(e,ee,t),l(e,ze,t),h(te,e,t),l(e,Ve,t),l(e,ae,t),l(e,Le,t),l(e,le,t),l(e,qe,t),h(se,e,t),l(e,Oe,t),h(ne,e,t),l(e,De,t),l(e,ie,t),l(e,Ke,t),l(e,re,t),l(e,et,t),h(pe,e,t),l(e,tt,t),l(e,oe,t),l(e,at,t),l(e,ce,t),l(e,lt,t),h(me,e,t),l(e,st,t),l(e,he,t),l(e,nt,t),h(de,e,t),l(e,it,t),l(e,fe,t),l(e,rt,t),l(e,ue,t),l(e,pt,t),l(e,Me,t),l(e,ot,t),h(ye,e,t),l(e,ct,t),l(e,ge,t),l(e,mt,t),l(e,be,t),l(e,ht,t),h(we,e,t),l(e,dt,t),l(e,Te,t),ft=!0},p(e,[t]){const St={};t&2&&(St.$$scope={dirty:t,ctx:e}),v.$set(St)},i(e){ft||(d(U.$$.fragment,e),d(y.$$.fragment,e),d(o.$$.fragment,e),d(R.$$.fragment,e),d(X.$$.fragment,e),d(G.$$.fragment,e),d(H.$$.fragment,e),d(N.$$.fragment,e),d(v.$$.fragment,e),d(z.$$.fragment,e),d(q.$$.fragment,e),d(O.$$.fragment,e),d(te.$$.fragment,e),d(se.$$.fragment,e),d(ne.$$.fragment,e),d(pe.$$.fragment,e),d(me.$$.fragment,e),d(de.$$.fragment,e),d(ye.$$.fragment,e),d(we.$$.fragment,e),ft=!0)},o(e){f(U.$$.fragment,e),f(y.$$.fragment,e),f(o.$$.fragment,e),f(R.$$.fragment,e),f(X.$$.fragment,e),f(G.$$.fragment,e),f(H.$$.fragment,e),f(N.$$.fragment,e),f(v.$$.fragment,e),f(z.$$.fragment,e),f(q.$$.fragment,e),f(O.$$.fragment,e),f(te.$$.fragment,e),f(se.$$.fragment,e),f(ne.$$.fragment,e),f(pe.$$.fragment,e),f(me.$$.fragment,e),f(de.$$.fragment,e),f(ye.$$.fragment,e),f(we.$$.fragment,e),ft=!1},d(e){e&&(a(Z),a(T),a(g),a(J),a(B),a(b),a(C),a(w),a(F),a(je),a(x),a(Je),a(k),a($e),a(W),a(ve),a(_e),a(A),a(Ze),a(E),a(Be),a(Ce),a(P),a(Fe),a(xe),a(ke),a(We),a(Re),a(S),a(Ae),a(I),a(Ee),a(Q),a(Xe),a(Y),a(Pe),a(Ge),a(V),a(He),a(L),a(Ne),a(Se),a(Ie),a(D),a(Qe),a(K),a(Ye),a(ee),a(ze),a(Ve),a(ae),a(Le),a(le),a(qe),a(Oe),a(De),a(ie),a(Ke),a(re),a(et),a(tt),a(oe),a(at),a(ce),a(lt),a(st),a(he),a(nt),a(it),a(fe),a(rt),a(ue),a(pt),a(Me),a(ot),a(ct),a(ge),a(mt),a(be),a(ht),a(dt),a(Te)),a(M),u(U,e),u(y,e),u(o,e),u(R,e),u(X,e),u(G,e),u(H,e),u(N,e),u(v,e),u(z,e),u(q,e),u(O,e),u(te,e),u(se,e),u(ne,e),u(pe,e),u(me,e),u(de,e),u(ye,e),u(we,e)}}}const sa='{"title":"Low Precision Training Methods","local":"low-precision-training-methods","sections":[{"title":"What training on FP8 means","local":"what-training-on-fp8-means","sections":[],"depth":2},{"title":"Configuring the Accelerator","local":"configuring-the-accelerator","sections":[],"depth":2},{"title":"Configuring MS-AMP","local":"configuring-ms-amp","sections":[],"depth":2},{"title":"Configuring TransformersEngine","local":"configuring-transformersengine","sections":[],"depth":2},{"title":"Configuring torchao","local":"configuring-torchao","sections":[],"depth":2},{"title":"Example Zoo","local":"example-zoo","sections":[],"depth":2},{"title":"Further Reading","local":"further-reading","sections":[],"depth":2}],"depth":1}';function na(Ue){return zt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ma extends Lt{constructor(M){super(),qt(this,M,na,la,Yt,{})}}export{ma as component};