accelerate/pr_4064/en/_app/immutable/nodes/13.fbc345cd.js

import{s as it,n as at,o as nt}from"../chunks/scheduler.b9285784.js";import{S as dt,i as lt,e as n,s as i,c as r,h as rt,a as d,d as s,b as a,f as st,g as c,j as l,k as J,l as ct,m as o,n as p,t as f,o as h,p as m}from"../chunks/index.26bc89a1.js";import{C as pt,H as g,E as ft}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.d17575b1.js";import{C as ot}from"../chunks/CodeBlock.2228289f.js";function ht(Ae){let u,K,X,ee,_,te,P,se,T,Ie="This guide explains the key differences between <code>FSDP1</code> and <code>FSDP2</code> and helps you migrate your existing code to use <code>FSDP2</code> with minimal changes.",oe,b,ie,D,Ue="First, we want to understand how <code>FSDP1</code> and <code>FSDP2</code> work internally to understand the differences between them. This also helps us understand the limitations of <code>FSDP1</code> and how <code>FSDP2</code> solves them.",ae,S,je="We’ll be discussing a scenario where we have a single <code>Layer</code> that contains 3 <code>Linear</code> layers and is wrapped using <code>FSDP</code> to be sharded across 2 GPUs.",ne,w,Ze='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/accelerate/layer.png" alt="Layer"/>',de,M,le,F,Ge="First, we have to understand the original <code>FSDP1</code> and the limitations it brings. It represents each <code>FSDP</code> module as a single <code>FlatParameter</code> which is a single 1D tensor that contains all of the module parameters, which then get sharded across ranks. I.e. if you wrap the <code>Layer</code> with <code>FSDP1</code>, you’d achieve something as such:",re,y,Re='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/accelerate/fsdp1.png" alt="FSDP1"/>',ce,L,We="You might notice a problem. The whole <code>Layer</code> gets flattened into a single <code>FlatParameter</code>, which then gets sharded across ranks. But if it’s a single <code>FlatParameter</code> object, how do we store metadata? That is one of the limitations. Properly storing per-parameter metadata such as <code>dtype</code>, <code>requires_grad</code>, etc. is not possible without some ugly hacks.",pe,H,fe,x,Be='This is why <code>FSDP2</code> was introduced. It doesn’t use <code>FlatParameter</code>, instead it uses <code>DTensor</code> which is short for “Distributed Tensor”. Each <code>DTensor</code> basically represents a vanilla <code>torch.Tensor</code> that has been sharded across ranks. It contains metadata about the original <code>torch.Tensor</code> and how it’s sharded, what is the <a href="https://pytorch.org/docs/stable/distributed.tensor.html#module-torch.distributed.tensor.placement_types" rel="nofollow">placement type</a> and so on. This is why it’s called <code>per-parameter sharding</code>. The following figure shows the difference:',he,v,Ve='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/accelerate/fsdp2.png" alt="FSDP2"/>',me,C,Ye="Each Parameter of the original <code>Layer</code> is sharded across the 0th dimension, and split between 2 GPUs. Now, each <code>Linear</code> layer is a separate <code>DTensor</code> and storing metadata per-parameter is possible and straightforward.",ue,$,ze="<p>In the image above, the tensors were sharded across the 1st dimension for the sake of fitting the image on the screen, in reality, they are sharded across the 0th dimension as stated above</p>",ge,k,we,E,Ne="<code>FSDP2</code> is a new and improved version of PyTorch’s fully-sharded data parallel training API. Its main advantage is using <code>DTensor</code> to represent sharded parameters. Compared to <code>FSDP1</code>, it offers:",ye,A,Oe='<li>Simpler internal implementation, where each <code>Parameter</code> is a separate <code>DTensor</code></li> <li>Enables simple partial parameter freezing because of the above, which makes methods as <a href="https://huggingface.co/papers/2106.09685" rel="nofollow"><code>LORA</code></a> work out of the box</li> <li>With <code>DTensor</code>, <code>FSDP2</code> supports mixing <code>fp8</code> and other parameter types in the same model out of the box</li> <li>Faster and simpler checkpointing without extra communication across ranks using <code>SHARDED_STATE_DICT</code> and <a href="https://pytorch.org/docs/stable/distributed.checkpoint.html" rel="nofollow"><code>torch.distributed.checkpoint</code></a>, this way, each rank only saves its own shard and corresponding metadata</li> <li>For loading, it uses a <code>state_dict</code> of the sharded model to directly load the sharded parameters</li> <li>Support for asynchronous checkpointing, where parameters are first copied to CPU memory, after this, main thread continues training while another thread stores the parameters on disk</li> <li>Memory efficiency and deterministic memory usage, <code>FSDP2</code> doesn’t use <code>recordStream</code> anymore and uses stream-to-stream synchronization (for more technical details see <a href="https://dev-discuss.pytorch.org/t/fsdp-cudacachingallocator-an-outsider-newb-perspective/1486" rel="nofollow">this forum post</a> and <a href="https://github.com/pytorch/pytorch/issues/114299" rel="nofollow">this issue</a>)</li> <li>In the future, optimizations of the communication patterns via <code>torch.compile</code> are planned, further improving the performance and memory efficiency</li>',ve,I,$e,U,qe="We have already discussed the internal differences, now let’s discuss the differences, you, as a user, will need to know.",_e,j,Je="Here are the main changes in configuration options when using <code>FSDP2</code> through the <code>accelerate</code> CLI:",Pe,Z,Xe="<thead><tr><th>Previous (<code>FSDP1</code>)</th> <th>New (<code>FSDP2</code>)</th> <th>What Changed</th></tr></thead> <tbody><tr><td><code>--fsdp_sharding_strategy</code></td> <td><code>--fsdp_reshard_after_forward</code></td> <td>replaces <code>--fsdp_sharding_strategy</code>, changed to <code>true</code> (previously <code>FULL_SHARD</code>) or <code>false</code> (previously <code>SHARD_GRAD_OP</code>)</td></tr> <tr><td><code>--fsdp_backward_prefetch</code></td> <td>**<strong>REMOVED</strong>**</td> <td><code>FSDP2</code> uses previous <code>BACKWARD_PRE</code> option by default, as only this allows communication and computation overlap</td></tr> <tr><td><code>--fsdp_forward_prefetch</code></td> <td>**<strong>NOT YET IMPLEMENTED</strong>**</td> <td>How to implement this is under active discussion, for now it is not supported in <code>FSDP2</code></td></tr> <tr><td><code>--fsdp_sync_module_states</code></td> <td>**<strong>REMOVED</strong>**</td> <td>with <code>FSDP2</code>, this parameter becomes redundant</td></tr> <tr><td><code>--fsdp_cpu_ram_efficient_loading</code></td> <td><code>--fsdp_cpu_ram_efficient_loading</code></td> <td>if <code>true</code>, <code>FSDP2</code> will similarly load the model only on rank 0, and then parameters get synced to other ranks, this is the same behavior as <code>FSDP1</code>, however, setting <code>--fsdp_sync_module_states</code> isn’t required anymore</td></tr> <tr><td><code>--fsdp_state_dict_type</code></td> <td><code>--fsdp_state_dict_type</code></td> <td><code>LOCAL_STATE_DICT</code> becomes obsolete and with <code>FSDP2</code> <code>SHARDED_STATE_DICT</code> is the default option, which results in no extra communication and each rank saving its own shard, other possible option is <code>FULL_STATE_DICT</code> which results in extra communication and spike in memory usage but saves the full model from rank 0.</td></tr> <tr><td><code>--fsdp_use_orig_params</code></td> <td>**<strong>REMOVED</strong>**</td> <td><code>FSDP2</code> uses a <code>DTensor</code> class on the background, which means it <em>always</em> uses the original parameters by default</td></tr> <tr><td>**<strong>NEW</strong>**</td> <td><code>--fsdp_version</code></td> <td><code>1</code> is the default option, to not break existing code, set to <code>2</code> to use <code>FSDP2</code></td></tr></tbody>",Te,G,Qe='For all other options that remain unchanged, see the <a href="../usage_guides/fsdp"><code>FSDP</code> documentation</a>.',be,R,De,W,Se,B,Ke="Simply set <code>fsdp_version=2</code> when creating your plugin and replace options according to the table above.",Me,V,Fe,Y,Le,z,et="Use our conversion tool:",He,N,xe,O,tt="This will automatically convert all FSDP1 settings to their FSDP2 equivalents. Use <code>--overwrite</code> to update the existing file instead of creating a new one.",Ce,q,ke,Q,Ee;return _=new pt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),P=new g({props:{title:"FSDP1 vs FSDP2",local:"fsdp1-vs-fsdp2",headingTag:"h1"}}),b=new g({props:{title:"How is FSDP2 better than FSDP1?",local:"how-is-fsdp2-better-than-fsdp1",headingTag:"h2"}}),M=new g({props:{title:"FSDP1",local:"fsdp1",headingTag:"h3"}}),H=new g({props:{title:"FSDP2",local:"fsdp2",headingTag:"h3"}}),k=new g({props:{title:"What does FSDP2 offer?",local:"what-does-fsdp2-offer",headingTag:"h2"}}),I=new g({props:{title:"API Differences",local:"api-differences",headingTag:"h2"}}),R=new g({props:{title:"How to Switch to FSDP2",local:"how-to-switch-to-fsdp2",headingTag:"h2"}}),W=new g({props:{title:"If using Python code:",local:"if-using-python-code",headingTag:"h3"}}),V=new ot({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBGdWxseVNoYXJkZWREYXRhUGFyYWxsZWxQbHVnaW4lMkMlMjBBY2NlbGVyYXRvciUwQSUwQWZzZHBfcGx1Z2luJTIwJTNEJTIwRnVsbHlTaGFyZGVkRGF0YVBhcmFsbGVsUGx1Z2luKCUwQSUyMCUyMCUyMCUyMGZzZHBfdmVyc2lvbiUzRDIlMEElMjAlMjAlMjAlMjAlMjMlMjBvdGhlciUyMG9wdGlvbnMuLi4lMEEpJTBBYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihmc2RwX3BsdWdpbiUzRGZzZHBfcGx1Z2luKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> FullyShardedDataParallelPlugin, Accelerator

fsdp_plugin = FullyShardedDataParallelPlugin(
    fsdp_version=<span class="hljs-number">2</span>
    <span class="hljs-comment"># other options...</span>
)
accelerator = Accelerator(fsdp_plugin=fsdp_plugin)`,lang:"python",wrap:!1}}),Y=new g({props:{title:"If using YAML config:",local:"if-using-yaml-config",headingTag:"h3"}}),N=new ot({props:{code:"YWNjZWxlcmF0ZSUyMHRvLWZzZHAyJTIwLS1jb25maWdfZmlsZSUyMGNvbmZpZy55YW1sJTIwLS1vdXRwdXRfZmlsZSUyMG5ld19jb25maWcueWFtbA==",highlighted:"accelerate to-fsdp2 --config_file config.yaml --output_file new_config.yaml",lang:"bash",wrap:!1}}),q=new ft({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/concept_guides/fsdp1_vs_fsdp2.md"}}),{c(){u=n("meta"),K=i(),X=n("p"),ee=i(),r(_.$$.fragment),te=i(),r(P.$$.fragment),se=i(),T=n("p"),T.innerHTML=Ie,oe=i(),r(b.$$.fragment),ie=i(),D=n("p"),D.innerHTML=Ue,ae=i(),S=n("p"),S.innerHTML=je,ne=i(),w=n("div"),w.innerHTML=Ze,de=i(),r(M.$$.fragment),le=i(),F=n("p"),F.innerHTML=Ge,re=i(),y=n("div"),y.innerHTML=Re,ce=i(),L=n("p"),L.innerHTML=We,pe=i(),r(H.$$.fragment),fe=i(),x=n("p"),x.innerHTML=Be,he=i(),v=n("div"),v.innerHTML=Ve,me=i(),C=n("p"),C.innerHTML=Ye,ue=i(),$=n("blockquote"),$.innerHTML=ze,ge=i(),r(k.$$.fragment),we=i(),E=n("p"),E.innerHTML=Ne,ye=i(),A=n("ul"),A.innerHTML=Oe,ve=i(),r(I.$$.fragment),$e=i(),U=n("p"),U.textContent=qe,_e=i(),j=n("p"),j.innerHTML=Je,Pe=i(),Z=n("table"),Z.innerHTML=Xe,Te=i(),G=n("p"),G.innerHTML=Qe,be=i(),r(R.$$.fragment),De=i(),r(W.$$.fragment),Se=i(),B=n("p"),B.innerHTML=Ke,Me=i(),r(V.$$.fragment),Fe=i(),r(Y.$$.fragment),Le=i(),z=n("p"),z.textContent=et,He=i(),r(N.$$.fragment),xe=i(),O=n("p"),O.innerHTML=tt,Ce=i(),r(q.$$.fragment),ke=i(),Q=n("p"),this.h()},l(e){const t=rt("svelte-u9bgzb",document.head);u=d(t,"META",{name:!0,content:!0}),t.forEach(s),K=a(e),X=d(e,"P",{}),st(X).forEach(s),ee=a(e),c(_.$$.fragment,e),te=a(e),c(P.$$.fragment,e),se=a(e),T=d(e,"P",{"data-svelte-h":!0}),l(T)!=="svelte-j1458a"&&(T.innerHTML=Ie),oe=a(e),c(b.$$.fragment,e),ie=a(e),D=d(e,"P",{"data-svelte-h":!0}),l(D)!=="svelte-i773ia"&&(D.innerHTML=Ue),ae=a(e),S=d(e,"P",{"data-svelte-h":!0}),l(S)!=="svelte-64broj"&&(S.innerHTML=je),ne=a(e),w=d(e,"DIV",{align:!0,"data-svelte-h":!0}),l(w)!=="svelte-1ksxz48"&&(w.innerHTML=Ze),de=a(e),c(M.$$.fragment,e),le=a(e),F=d(e,"P",{"data-svelte-h":!0}),l(F)!=="svelte-jq6gpq"&&(F.innerHTML=Ge),re=a(e),y=d(e,"DIV",{align:!0,"data-svelte-h":!0}),l(y)!=="svelte-1b5af3k"&&(y.innerHTML=Re),ce=a(e),L=d(e,"P",{"data-svelte-h":!0}),l(L)!=="svelte-uuw76g"&&(L.innerHTML=We),pe=a(e),c(H.$$.fragment,e),fe=a(e),x=d(e,"P",{"data-svelte-h":!0}),l(x)!=="svelte-metqvu"&&(x.innerHTML=Be),he=a(e),v=d(e,"DIV",{align:!0,"data-svelte-h":!0}),l(v)!=="svelte-reftl0"&&(v.innerHTML=Ve),me=a(e),C=d(e,"P",{"data-svelte-h":!0}),l(C)!=="svelte-14923cw"&&(C.innerHTML=Ye),ue=a(e),$=d(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),l($)!=="svelte-1cc0a20"&&($.innerHTML=ze),ge=a(e),c(k.$$.fragment,e),we=a(e),E=d(e,"P",{"data-svelte-h":!0}),l(E)!=="svelte-pi572c"&&(E.innerHTML=Ne),ye=a(e),A=d(e,"UL",{"data-svelte-h":!0}),l(A)!=="svelte-ot1awy"&&(A.innerHTML=Oe),ve=a(e),c(I.$$.fragment,e),$e=a(e),U=d(e,"P",{"data-svelte-h":!0}),l(U)!=="svelte-1ckeirw"&&(U.textContent=qe),_e=a(e),j=d(e,"P",{"data-svelte-h":!0}),l(j)!=="svelte-ovpb8y"&&(j.innerHTML=Je),Pe=a(e),Z=d(e,"TABLE",{"data-svelte-h":!0}),l(Z)!=="svelte-c2d485"&&(Z.innerHTML=Xe),Te=a(e),G=d(e,"P",{"data-svelte-h":!0}),l(G)!=="svelte-13cj0f3"&&(G.innerHTML=Qe),be=a(e),c(R.$$.fragment,e),De=a(e),c(W.$$.fragment,e),Se=a(e),B=d(e,"P",{"data-svelte-h":!0}),l(B)!=="svelte-1bc4t35"&&(B.innerHTML=Ke),Me=a(e),c(V.$$.fragment,e),Fe=a(e),c(Y.$$.fragment,e),Le=a(e),z=d(e,"P",{"data-svelte-h":!0}),l(z)!=="svelte-t6vwot"&&(z.textContent=et),He=a(e),c(N.$$.fragment,e),xe=a(e),O=d(e,"P",{"data-svelte-h":!0}),l(O)!=="svelte-nil0yf"&&(O.innerHTML=tt),Ce=a(e),c(q.$$.fragment,e),ke=a(e),Q=d(e,"P",{}),st(Q).forEach(s),this.h()},h(){J(u,"name","hf:doc:metadata"),J(u,"content",mt),J(w,"align","center"),J(y,"align","center"),J(v,"align","center"),J($,"class","tip")},m(e,t){ct(document.head,u),o(e,K,t),o(e,X,t),o(e,ee,t),p(_,e,t),o(e,te,t),p(P,e,t),o(e,se,t),o(e,T,t),o(e,oe,t),p(b,e,t),o(e,ie,t),o(e,D,t),o(e,ae,t),o(e,S,t),o(e,ne,t),o(e,w,t),o(e,de,t),p(M,e,t),o(e,le,t),o(e,F,t),o(e,re,t),o(e,y,t),o(e,ce,t),o(e,L,t),o(e,pe,t),p(H,e,t),o(e,fe,t),o(e,x,t),o(e,he,t),o(e,v,t),o(e,me,t),o(e,C,t),o(e,ue,t),o(e,$,t),o(e,ge,t),p(k,e,t),o(e,we,t),o(e,E,t),o(e,ye,t),o(e,A,t),o(e,ve,t),p(I,e,t),o(e,$e,t),o(e,U,t),o(e,_e,t),o(e,j,t),o(e,Pe,t),o(e,Z,t),o(e,Te,t),o(e,G,t),o(e,be,t),p(R,e,t),o(e,De,t),p(W,e,t),o(e,Se,t),o(e,B,t),o(e,Me,t),p(V,e,t),o(e,Fe,t),p(Y,e,t),o(e,Le,t),o(e,z,t),o(e,He,t),p(N,e,t),o(e,xe,t),o(e,O,t),o(e,Ce,t),p(q,e,t),o(e,ke,t),o(e,Q,t),Ee=!0},p:at,i(e){Ee||(f(_.$$.fragment,e),f(P.$$.fragment,e),f(b.$$.fragment,e),f(M.$$.fragment,e),f(H.$$.fragment,e),f(k.$$.fragment,e),f(I.$$.fragment,e),f(R.$$.fragment,e),f(W.$$.fragment,e),f(V.$$.fragment,e),f(Y.$$.fragment,e),f(N.$$.fragment,e),f(q.$$.fragment,e),Ee=!0)},o(e){h(_.$$.fragment,e),h(P.$$.fragment,e),h(b.$$.fragment,e),h(M.$$.fragment,e),h(H.$$.fragment,e),h(k.$$.fragment,e),h(I.$$.fragment,e),h(R.$$.fragment,e),h(W.$$.fragment,e),h(V.$$.fragment,e),h(Y.$$.fragment,e),h(N.$$.fragment,e),h(q.$$.fragment,e),Ee=!1},d(e){e&&(s(K),s(X),s(ee),s(te),s(se),s(T),s(oe),s(ie),s(D),s(ae),s(S),s(ne),s(w),s(de),s(le),s(F),s(re),s(y),s(ce),s(L),s(pe),s(fe),s(x),s(he),s(v),s(me),s(C),s(ue),s($),s(ge),s(we),s(E),s(ye),s(A),s(ve),s($e),s(U),s(_e),s(j),s(Pe),s(Z),s(Te),s(G),s(be),s(De),s(Se),s(B),s(Me),s(Fe),s(Le),s(z),s(He),s(xe),s(O),s(Ce),s(ke),s(Q)),s(u),m(_,e),m(P,e),m(b,e),m(M,e),m(H,e),m(k,e),m(I,e),m(R,e),m(W,e),m(V,e),m(Y,e),m(N,e),m(q,e)}}}const mt='{"title":"FSDP1 vs FSDP2","local":"fsdp1-vs-fsdp2","sections":[{"title":"How is FSDP2 better than FSDP1?","local":"how-is-fsdp2-better-than-fsdp1","sections":[{"title":"FSDP1","local":"fsdp1","sections":[],"depth":3},{"title":"FSDP2","local":"fsdp2","sections":[],"depth":3}],"depth":2},{"title":"What does FSDP2 offer?","local":"what-does-fsdp2-offer","sections":[],"depth":2},{"title":"API Differences","local":"api-differences","sections":[],"depth":2},{"title":"How to Switch to FSDP2","local":"how-to-switch-to-fsdp2","sections":[{"title":"If using Python code:","local":"if-using-python-code","sections":[],"depth":3},{"title":"If using YAML config:","local":"if-using-yaml-config","sections":[],"depth":3}],"depth":2}],"depth":1}';function ut(Ae){return nt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $t extends dt{constructor(u){super(),lt(this,u,ut,ht,it,{})}}export{$t as component};