accelerate/pr_4046/en/_app/immutable/nodes/40.cfd0dfae.js

import{s as _e,n as xe,o as ke}from"../chunks/scheduler.b9285784.js";import{S as Ze,i as Re,e as a,s as o,c,h as Ve,a as s,d as l,b as n,f as Be,g as m,j as r,k as We,l as Pe,m as i,n as p,t as d,o as u,p as h}from"../chunks/index.26bc89a1.js";import{C as ze,H as z,E as Le}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e7fb8fb8.js";import{C as Ge}from"../chunks/CodeBlock.bc8424c4.js";function qe(ge){let f,Y,L,E,g,F,y,A,M,S,T,ye="Pytorch 2.0 introduced <code>torch.compile</code>, a powerful feature that makes PyTorch code run faster by JIT-compiling PyTorch code into optimized kernels. Key features of <code>torch.compile</code> include:",X,w,Me="<li><strong>Performance Improvement</strong>: Significantly speeds up model execution by optimizing the computation graph.</li> <li><strong>Ease of Use</strong>: Requires minimal code changes to implement, making it highly accessible.</li> <li><strong>Compatibility</strong>: Works seamlessly with existing PyTorch code and models.</li>",Q,b,Te="When used with Accelerate, <code>torch.compile</code> integrates smoothly into distributed training workflows, allowing you to benefit from both distributed execution and compilation optimizations simultaneously.",N,$,we="The first execution of compiled code typically takes longer as it includes the compilation time, but subsequent runs are significantly faster. For optimal performance in different scenarios, <code>torch.compile</code> offers various modes like <code>&quot;default&quot;</code>, <code>&quot;reduce-overhead&quot;</code> (which uses CUDA graphs to further reduce overhead), and <code>&quot;max-autotune&quot;</code> (which performs extensive autotuning to find the best kernels for your model).",D,J,O,v,be="Accelerate provides <code>TorchDynamoPlugin</code> for easy and seamless integration of <code>torch.compile</code> into your training scripts.",K,j,ee,U,$e="It is compatible with all other features and plugins of Accelerate, including mixed precision, distributed training (DDP, FSDP, Deepspeed), etc.",te,C,le,I,Je="Instead of trying to compile the whole model, which usually has a big problem space for optimization. Regional compilation targets repeated blocks of the same class and compiles them sequentially to hit the compiler’s cache. For example, in <code>GPT2LMHeadModel</code>, the repeated block/class is <code>GPT2Block</code>, and can be accessed as <code>model.transformer.h[0]</code>. The rest of the model (e.g model.lm_head) is compiled separately.",ie,H,ve=`This allows us to speed up the compilation overhead / cold start of models like LLMs and Transformers in general.
See <a href="https://pytorch.org/tutorials/recipes/regional_compilation.html" rel="nofollow">https://pytorch.org/tutorials/recipes/regional_compilation.html</a> for more details.`,oe,B,ne,W,je="It can be enabled by setting <code>use_regional_compilation=True</code> in the <code>TorchDynamoPlugin</code> configuration:",ae,G,se,_,Ue="You could also use the <code>accelerate.utils.compile_regions</code> utility directly the same way you would use <code>torch.compile</code>.",re,x,ce,k,Ce='We have conducted extensive benchmarks comparing full compilation and regional compilation using the <code>torch.compile</code> feature in PyTorch. The full results are available in the <a href="https://github.com/huggingface/accelerate/tree/main/benchmarks/torch.compile/regional_compilation" rel="nofollow">accelerate repository</a>. The key findings from our benchmarks are:',me,Z,Ie="<li><strong>Comparable Performance</strong>: Regional compilation delivers performance speedups similar to full compilation, especially for larger models.</li> <li><strong>Faster Compilation</strong>: Regional compilation significantly reduces the time taken to compile models, making it a more efficient choice for deployment.</li> <li><strong>Batch Size Impact</strong>: The performance difference between compilation strategies diminishes with larger batch sizes, indicating that the overhead of compilation is less impactful in those scenarios.</li> <li><strong>Model Size Consideration</strong>: The benefits of regional compilation are more pronounced in larger models, where the compilation time savings can be substantial.</li> <li><strong>Practical Application</strong>: For real-world applications, regional compilation is a practical choice for optimizing training cold start times, especially when working with large models.</li>",pe,R,de,V,He="Both full and regional compilation can significantly speed up your models. Regional compilation offers a practical balance between compilation time and runtime performance, especially for training large models with substantial batch sizes.",ue,P,he,q,fe;return g=new ze({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),y=new z({props:{title:"Compilation",local:"compilation",headingTag:"h1"}}),M=new z({props:{title:"Overview",local:"overview",headingTag:"h2"}}),J=new z({props:{title:"Using torch.compile with Accelerate",local:"using-torchcompile-with-accelerate",headingTag:"h2"}}),j=new Ge({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnV0aWxzJTIwaW1wb3J0JTIwVG9yY2hEeW5hbW9QbHVnaW4lMEElMEElMjMlMjBDb25maWd1cmUlMjB0aGUlMjBjb21waWxhdGlvbiUyMGJhY2tlbmQlMEFkeW5hbW9fcGx1Z2luJTIwJTNEJTIwVG9yY2hEeW5hbW9QbHVnaW4oJTBBJTIwJTIwJTIwJTIwYmFja2VuZCUzRCUyMmluZHVjdG9yJTIyJTJDJTIwJTIwJTIzJTIwT3B0aW9ucyUzQSUyMCUyMmluZHVjdG9yJTIyJTJDJTIwJTIyYW90X2VhZ2VyJTIyJTJDJTIwJTIyYW90X252ZnVzZXIlMjIlMkMlMjBldGMuJTBBJTIwJTIwJTIwJTIwbW9kZSUzRCUyMmRlZmF1bHQlMjIlMkMlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBPcHRpb25zJTNBJTIwJTIyZGVmYXVsdCUyMiUyQyUyMCUyMnJlZHVjZS1vdmVyaGVhZCUyMiUyQyUyMCUyMm1heC1hdXRvdHVuZSUyMiUwQSUyMCUyMCUyMCUyMGZ1bGxncmFwaCUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjBkeW5hbWljJTNERmFsc2UlMEEpJTBBJTBBJTIzJTIwSW5pdGlhbGl6ZSUyMGFjY2VsZXJhdG9yJTIwd2l0aCUyMHRoZSUyMHBsdWdpbiUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZHluYW1vX3BsdWdpbiUzRGR5bmFtb19wbHVnaW4pJTBBJTIzJTIwVGhpcyUyMHdpbGwlMjBhcHBseSUyMHRvcmNoLmNvbXBpbGUlMjB0byUyMHlvdXIlMjBtb2RlbCUwQW1vZGVsJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZShtb2RlbCk=",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator
<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> TorchDynamoPlugin

<span class="hljs-comment"># Configure the compilation backend</span>
dynamo_plugin = TorchDynamoPlugin(
    backend=<span class="hljs-string">&quot;inductor&quot;</span>,  <span class="hljs-comment"># Options: &quot;inductor&quot;, &quot;aot_eager&quot;, &quot;aot_nvfuser&quot;, etc.</span>
    mode=<span class="hljs-string">&quot;default&quot;</span>,      <span class="hljs-comment"># Options: &quot;default&quot;, &quot;reduce-overhead&quot;, &quot;max-autotune&quot;</span>
    fullgraph=<span class="hljs-literal">True</span>,
    dynamic=<span class="hljs-literal">False</span>
)

<span class="hljs-comment"># Initialize accelerator with the plugin</span>
accelerator = Accelerator(dynamo_plugin=dynamo_plugin)
<span class="hljs-comment"># This will apply torch.compile to your model</span>
model = accelerator.prepare(model)`,lang:"python",wrap:!1}}),C=new z({props:{title:"Regional Compilation",local:"regional-compilation",headingTag:"h2"}}),B=new z({props:{title:"How to Use Regional Compilation",local:"how-to-use-regional-compilation",headingTag:"h3"}}),G=new Ge({props:{code:"JTIzJTIwQ29uZmlndXJlJTIwdGhlJTIwY29tcGlsYXRpb24lMjBiYWNrZW5kJTBBZHluYW1vX3BsdWdpbiUyMCUzRCUyMFRvcmNoRHluYW1vUGx1Z2luKCUwQSUyMCUyMCUyMCUyMHVzZV9yZWdpb25hbF9jb21waWxhdGlvbiUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjAuLi4lMjAlMjMlMjBvdGhlciUyMHBhcmFtZXRlcnMlMEEpJTBBJTIzJTIwSW5pdGlhbGl6ZSUyMGFjY2VsZXJhdG9yJTIwd2l0aCUyMHRoZSUyMHBsdWdpbiUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZHluYW1vX3BsdWdpbiUzRGR5bmFtb19wbHVnaW4pJTBBJTIzJTIwVGhpcyUyMHdpbGwlMjBhcHBseSUyMGNvbXBpbGVfcmVnaW9ucyUyMHRvJTIweW91ciUyMG1vZGVsJTBBbW9kZWwlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKG1vZGVsKQ==",highlighted:`<span class="hljs-comment"># Configure the compilation backend</span>
dynamo_plugin = TorchDynamoPlugin(
    use_regional_compilation=<span class="hljs-literal">True</span>,
    ... <span class="hljs-comment"># other parameters</span>
)
<span class="hljs-comment"># Initialize accelerator with the plugin</span>
accelerator = Accelerator(dynamo_plugin=dynamo_plugin)
<span class="hljs-comment"># This will apply compile_regions to your model</span>
model = accelerator.prepare(model)`,lang:"python",wrap:!1}}),x=new z({props:{title:"Benefits of Regional Compilation",local:"benefits-of-regional-compilation",headingTag:"h3"}}),R=new z({props:{title:"Conclusion",local:"conclusion",headingTag:"h2"}}),P=new Le({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/compilation.md"}}),{c(){f=a("meta"),Y=o(),L=a("p"),E=o(),c(g.$$.fragment),F=o(),c(y.$$.fragment),A=o(),c(M.$$.fragment),S=o(),T=a("p"),T.innerHTML=ye,X=o(),w=a("ul"),w.innerHTML=Me,Q=o(),b=a("p"),b.innerHTML=Te,N=o(),$=a("p"),$.innerHTML=we,D=o(),c(J.$$.fragment),O=o(),v=a("p"),v.innerHTML=be,K=o(),c(j.$$.fragment),ee=o(),U=a("p"),U.textContent=$e,te=o(),c(C.$$.fragment),le=o(),I=a("p"),I.innerHTML=Je,ie=o(),H=a("p"),H.innerHTML=ve,oe=o(),c(B.$$.fragment),ne=o(),W=a("p"),W.innerHTML=je,ae=o(),c(G.$$.fragment),se=o(),_=a("p"),_.innerHTML=Ue,re=o(),c(x.$$.fragment),ce=o(),k=a("p"),k.innerHTML=Ce,me=o(),Z=a("ol"),Z.innerHTML=Ie,pe=o(),c(R.$$.fragment),de=o(),V=a("p"),V.textContent=He,ue=o(),c(P.$$.fragment),he=o(),q=a("p"),this.h()},l(e){const t=Ve("svelte-u9bgzb",document.head);f=s(t,"META",{name:!0,content:!0}),t.forEach(l),Y=n(e),L=s(e,"P",{}),Be(L).forEach(l),E=n(e),m(g.$$.fragment,e),F=n(e),m(y.$$.fragment,e),A=n(e),m(M.$$.fragment,e),S=n(e),T=s(e,"P",{"data-svelte-h":!0}),r(T)!=="svelte-az9dn0"&&(T.innerHTML=ye),X=n(e),w=s(e,"UL",{"data-svelte-h":!0}),r(w)!=="svelte-pkkxsd"&&(w.innerHTML=Me),Q=n(e),b=s(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-gd5iye"&&(b.innerHTML=Te),N=n(e),$=s(e,"P",{"data-svelte-h":!0}),r($)!=="svelte-1qamxj"&&($.innerHTML=we),D=n(e),m(J.$$.fragment,e),O=n(e),v=s(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-4740v4"&&(v.innerHTML=be),K=n(e),m(j.$$.fragment,e),ee=n(e),U=s(e,"P",{"data-svelte-h":!0}),r(U)!=="svelte-5uylqs"&&(U.textContent=$e),te=n(e),m(C.$$.fragment,e),le=n(e),I=s(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-1fkldr7"&&(I.innerHTML=Je),ie=n(e),H=s(e,"P",{"data-svelte-h":!0}),r(H)!=="svelte-cs154s"&&(H.innerHTML=ve),oe=n(e),m(B.$$.fragment,e),ne=n(e),W=s(e,"P",{"data-svelte-h":!0}),r(W)!=="svelte-50g4qf"&&(W.innerHTML=je),ae=n(e),m(G.$$.fragment,e),se=n(e),_=s(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-j2ilao"&&(_.innerHTML=Ue),re=n(e),m(x.$$.fragment,e),ce=n(e),k=s(e,"P",{"data-svelte-h":!0}),r(k)!=="svelte-w1r301"&&(k.innerHTML=Ce),me=n(e),Z=s(e,"OL",{"data-svelte-h":!0}),r(Z)!=="svelte-qsfo8r"&&(Z.innerHTML=Ie),pe=n(e),m(R.$$.fragment,e),de=n(e),V=s(e,"P",{"data-svelte-h":!0}),r(V)!=="svelte-1iudzer"&&(V.textContent=He),ue=n(e),m(P.$$.fragment,e),he=n(e),q=s(e,"P",{}),Be(q).forEach(l),this.h()},h(){We(f,"name","hf:doc:metadata"),We(f,"content",Ye)},m(e,t){Pe(document.head,f),i(e,Y,t),i(e,L,t),i(e,E,t),p(g,e,t),i(e,F,t),p(y,e,t),i(e,A,t),p(M,e,t),i(e,S,t),i(e,T,t),i(e,X,t),i(e,w,t),i(e,Q,t),i(e,b,t),i(e,N,t),i(e,$,t),i(e,D,t),p(J,e,t),i(e,O,t),i(e,v,t),i(e,K,t),p(j,e,t),i(e,ee,t),i(e,U,t),i(e,te,t),p(C,e,t),i(e,le,t),i(e,I,t),i(e,ie,t),i(e,H,t),i(e,oe,t),p(B,e,t),i(e,ne,t),i(e,W,t),i(e,ae,t),p(G,e,t),i(e,se,t),i(e,_,t),i(e,re,t),p(x,e,t),i(e,ce,t),i(e,k,t),i(e,me,t),i(e,Z,t),i(e,pe,t),p(R,e,t),i(e,de,t),i(e,V,t),i(e,ue,t),p(P,e,t),i(e,he,t),i(e,q,t),fe=!0},p:xe,i(e){fe||(d(g.$$.fragment,e),d(y.$$.fragment,e),d(M.$$.fragment,e),d(J.$$.fragment,e),d(j.$$.fragment,e),d(C.$$.fragment,e),d(B.$$.fragment,e),d(G.$$.fragment,e),d(x.$$.fragment,e),d(R.$$.fragment,e),d(P.$$.fragment,e),fe=!0)},o(e){u(g.$$.fragment,e),u(y.$$.fragment,e),u(M.$$.fragment,e),u(J.$$.fragment,e),u(j.$$.fragment,e),u(C.$$.fragment,e),u(B.$$.fragment,e),u(G.$$.fragment,e),u(x.$$.fragment,e),u(R.$$.fragment,e),u(P.$$.fragment,e),fe=!1},d(e){e&&(l(Y),l(L),l(E),l(F),l(A),l(S),l(T),l(X),l(w),l(Q),l(b),l(N),l($),l(D),l(O),l(v),l(K),l(ee),l(U),l(te),l(le),l(I),l(ie),l(H),l(oe),l(ne),l(W),l(ae),l(se),l(_),l(re),l(ce),l(k),l(me),l(Z),l(pe),l(de),l(V),l(ue),l(he),l(q)),l(f),h(g,e),h(y,e),h(M,e),h(J,e),h(j,e),h(C,e),h(B,e),h(G,e),h(x,e),h(R,e),h(P,e)}}}const Ye='{"title":"Compilation","local":"compilation","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Using torch.compile with Accelerate","local":"using-torchcompile-with-accelerate","sections":[],"depth":2},{"title":"Regional Compilation","local":"regional-compilation","sections":[{"title":"How to Use Regional Compilation","local":"how-to-use-regional-compilation","sections":[],"depth":3},{"title":"Benefits of Regional Compilation","local":"benefits-of-regional-compilation","sections":[],"depth":3}],"depth":2},{"title":"Conclusion","local":"conclusion","sections":[],"depth":2}],"depth":1}';function Ee(ge){return ke(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Qe extends Ze{constructor(f){super(),Re(this,f,Ee,qe,_e,{})}}export{Qe as component};