accelerate/pr_4039/en/_app/immutable/nodes/43.7141cc4e.js

import{s as Ft,o as Et,n as St}from"../chunks/scheduler.b9285784.js";import{S as Lt,i as Dt,e as p,s as a,c as u,h as Pt,a as i,d as l,b as n,f as Ht,g as r,j as c,k as xt,l as Kt,m as s,n as M,t as d,o as J,p as y}from"../chunks/index.26bc89a1.js";import{T as Qt}from"../chunks/Tip.e4eba3d6.js";import{C as Ot,H as Te,E as el}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.91c9ff84.js";import{C as w}from"../chunks/CodeBlock.ef23fd93.js";function tl(we){let o,m='This guide assumes that you have read and understood the <a href="./deepspeed">DeepSpeed usage guide</a>.';return{c(){o=p("p"),o.innerHTML=m},l(T){o=i(T,"P",{"data-svelte-h":!0}),c(o)!=="svelte-1geb81d"&&(o.innerHTML=m)},m(T,h){s(T,o,h)},p:St,d(T){T&&l(o)}}}function ll(we){let o,m="DeepSpeed will raise an error if <code>train_micro_batch_size_per_gpu</code> isn’t specified, even if this particular model isn’t being trained.";return{c(){o=p("p"),o.innerHTML=m},l(T){o=i(T,"P",{"data-svelte-h":!0}),c(o)!=="svelte-qz5bxi"&&(o.innerHTML=m)},m(T,h){s(T,o,h)},p:St,d(T){T&&l(o)}}}function sl(we){let o,m,T,h,I,he,U,je,j,fe,g,Tt="Running multiple models with Accelerate and DeepSpeed is useful for:",Ie,b,wt='<li>Knowledge distillation</li> <li>Post-training techniques like RLHF (see the <a href="https://github.com/huggingface/trl" rel="nofollow">TRL</a> library for more examples)</li> <li>Training multiple models at once</li>',Ue,C,mt="Currently, Accelerate has a <strong>very experimental API</strong> to help you use multiple models.",ge,_,ht="This tutorial will focus on two common use cases:",be,$,jt="<li>Knowledge distillation, where a smaller student model is trained to mimic a larger, better-performing teacher.  If the student model fits on a single GPU, we can use ZeRO-2 for training and ZeRO-3 to shard the teacher for inference. This is significantly faster than using ZeRO-3 for both models.</li> <li>Training multiple <em>disjoint</em> models at once.</li>",Ce,Z,_e,B,ft="Knowledge distillation is a good example of using multiple models, but only training one of them.",$e,W,It='Normally, you would use a single <a href="/docs/accelerate/pr_4039/en/package_reference/deepspeed#accelerate.DeepSpeedPlugin">utils.DeepSpeedPlugin</a> for both models. However, in this case, there are two separate configurations. Accelerate allows you to create and use multiple plugins <strong>if and only if</strong> they are in a <code>dict</code> so that you can reference and enable the proper plugin when needed.',Ze,X,Be,G,Ut="The <code>zero2_config.json</code> should be configured for full training (so specify <code>scheduler</code> and <code>optimizer</code> if you are not utilizing your own), while <code>zero3_config.json</code> should only be configured for the inference model, as shown in the example below.",We,R,Xe,v,gt="An example <code>zero2_config.json</code> configuration is shown below.",Ge,V,Re,f,ve,z,bt='From here, create a single <a href="/docs/accelerate/pr_4039/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> and pass in both configurations.',Ve,Y,ze,q,Ct="Now let’s see how to use them.",Ye,A,qe,k,_t='By default, Accelerate sets the first item in the <code>dict</code> as the default or enabled plugin (<code>&quot;student&quot;</code> plugin). Verify this by using the <a href="/docs/accelerate/pr_4039/en/package_reference/deepspeed#accelerate.utils.get_active_deepspeed_plugin">utils.deepspeed.get_active_deepspeed_plugin()</a> function to see which plugin is enabled.',Ae,N,ke,H,$t="<code>AcceleratorState</code> also keeps the active DeepSpeed plugin saved in <code>state.deepspeed_plugin</code>.",Ne,x,He,Q,Zt="Since <code>student</code> is the currently active plugin, let’s go ahead and prepare the model, optimizer, and scheduler.",xe,S,Qe,F,Bt="Now it’s time to deal with the teacher model.",Se,E,Fe,L,Wt='First, you need to specify in <a href="/docs/accelerate/pr_4039/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> that the <code>zero3_config.json</code> configuration should be used.',Ee,D,Le,P,Xt=`This disables the <code>&quot;student&quot;</code> plugin and enables the <code>&quot;teacher&quot;</code> plugin instead. The
DeepSpeed stateful config inside of Transformers is updated, and it changes which plugin configuration gets called when using
<code>deepspeed.initialize()</code>. This allows you to use the automatic <code>deepspeed.zero.Init</code>  context manager integration Transformers provides.`,De,K,Pe,O,Gt="Otherwise, you should manually initialize the model with <code>deepspeed.zero.Init</code>.",Ke,ee,Oe,te,et,le,Rt="From here, your training loop can be whatever you like, as long as <code>teacher_model</code> is never being trained on.",tt,se,lt,ae,st,ne,vt=`Training multiple models is a more complicated scenario.
In its current state, we assume each model is <strong>completely disjointed</strong> from the other during training.`,at,pe,Vt='This scenario still requires two <a href="/docs/accelerate/pr_4039/en/package_reference/deepspeed#accelerate.DeepSpeedPlugin">utils.DeepSpeedPlugin</a>’s to be made. However, you also need a second <a href="/docs/accelerate/pr_4039/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a>, since different <code>deepspeed</code> engines are being called at different times. A single <a href="/docs/accelerate/pr_4039/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> can only carry one instance at a time.',nt,ie,zt='Since the <a href="/docs/accelerate/pr_4039/en/package_reference/state#accelerate.state.AcceleratorState">state.AcceleratorState</a> is a stateful object though, it is already aware of both <a href="/docs/accelerate/pr_4039/en/package_reference/deepspeed#accelerate.DeepSpeedPlugin">utils.DeepSpeedPlugin</a>’s available. You can just instantiate a second <a href="/docs/accelerate/pr_4039/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> with no extra arguments.',pt,ce,it,oe,Yt=`You can call either <code>first_accelerator.state.select_deepspeed_plugin()</code> to enable or disable
a particular plugin, and then call <code>prepare</code>.`,ct,ue,ot,re,qt="And now you can train:",ut,Me,rt,de,Mt,Je,At='To see more examples, please check out the <a href="https://github.com/huggingface/accelerate/blob/main/src/accelerate/test_utils/scripts/external_deps/test_ds_multiple_model.py" rel="nofollow">related tests</a> currently in [Accelerate].',dt,ye,Jt,me,yt;return I=new Ot({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new Te({props:{title:"Using multiple models with DeepSpeed",local:"using-multiple-models-with-deepspeed",headingTag:"h1"}}),j=new Qt({props:{warning:!0,$$slots:{default:[tl]},$$scope:{ctx:we}}}),Z=new Te({props:{title:"Knowledge distillation",local:"knowledge-distillation",headingTag:"h2"}}),X=new w({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBEZWVwU3BlZWRQbHVnaW4lMEElMEF6ZXJvMl9wbHVnaW4lMjAlM0QlMjBEZWVwU3BlZWRQbHVnaW4oaGZfZHNfY29uZmlnJTNEJTIyemVybzJfY29uZmlnLmpzb24lMjIpJTBBemVybzNfcGx1Z2luJTIwJTNEJTIwRGVlcFNwZWVkUGx1Z2luKGhmX2RzX2NvbmZpZyUzRCUyMnplcm8zX2NvbmZpZy5qc29uJTIyKSUwQSUwQWRlZXBzcGVlZF9wbHVnaW5zJTIwJTNEJTIwJTdCJTIyc3R1ZGVudCUyMiUzQSUyMHplcm8yX3BsdWdpbiUyQyUyMCUyMnRlYWNoZXIlMjIlM0ElMjB6ZXJvM19wbHVnaW4lN0Q=",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> DeepSpeedPlugin

zero2_plugin = DeepSpeedPlugin(hf_ds_config=<span class="hljs-string">&quot;zero2_config.json&quot;</span>)
zero3_plugin = DeepSpeedPlugin(hf_ds_config=<span class="hljs-string">&quot;zero3_config.json&quot;</span>)

deepspeed_plugins = {<span class="hljs-string">&quot;student&quot;</span>: zero2_plugin, <span class="hljs-string">&quot;teacher&quot;</span>: zero3_plugin}`,lang:"python",wrap:!1}}),R=new w({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyYmYxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19vcHRpbWl6YXRpb24lMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZSUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvdmVybGFwX2NvbW0lMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycmVkdWNlX2J1Y2tldF9zaXplJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19wcmVmZXRjaF9idWNrZXRfc2l6ZSUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZTNfcGFyYW1fcGVyc2lzdGVuY2VfdGhyZXNob2xkJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19tYXhfbGl2ZV9wYXJhbWV0ZXJzJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19tYXhfcmV1c2VfZGlzdGFuY2UlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JTIyJTNBJTIwMSUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;bf16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">3</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;overlap_comm&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;reduce_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_prefetch_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_param_persistence_threshold&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_max_live_parameters&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_max_reuse_distance&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span>
<span class="hljs-punctuation">}</span>`,lang:"json",wrap:!1}}),V=new w({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyYmYxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyb3B0aW1pemVyJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydHlwZSUyMiUzQSUyMCUyMkFkYW1XJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGFyYW1zJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybHIlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyd2VpZ2h0X2RlY2F5JTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRvcmNoX2FkYW0lMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWRhbV93X21vZGUlMjIlM0ElMjB0cnVlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyc2NoZWR1bGVyJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydHlwZSUyMiUzQSUyMCUyMldhcm11cExSJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGFyYW1zJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyd2FybXVwX21pbl9sciUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ3YXJtdXBfbWF4X2xyJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMndhcm11cF9udW1fc3RlcHMlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19vcHRpbWl6YXRpb24lMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZSUyMiUzQSUyMDIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvZmZsb2FkX29wdGltaXplciUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmRldmljZSUyMiUzQSUyMCUyMmNwdSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBpbl9tZW1vcnklMjIlM0ElMjB0cnVlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIyJTNBJTIwMSUyQyUwQSUyMCUyMCUyMCUyMCUyMmdyYWRpZW50X2NsaXBwaW5nJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYWluX2JhdGNoX3NpemUlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;bf16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;AdamW&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;weight_decay&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;torch_adam&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;adam_w_mode&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;scheduler&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;WarmupLR&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;warmup_min_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_max_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_num_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;offload_optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;device&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;cpu&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;pin_memory&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
        <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_accumulation_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_clipping&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_batch_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
<span class="hljs-punctuation">}</span>`,lang:"json",wrap:!1}}),f=new Qt({props:{$$slots:{default:[ll]},$$scope:{ctx:we}}}),Y=new w({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZGVlcHNwZWVkX3BsdWdpbnMlM0RkZWVwc3BlZWRfcGx1Z2lucyk=",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator(deepspeed_plugins=deepspeed_plugins)`,lang:"python",wrap:!1}}),A=new Te({props:{title:"Student model",local:"student-model",headingTag:"h3"}}),N=new w({props:{code:"YWN0aXZlX3BsdWdpbiUyMCUzRCUyMGdldF9hY3RpdmVfZGVlcHNwZWVkX3BsdWdpbihhY2NlbGVyYXRvci5zdGF0ZSklMEFhc3NlcnQlMjBhY3RpdmVfcGx1Z2luJTIwaXMlMjBkZWVwc3BlZWRfcGx1Z2lucyU1QiUyMnN0dWRlbnQlMjIlNUQ=",highlighted:`active_plugin = get_active_deepspeed_plugin(accelerator.state)
<span class="hljs-keyword">assert</span> active_plugin <span class="hljs-keyword">is</span> deepspeed_plugins[<span class="hljs-string">&quot;student&quot;</span>]`,lang:"python",wrap:!1}}),x=new w({props:{code:"YXNzZXJ0JTIwYWN0aXZlX3BsdWdpbiUyMGlzJTIwYWNjZWxlcmF0b3IuZGVlcHNwZWVkX3BsdWdpbg==",highlighted:'<span class="hljs-keyword">assert</span> active_plugin <span class="hljs-keyword">is</span> accelerator.deepspeed_plugin',lang:"python",wrap:!1}}),S=new w({props:{code:"c3R1ZGVudF9tb2RlbCUyQyUyMG9wdGltaXplciUyQyUyMHNjaGVkdWxlciUyMCUzRCUyMC4uLiUwQXN0dWRlbnRfbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjBzY2hlZHVsZXIlMkMlMjB0cmFpbl9kYXRhbG9hZGVyJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZShzdHVkZW50X21vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwc2NoZWR1bGVyJTJDJTIwdHJhaW5fZGF0YWxvYWRlcik=",highlighted:`student_model, optimizer, scheduler = ...
student_model, optimizer, scheduler, train_dataloader = accelerator.prepare(student_model, optimizer, scheduler, train_dataloader)`,lang:"python",wrap:!1}}),E=new Te({props:{title:"Teacher model",local:"teacher-model",headingTag:"h3"}}),D=new w({props:{code:"YWNjZWxlcmF0b3Iuc3RhdGUuc2VsZWN0X2RlZXBzcGVlZF9wbHVnaW4oJTIydGVhY2hlciUyMik=",highlighted:'accelerator.state.select_deepspeed_plugin(<span class="hljs-string">&quot;teacher&quot;</span>)',lang:"python",wrap:!1}}),K=new w({props:{code:"dGVhY2hlcl9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoLi4uKSUwQXRlYWNoZXJfbW9kZWwlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKHRlYWNoZXJfbW9kZWwp",highlighted:`teacher_model = AutoModel.from_pretrained(...)
teacher_model = accelerator.prepare(teacher_model)`,lang:"python",wrap:!1}}),ee=new w({props:{code:"d2l0aCUyMGRlZXBzcGVlZC56ZXJvLkluaXQoYWNjZWxlcmF0b3IuZGVlcHNwZWVkX3BsdWdpbi5jb25maWcpJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwlMjAlM0QlMjBNeU1vZGVsKC4uLik=",highlighted:`<span class="hljs-keyword">with</span> deepspeed.zero.Init(accelerator.deepspeed_plugin.config):
    model = MyModel(...)`,lang:"python",wrap:!1}}),te=new Te({props:{title:"Training",local:"training",headingTag:"h3"}}),se=new w({props:{code:"dGVhY2hlcl9tb2RlbC5ldmFsKCklMEFzdHVkZW50X21vZGVsLnRyYWluKCklMEFmb3IlMjBiYXRjaCUyMGluJTIwdHJhaW5fZGF0YWxvYWRlciUzQSUwQSUyMCUyMCUyMCUyMHdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRfdGVhY2hlciUyMCUzRCUyMHRlYWNoZXJfbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjBvdXRwdXRfc3R1ZGVudCUyMCUzRCUyMHN0dWRlbnRfbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjAlMjMlMjBDb21iaW5lJTIwdGhlJTIwbG9zc2VzJTIwb3IlMjBtb2RpZnklMjBpdCUyMGluJTIwc29tZSUyMHdheSUwQSUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBvdXRwdXRfdGVhY2hlci5sb3NzJTIwJTJCJTIwb3V0cHV0X3N0dWRlbnQubG9zcyUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMHNjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCk=",highlighted:`teacher_model.<span class="hljs-built_in">eval</span>()
student_model.train()
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
    <span class="hljs-keyword">with</span> torch.no_grad():
        output_teacher = teacher_model(**batch)
    output_student = student_model(**batch)
    <span class="hljs-comment"># Combine the losses or modify it in some way</span>
    loss = output_teacher.loss + output_student.loss
    accelerator.backward(loss)
    optimizer.step()
    scheduler.step()
    optimizer.zero_grad()`,lang:"python",wrap:!1}}),ae=new Te({props:{title:"Train multiple disjoint models",local:"train-multiple-disjoint-models",headingTag:"h2"}}),ce=new w({props:{code:"Zmlyc3RfYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihkZWVwc3BlZWRfcGx1Z2lucyUzRGRlZXBzcGVlZF9wbHVnaW5zKSUwQXNlY29uZF9hY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKCk=",highlighted:`first_accelerator = Accelerator(deepspeed_plugins=deepspeed_plugins)
second_accelerator = Accelerator()`,lang:"python",wrap:!1}}),ue=new w({props:{code:"JTIzJTIwY2FuJTIwYmUlMjAlNjBhY2NlbGVyYXRvcl8wJTYwJTJDJTIwJTYwYWNjZWxlcmF0b3JfMSU2MCUyQyUyMG9yJTIwYnklMjBjYWxsaW5nJTIwJTYwQWNjZWxlcmF0b3JTdGF0ZSgpLnNlbGVjdF9kZWVwc3BlZWRfcGx1Z2luKC4uLiklNjAlMEFmaXJzdF9hY2NlbGVyYXRvci5zdGF0ZS5zZWxlY3RfZGVlcHNwZWVkX3BsdWdpbiglMjJmaXJzdF9tb2RlbCUyMiklMEFmaXJzdF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoLi4uKSUwQSUyMyUyMEZvciUyMHRoaXMlMjBleGFtcGxlJTJDJTIwJTYwZ2V0X3RyYWluaW5nX2l0ZW1zJTYwJTIwaXMlMjBhJTIwbm9uZXhpc3RlbnQlMjBmdW5jdGlvbiUyMHRoYXQlMjBnZXRzJTIwdGhlJTIwc2V0dXAlMjB3ZSUyMG5lZWQlMjBmb3IlMjB0cmFpbmluZyUwQWZpcnN0X29wdGltaXplciUyQyUyMGZpcnN0X3NjaGVkdWxlciUyQyUyMHRyYWluX2RsJTJDJTIwZXZhbF9kbCUyMCUzRCUyMGdldF90cmFpbmluZ19pdGVtcyhtb2RlbDEpJTBBZmlyc3RfbW9kZWwlMkMlMjBmaXJzdF9vcHRpbWl6ZXIlMkMlMjBmaXJzdF9zY2hlZHVsZXIlMkMlMjB0cmFpbl9kbCUyQyUyMGV2YWxfZGwlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKCUwQSUyMCUyMCUyMCUyMGZpcnN0X21vZGVsJTJDJTIwZmlyc3Rfb3B0aW1pemVyJTJDJTIwZmlyc3Rfc2NoZWR1bGVyJTJDJTIwdHJhaW5fZGwlMkMlMjBldmFsX2RsJTBBKSUwQSUwQXNlY29uZF9hY2NlbGVyYXRvci5zdGF0ZS5zZWxlY3RfZGVlcHNwZWVkX3BsdWdpbiglMjJzZWNvbmRfbW9kZWwlMjIpJTBBc2Vjb25kX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCguLi4pJTBBJTIzJTIwRm9yJTIwdGhpcyUyMGV4YW1wbGUlMkMlMjAlNjBnZXRfdHJhaW5pbmdfaXRlbXMlNjAlMjBpcyUyMGElMjBub25leGlzdGVudCUyMGZ1bmN0aW9uJTIwdGhhdCUyMGdldHMlMjB0aGUlMjBzZXR1cCUyMHdlJTIwbmVlZCUyMGZvciUyMHRyYWluaW5nJTBBc2Vjb25kX29wdGltaXplciUyQyUyMHNlY29uZF9zY2hlZHVsZXIlMkMlMjBfJTJDJTIwXyUyMCUzRCUyMGdldF90cmFpbmluZ19pdGVtcyhtb2RlbDIpJTBBc2Vjb25kX21vZGVsJTJDJTIwc2Vjb25kX29wdGltaXplciUyQyUyMHNlY29uZF9zY2hlZHVsZXIlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKCUwQSUyMCUyMCUyMCUyMHNlY29uZF9tb2RlbCUyQyUyMHNlY29uZF9vcHRpbWl6ZXIlMkMlMjBzZWNvbmRfc2NoZWR1bGVyJTBBKQ==",highlighted:`<span class="hljs-comment"># can be \`accelerator_0\`, \`accelerator_1\`, or by calling \`AcceleratorState().select_deepspeed_plugin(...)\`</span>
first_accelerator.state.select_deepspeed_plugin(<span class="hljs-string">&quot;first_model&quot;</span>)
first_model = AutoModel.from_pretrained(...)
<span class="hljs-comment"># For this example, \`get_training_items\` is a nonexistent function that gets the setup we need for training</span>
first_optimizer, first_scheduler, train_dl, eval_dl = get_training_items(model1)
first_model, first_optimizer, first_scheduler, train_dl, eval_dl = accelerator.prepare(
    first_model, first_optimizer, first_scheduler, train_dl, eval_dl
)

second_accelerator.state.select_deepspeed_plugin(<span class="hljs-string">&quot;second_model&quot;</span>)
second_model = AutoModel.from_pretrained(...)
<span class="hljs-comment"># For this example, \`get_training_items\` is a nonexistent function that gets the setup we need for training</span>
second_optimizer, second_scheduler, _, _ = get_training_items(model2)
second_model, second_optimizer, second_scheduler = accelerator.prepare(
    second_model, second_optimizer, second_scheduler
)`,lang:"python",wrap:!1}}),Me=new w({props:{code:"Zm9yJTIwYmF0Y2glMjBpbiUyMGRsJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0czElMjAlM0QlMjBmaXJzdF9tb2RlbCgqKmJhdGNoKSUwQSUyMCUyMCUyMCUyMGZpcnN0X2FjY2VsZXJhdG9yLmJhY2t3YXJkKG91dHB1dHMxLmxvc3MpJTBBJTIwJTIwJTIwJTIwZmlyc3Rfb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMGZpcnN0X3NjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjBmaXJzdF9vcHRpbWl6ZXIuemVyb19ncmFkKCklMEElMjAlMjAlMjAlMjAlMEElMjAlMjAlMjAlMjBvdXRwdXRzMiUyMCUzRCUyMG1vZGVsMigqKmJhdGNoKSUwQSUyMCUyMCUyMCUyMHNlY29uZF9hY2NlbGVyYXRvci5iYWNrd2FyZChvdXRwdXRzMi5sb3NzKSUwQSUyMCUyMCUyMCUyMHNlY29uZF9vcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwc2Vjb25kX3NjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjBzZWNvbmRfb3B0aW1pemVyLnplcm9fZ3JhZCgp",highlighted:`<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> dl:
    outputs1 = first_model(**batch)
    first_accelerator.backward(outputs1.loss)
    first_optimizer.step()
    first_scheduler.step()
    first_optimizer.zero_grad()
    
    outputs2 = model2(**batch)
    second_accelerator.backward(outputs2.loss)
    second_optimizer.step()
    second_scheduler.step()
    second_optimizer.zero_grad()`,lang:"python",wrap:!1}}),de=new Te({props:{title:"Resources",local:"resources",headingTag:"h2"}}),ye=new el({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/deepspeed_multiple_model.md"}}),{c(){o=p("meta"),m=a(),T=p("p"),h=a(),u(I.$$.fragment),he=a(),u(U.$$.fragment),je=a(),u(j.$$.fragment),fe=a(),g=p("p"),g.textContent=Tt,Ie=a(),b=p("ul"),b.innerHTML=wt,Ue=a(),C=p("p"),C.innerHTML=mt,ge=a(),_=p("p"),_.textContent=ht,be=a(),$=p("ol"),$.innerHTML=jt,Ce=a(),u(Z.$$.fragment),_e=a(),B=p("p"),B.textContent=ft,$e=a(),W=p("p"),W.innerHTML=It,Ze=a(),u(X.$$.fragment),Be=a(),G=p("p"),G.innerHTML=Ut,We=a(),u(R.$$.fragment),Xe=a(),v=p("p"),v.innerHTML=gt,Ge=a(),u(V.$$.fragment),Re=a(),u(f.$$.fragment),ve=a(),z=p("p"),z.innerHTML=bt,Ve=a(),u(Y.$$.fragment),ze=a(),q=p("p"),q.textContent=Ct,Ye=a(),u(A.$$.fragment),qe=a(),k=p("p"),k.innerHTML=_t,Ae=a(),u(N.$$.fragment),ke=a(),H=p("p"),H.innerHTML=$t,Ne=a(),u(x.$$.fragment),He=a(),Q=p("p"),Q.innerHTML=Zt,xe=a(),u(S.$$.fragment),Qe=a(),F=p("p"),F.textContent=Bt,Se=a(),u(E.$$.fragment),Fe=a(),L=p("p"),L.innerHTML=Wt,Ee=a(),u(D.$$.fragment),Le=a(),P=p("p"),P.innerHTML=Xt,De=a(),u(K.$$.fragment),Pe=a(),O=p("p"),O.innerHTML=Gt,Ke=a(),u(ee.$$.fragment),Oe=a(),u(te.$$.fragment),et=a(),le=p("p"),le.innerHTML=Rt,tt=a(),u(se.$$.fragment),lt=a(),u(ae.$$.fragment),st=a(),ne=p("p"),ne.innerHTML=vt,at=a(),pe=p("p"),pe.innerHTML=Vt,nt=a(),ie=p("p"),ie.innerHTML=zt,pt=a(),u(ce.$$.fragment),it=a(),oe=p("p"),oe.innerHTML=Yt,ct=a(),u(ue.$$.fragment),ot=a(),re=p("p"),re.textContent=qt,ut=a(),u(Me.$$.fragment),rt=a(),u(de.$$.fragment),Mt=a(),Je=p("p"),Je.innerHTML=At,dt=a(),u(ye.$$.fragment),Jt=a(),me=p("p"),this.h()},l(e){const t=Pt("svelte-u9bgzb",document.head);o=i(t,"META",{name:!0,content:!0}),t.forEach(l),m=n(e),T=i(e,"P",{}),Ht(T).forEach(l),h=n(e),r(I.$$.fragment,e),he=n(e),r(U.$$.fragment,e),je=n(e),r(j.$$.fragment,e),fe=n(e),g=i(e,"P",{"data-svelte-h":!0}),c(g)!=="svelte-1b1gxyz"&&(g.textContent=Tt),Ie=n(e),b=i(e,"UL",{"data-svelte-h":!0}),c(b)!=="svelte-7mvbd"&&(b.innerHTML=wt),Ue=n(e),C=i(e,"P",{"data-svelte-h":!0}),c(C)!=="svelte-e8xs1z"&&(C.innerHTML=mt),ge=n(e),_=i(e,"P",{"data-svelte-h":!0}),c(_)!=="svelte-owdl16"&&(_.textContent=ht),be=n(e),$=i(e,"OL",{"data-svelte-h":!0}),c($)!=="svelte-to0mxc"&&($.innerHTML=jt),Ce=n(e),r(Z.$$.fragment,e),_e=n(e),B=i(e,"P",{"data-svelte-h":!0}),c(B)!=="svelte-zc35yl"&&(B.textContent=ft),$e=n(e),W=i(e,"P",{"data-svelte-h":!0}),c(W)!=="svelte-8pag9q"&&(W.innerHTML=It),Ze=n(e),r(X.$$.fragment,e),Be=n(e),G=i(e,"P",{"data-svelte-h":!0}),c(G)!=="svelte-4l2obz"&&(G.innerHTML=Ut),We=n(e),r(R.$$.fragment,e),Xe=n(e),v=i(e,"P",{"data-svelte-h":!0}),c(v)!=="svelte-2zuo2r"&&(v.innerHTML=gt),Ge=n(e),r(V.$$.fragment,e),Re=n(e),r(f.$$.fragment,e),ve=n(e),z=i(e,"P",{"data-svelte-h":!0}),c(z)!=="svelte-kzhh4q"&&(z.innerHTML=bt),Ve=n(e),r(Y.$$.fragment,e),ze=n(e),q=i(e,"P",{"data-svelte-h":!0}),c(q)!=="svelte-edf8im"&&(q.textContent=Ct),Ye=n(e),r(A.$$.fragment,e),qe=n(e),k=i(e,"P",{"data-svelte-h":!0}),c(k)!=="svelte-1ov08z8"&&(k.innerHTML=_t),Ae=n(e),r(N.$$.fragment,e),ke=n(e),H=i(e,"P",{"data-svelte-h":!0}),c(H)!=="svelte-m0abi0"&&(H.innerHTML=$t),Ne=n(e),r(x.$$.fragment,e),He=n(e),Q=i(e,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-107qzzz"&&(Q.innerHTML=Zt),xe=n(e),r(S.$$.fragment,e),Qe=n(e),F=i(e,"P",{"data-svelte-h":!0}),c(F)!=="svelte-202ivx"&&(F.textContent=Bt),Se=n(e),r(E.$$.fragment,e),Fe=n(e),L=i(e,"P",{"data-svelte-h":!0}),c(L)!=="svelte-wdo2ze"&&(L.innerHTML=Wt),Ee=n(e),r(D.$$.fragment,e),Le=n(e),P=i(e,"P",{"data-svelte-h":!0}),c(P)!=="svelte-171sb04"&&(P.innerHTML=Xt),De=n(e),r(K.$$.fragment,e),Pe=n(e),O=i(e,"P",{"data-svelte-h":!0}),c(O)!=="svelte-14r7t5t"&&(O.innerHTML=Gt),Ke=n(e),r(ee.$$.fragment,e),Oe=n(e),r(te.$$.fragment,e),et=n(e),le=i(e,"P",{"data-svelte-h":!0}),c(le)!=="svelte-1843po4"&&(le.innerHTML=Rt),tt=n(e),r(se.$$.fragment,e),lt=n(e),r(ae.$$.fragment,e),st=n(e),ne=i(e,"P",{"data-svelte-h":!0}),c(ne)!=="svelte-16bzhhs"&&(ne.innerHTML=vt),at=n(e),pe=i(e,"P",{"data-svelte-h":!0}),c(pe)!=="svelte-129ksn5"&&(pe.innerHTML=Vt),nt=n(e),ie=i(e,"P",{"data-svelte-h":!0}),c(ie)!=="svelte-1r4gizw"&&(ie.innerHTML=zt),pt=n(e),r(ce.$$.fragment,e),it=n(e),oe=i(e,"P",{"data-svelte-h":!0}),c(oe)!=="svelte-5wgsij"&&(oe.innerHTML=Yt),ct=n(e),r(ue.$$.fragment,e),ot=n(e),re=i(e,"P",{"data-svelte-h":!0}),c(re)!=="svelte-1oe955w"&&(re.textContent=qt),ut=n(e),r(Me.$$.fragment,e),rt=n(e),r(de.$$.fragment,e),Mt=n(e),Je=i(e,"P",{"data-svelte-h":!0}),c(Je)!=="svelte-1hqyy97"&&(Je.innerHTML=At),dt=n(e),r(ye.$$.fragment,e),Jt=n(e),me=i(e,"P",{}),Ht(me).forEach(l),this.h()},h(){xt(o,"name","hf:doc:metadata"),xt(o,"content",al)},m(e,t){Kt(document.head,o),s(e,m,t),s(e,T,t),s(e,h,t),M(I,e,t),s(e,he,t),M(U,e,t),s(e,je,t),M(j,e,t),s(e,fe,t),s(e,g,t),s(e,Ie,t),s(e,b,t),s(e,Ue,t),s(e,C,t),s(e,ge,t),s(e,_,t),s(e,be,t),s(e,$,t),s(e,Ce,t),M(Z,e,t),s(e,_e,t),s(e,B,t),s(e,$e,t),s(e,W,t),s(e,Ze,t),M(X,e,t),s(e,Be,t),s(e,G,t),s(e,We,t),M(R,e,t),s(e,Xe,t),s(e,v,t),s(e,Ge,t),M(V,e,t),s(e,Re,t),M(f,e,t),s(e,ve,t),s(e,z,t),s(e,Ve,t),M(Y,e,t),s(e,ze,t),s(e,q,t),s(e,Ye,t),M(A,e,t),s(e,qe,t),s(e,k,t),s(e,Ae,t),M(N,e,t),s(e,ke,t),s(e,H,t),s(e,Ne,t),M(x,e,t),s(e,He,t),s(e,Q,t),s(e,xe,t),M(S,e,t),s(e,Qe,t),s(e,F,t),s(e,Se,t),M(E,e,t),s(e,Fe,t),s(e,L,t),s(e,Ee,t),M(D,e,t),s(e,Le,t),s(e,P,t),s(e,De,t),M(K,e,t),s(e,Pe,t),s(e,O,t),s(e,Ke,t),M(ee,e,t),s(e,Oe,t),M(te,e,t),s(e,et,t),s(e,le,t),s(e,tt,t),M(se,e,t),s(e,lt,t),M(ae,e,t),s(e,st,t),s(e,ne,t),s(e,at,t),s(e,pe,t),s(e,nt,t),s(e,ie,t),s(e,pt,t),M(ce,e,t),s(e,it,t),s(e,oe,t),s(e,ct,t),M(ue,e,t),s(e,ot,t),s(e,re,t),s(e,ut,t),M(Me,e,t),s(e,rt,t),M(de,e,t),s(e,Mt,t),s(e,Je,t),s(e,dt,t),M(ye,e,t),s(e,Jt,t),s(e,me,t),yt=!0},p(e,[t]){const kt={};t&2&&(kt.$$scope={dirty:t,ctx:e}),j.$set(kt);const Nt={};t&2&&(Nt.$$scope={dirty:t,ctx:e}),f.$set(Nt)},i(e){yt||(d(I.$$.fragment,e),d(U.$$.fragment,e),d(j.$$.fragment,e),d(Z.$$.fragment,e),d(X.$$.fragment,e),d(R.$$.fragment,e),d(V.$$.fragment,e),d(f.$$.fragment,e),d(Y.$$.fragment,e),d(A.$$.fragment,e),d(N.$$.fragment,e),d(x.$$.fragment,e),d(S.$$.fragment,e),d(E.$$.fragment,e),d(D.$$.fragment,e),d(K.$$.fragment,e),d(ee.$$.fragment,e),d(te.$$.fragment,e),d(se.$$.fragment,e),d(ae.$$.fragment,e),d(ce.$$.fragment,e),d(ue.$$.fragment,e),d(Me.$$.fragment,e),d(de.$$.fragment,e),d(ye.$$.fragment,e),yt=!0)},o(e){J(I.$$.fragment,e),J(U.$$.fragment,e),J(j.$$.fragment,e),J(Z.$$.fragment,e),J(X.$$.fragment,e),J(R.$$.fragment,e),J(V.$$.fragment,e),J(f.$$.fragment,e),J(Y.$$.fragment,e),J(A.$$.fragment,e),J(N.$$.fragment,e),J(x.$$.fragment,e),J(S.$$.fragment,e),J(E.$$.fragment,e),J(D.$$.fragment,e),J(K.$$.fragment,e),J(ee.$$.fragment,e),J(te.$$.fragment,e),J(se.$$.fragment,e),J(ae.$$.fragment,e),J(ce.$$.fragment,e),J(ue.$$.fragment,e),J(Me.$$.fragment,e),J(de.$$.fragment,e),J(ye.$$.fragment,e),yt=!1},d(e){e&&(l(m),l(T),l(h),l(he),l(je),l(fe),l(g),l(Ie),l(b),l(Ue),l(C),l(ge),l(_),l(be),l($),l(Ce),l(_e),l(B),l($e),l(W),l(Ze),l(Be),l(G),l(We),l(Xe),l(v),l(Ge),l(Re),l(ve),l(z),l(Ve),l(ze),l(q),l(Ye),l(qe),l(k),l(Ae),l(ke),l(H),l(Ne),l(He),l(Q),l(xe),l(Qe),l(F),l(Se),l(Fe),l(L),l(Ee),l(Le),l(P),l(De),l(Pe),l(O),l(Ke),l(Oe),l(et),l(le),l(tt),l(lt),l(st),l(ne),l(at),l(pe),l(nt),l(ie),l(pt),l(it),l(oe),l(ct),l(ot),l(re),l(ut),l(rt),l(Mt),l(Je),l(dt),l(Jt),l(me)),l(o),y(I,e),y(U,e),y(j,e),y(Z,e),y(X,e),y(R,e),y(V,e),y(f,e),y(Y,e),y(A,e),y(N,e),y(x,e),y(S,e),y(E,e),y(D,e),y(K,e),y(ee,e),y(te,e),y(se,e),y(ae,e),y(ce,e),y(ue,e),y(Me,e),y(de,e),y(ye,e)}}}const al='{"title":"Using multiple models with DeepSpeed","local":"using-multiple-models-with-deepspeed","sections":[{"title":"Knowledge distillation","local":"knowledge-distillation","sections":[{"title":"Student model","local":"student-model","sections":[],"depth":3},{"title":"Teacher model","local":"teacher-model","sections":[],"depth":3},{"title":"Training","local":"training","sections":[],"depth":3}],"depth":2},{"title":"Train multiple disjoint models","local":"train-multiple-disjoint-models","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function nl(we){return Et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class rl extends Lt{constructor(o){super(),Dt(this,o,nl,sl,Ft,{})}}export{rl as component};