import{s as Qt,o as St,n as xt}from"../chunks/scheduler.defa9a21.js";import{S as Ft,i as Et,g as p,s as a,r as u,A as Lt,h as i,f as s,c as n,j as kt,u as r,x as c,k as Nt,y as Dt,a as l,v as M,d,t as J,w as T}from"../chunks/index.fe795e71.js";import{T as Ht}from"../chunks/Tip.179eb360.js";import{C as w}from"../chunks/CodeBlock.204b6c34.js";import{H as Te,E as Pt}from"../chunks/getInferenceSnippets.2234a8dd.js";function Kt(ye){let o,m='This guide assumes that you have read and understood the <a href="./deepspeed.md">DeepSpeed usage guide</a>.';return{c(){o=p("p"),o.innerHTML=m},l(y){o=i(y,"P",{"data-svelte-h":!0}),c(o)!=="svelte-vrzv4k"&&(o.innerHTML=m)},m(y,h){l(y,o,h)},p:xt,d(y){y&&s(o)}}}function Ot(ye){let o,m="DeepSpeed will raise an error if <code>train_micro_batch_size_per_gpu</code> isn’t specified, even if this particular model isn’t being trained.";return{c(){o=p("p"),o.innerHTML=m},l(y){o=i(y,"P",{"data-svelte-h":!0}),c(o)!=="svelte-qz5bxi"&&(o.innerHTML=m)},m(y,h){l(y,o,h)},p:xt,d(y){y&&s(o)}}}function es(ye){let o,m,y,h,I,me,j,he,U,Jt="Running multiple models with Accelerate and DeepSpeed is useful for:",je,b,Tt='<li>Knowledge distillation</li> <li>Post-training techniques like RLHF (see the <a href="https://github.com/huggingface/trl" rel="nofollow">TRL</a> library for more examples)</li> <li>Training multiple models at once</li>',fe,g,yt="Currently, Accelerate has a <strong>very experimental API</strong> to help you use multiple models.",Ie,C,wt="This tutorial will focus on two common use cases:",Ue,_,mt="<li>Knowledge distillation, where a smaller student model is trained to mimic a larger, better-performing teacher.  If the student model fits on a single GPU, we can use ZeRO-2 for training and ZeRO-3 to shard the teacher for inference. This is significantly faster than using ZeRO-3 for both models.</li> <li>Training multiple <em>disjoint</em> models at once.</li>",be,Z,ge,$,ht="Knowledge distillation is a good example of using multiple models, but only training one of them.",Ce,B,jt='Normally, you would use a single <a href="/docs/accelerate/main/en/package_reference/utilities#accelerate.DeepSpeedPlugin">utils.DeepSpeedPlugin</a> for both models. However, in this case, there are two separate configurations. Accelerate allows you to create and use multiple plugins <strong>if and only if</strong> they are in a <code>dict</code> so that you can reference and enable the proper plugin when needed.',_e,W,Ze,X,ft="The <code>zero2_config.json</code> should be configured for full training (so specify <code>scheduler</code> and <code>optimizer</code> if you are not utilizing your own), while <code>zero3_config.json</code> should only be configured for the inference model, as shown in the example below.",$e,v,Be,G,It="An example <code>zero2_config.json</code> configuration is shown below.",We,R,Xe,f,ve,V,Ut='From here, create a single <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> and pass in both configurations.',Ge,z,Re,Y,bt="Now let’s see how to use them.",Ve,A,ze,q,gt='By default, Accelerate sets the first item in the <code>dict</code> as the default or enabled plugin (<code>&quot;student&quot;</code> plugin). Verify this by using the <a href="/docs/accelerate/main/en/package_reference/deepspeed#accelerate.utils.get_active_deepspeed_plugin">utils.deepspeed.get_active_deepspeed_plugin()</a> function to see which plugin is enabled.',Ye,k,Ae,N,Ct="<code>AcceleratorState</code> also keeps the active DeepSpeed plugin saved in <code>state.deepspeed_plugin</code>.",qe,H,ke,x,_t="Since <code>student</code> is the currently active plugin, let’s go ahead and prepare the model, optimizer, and scheduler.",Ne,Q,He,S,Zt="Now it’s time to deal with the teacher model.",xe,F,Qe,E,$t='First, you need to specify in <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> that the <code>zero3_config.json</code> configuration should be used.',Se,L,Fe,D,Bt=`This disables the <code>&quot;student&quot;</code> plugin and enables the <code>&quot;teacher&quot;</code> plugin instead. The
DeepSpeed stateful config inside of Transformers is updated, and it changes which plugin configuration gets called when using
<code>deepspeed.initialize()</code>. This allows you to use the automatic <code>deepspeed.zero.Init</code>  context manager integration Transformers provides.`,Ee,P,Le,K,Wt="Otherwise, you should manually initialize the model with <code>deepspeed.zero.Init</code>.",De,O,Pe,ee,Ke,te,Xt="From here, your training loop can be whatever you like, as long as <code>teacher_model</code> is never being trained on.",Oe,se,et,le,tt,ae,vt=`Training multiple models is a more complicated scenario.
In its current state, we assume each model is <strong>completely disjointed</strong> from the other during training.`,st,ne,Gt='This scenario still requires two <a href="/docs/accelerate/main/en/package_reference/utilities#accelerate.DeepSpeedPlugin">utils.DeepSpeedPlugin</a>’s to be made. However, you also need a second <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a>, since different <code>deepspeed</code> engines are being called at different times. A single <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> can only carry one instance at a time.',lt,pe,Rt='Since the <a href="/docs/accelerate/main/en/package_reference/state#accelerate.state.AcceleratorState">state.AcceleratorState</a> is a stateful object though, it is already aware of both <a href="/docs/accelerate/main/en/package_reference/utilities#accelerate.DeepSpeedPlugin">utils.DeepSpeedPlugin</a>’s available. You can just instantiate a second <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> with no extra arguments.',at,ie,nt,ce,Vt=`You can call either <code>first_accelerator.state.select_deepspeed_plugin()</code> to enable or disable
a particular plugin, and then call <code>prepare</code>.`,pt,oe,it,ue,zt="And now you can train:",ct,re,ot,Me,ut,de,Yt='To see more examples, please check out the <a href="https://github.com/huggingface/accelerate/blob/main/src/accelerate/test_utils/scripts/external_deps/test_ds_multiple_model.py" rel="nofollow">related tests</a> currently in [Accelerate].',rt,Je,Mt,we,dt;return I=new Te({props:{title:"Using multiple models with DeepSpeed",local:"using-multiple-models-with-deepspeed",headingTag:"h1"}}),j=new Ht({props:{warning:!0,$$slots:{default:[Kt]},$$scope:{ctx:ye}}}),Z=new Te({props:{title:"Knowledge distillation",local:"knowledge-distillation",headingTag:"h2"}}),W=new w({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBEZWVwU3BlZWRQbHVnaW4lMEElMEF6ZXJvMl9wbHVnaW4lMjAlM0QlMjBEZWVwU3BlZWRQbHVnaW4oaGZfZHNfY29uZmlnJTNEJTIyemVybzJfY29uZmlnLmpzb24lMjIpJTBBemVybzNfcGx1Z2luJTIwJTNEJTIwRGVlcFNwZWVkUGx1Z2luKGhmX2RzX2NvbmZpZyUzRCUyMnplcm8zX2NvbmZpZy5qc29uJTIyKSUwQSUwQWRlZXBzcGVlZF9wbHVnaW5zJTIwJTNEJTIwJTdCJTIyc3R1ZGVudCUyMiUzQSUyMHplcm8yX3BsdWdpbiUyQyUyMCUyMnRlYWNoZXIlMjIlM0ElMjB6ZXJvM19wbHVnaW4lN0Q=",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> DeepSpeedPlugin

zero2_plugin = DeepSpeedPlugin(hf_ds_config=<span class="hljs-string">&quot;zero2_config.json&quot;</span>)
zero3_plugin = DeepSpeedPlugin(hf_ds_config=<span class="hljs-string">&quot;zero3_config.json&quot;</span>)

deepspeed_plugins = {<span class="hljs-string">&quot;student&quot;</span>: zero2_plugin, <span class="hljs-string">&quot;teacher&quot;</span>: zero3_plugin}`,wrap:!1}}),v=new w({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyYmYxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19vcHRpbWl6YXRpb24lMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZSUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvdmVybGFwX2NvbW0lMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycmVkdWNlX2J1Y2tldF9zaXplJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19wcmVmZXRjaF9idWNrZXRfc2l6ZSUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZTNfcGFyYW1fcGVyc2lzdGVuY2VfdGhyZXNob2xkJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19tYXhfbGl2ZV9wYXJhbWV0ZXJzJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19tYXhfcmV1c2VfZGlzdGFuY2UlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JTIyJTNBJTIwMSUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;bf16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">3</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;overlap_comm&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;reduce_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_prefetch_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_param_persistence_threshold&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_max_live_parameters&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_max_reuse_distance&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),R=new w({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyYmYxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyb3B0aW1pemVyJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydHlwZSUyMiUzQSUyMCUyMkFkYW1XJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGFyYW1zJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybHIlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyd2VpZ2h0X2RlY2F5JTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRvcmNoX2FkYW0lMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWRhbV93X21vZGUlMjIlM0ElMjB0cnVlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyc2NoZWR1bGVyJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydHlwZSUyMiUzQSUyMCUyMldhcm11cExSJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGFyYW1zJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyd2FybXVwX21pbl9sciUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ3YXJtdXBfbWF4X2xyJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMndhcm11cF9udW1fc3RlcHMlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19vcHRpbWl6YXRpb24lMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZSUyMiUzQSUyMDIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvZmZsb2FkX29wdGltaXplciUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmRldmljZSUyMiUzQSUyMCUyMmNwdSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBpbl9tZW1vcnklMjIlM0ElMjB0cnVlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIyJTNBJTIwMSUyQyUwQSUyMCUyMCUyMCUyMCUyMmdyYWRpZW50X2NsaXBwaW5nJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYWluX2JhdGNoX3NpemUlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;bf16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;AdamW&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;weight_decay&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;torch_adam&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;adam_w_mode&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;scheduler&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;WarmupLR&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;warmup_min_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_max_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_num_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;offload_optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;device&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;cpu&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;pin_memory&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
        <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_accumulation_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_clipping&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_batch_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),f=new Ht({props:{$$slots:{default:[Ot]},$$scope:{ctx:ye}}}),z=new w({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZGVlcHNwZWVkX3BsdWdpbnMlM0RkZWVwc3BlZWRfcGx1Z2lucyk=",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator(deepspeed_plugins=deepspeed_plugins)`,wrap:!1}}),A=new Te({props:{title:"Student model",local:"student-model",headingTag:"h3"}}),k=new w({props:{code:"YWN0aXZlX3BsdWdpbiUyMCUzRCUyMGdldF9hY3RpdmVfZGVlcHNwZWVkX3BsdWdpbihhY2NlbGVyYXRvci5zdGF0ZSklMEFhc3NlcnQlMjBhY3RpdmVfcGx1Z2luJTIwaXMlMjBkZWVwc3BlZWRfcGx1Z2lucyU1QiUyMnN0dWRlbnQlMjIlNUQ=",highlighted:`active_plugin = get_active_deepspeed_plugin(accelerator.state)
<span class="hljs-keyword">assert</span> active_plugin <span class="hljs-keyword">is</span> deepspeed_plugins[<span class="hljs-string">&quot;student&quot;</span>]`,wrap:!1}}),H=new w({props:{code:"YXNzZXJ0JTIwYWN0aXZlX3BsdWdpbiUyMGlzJTIwYWNjZWxlcmF0b3IuZGVlcHNwZWVkX3BsdWdpbg==",highlighted:'<span class="hljs-keyword">assert</span> active_plugin <span class="hljs-keyword">is</span> accelerator.deepspeed_plugin',wrap:!1}}),Q=new w({props:{code:"c3R1ZGVudF9tb2RlbCUyQyUyMG9wdGltaXplciUyQyUyMHNjaGVkdWxlciUyMCUzRCUyMC4uLiUwQXN0dWRlbnRfbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjBzY2hlZHVsZXIlMkMlMjB0cmFpbl9kYXRhbG9hZGVyJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZShzdHVkZW50X21vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwc2NoZWR1bGVyJTJDJTIwdHJhaW5fZGF0YWxvYWRlcik=",highlighted:`student_model, optimizer, scheduler = ...
student_model, optimizer, scheduler, train_dataloader = accelerator.prepare(student_model, optimizer, scheduler, train_dataloader)`,wrap:!1}}),F=new Te({props:{title:"Teacher model",local:"teacher-model",headingTag:"h3"}}),L=new w({props:{code:"YWNjZWxlcmF0b3Iuc3RhdGUuc2VsZWN0X2RlZXBzcGVlZF9wbHVnaW4oJTIydGVhY2hlciUyMik=",highlighted:'accelerator.state.select_deepspeed_plugin(<span class="hljs-string">&quot;teacher&quot;</span>)',wrap:!1}}),P=new w({props:{code:"dGVhY2hlcl9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoLi4uKSUwQXRlYWNoZXJfbW9kZWwlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKHRlYWNoZXJfbW9kZWwp",highlighted:`teacher_model = AutoModel.from_pretrained(...)
teacher_model = accelerator.prepare(teacher_model)`,wrap:!1}}),O=new w({props:{code:"d2l0aCUyMGRlZXBzcGVlZC56ZXJvLkluaXQoYWNjZWxlcmF0b3IuZGVlcHNwZWVkX3BsdWdpbi5jb25maWcpJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwlMjAlM0QlMjBNeU1vZGVsKC4uLik=",highlighted:`<span class="hljs-keyword">with</span> deepspeed.zero.Init(accelerator.deepspeed_plugin.config):
    model = MyModel(...)`,wrap:!1}}),ee=new Te({props:{title:"Training",local:"training",headingTag:"h3"}}),se=new w({props:{code:"dGVhY2hlcl9tb2RlbC5ldmFsKCklMEFzdHVkZW50X21vZGVsLnRyYWluKCklMEFmb3IlMjBiYXRjaCUyMGluJTIwdHJhaW5fZGF0YWxvYWRlciUzQSUwQSUyMCUyMCUyMCUyMHdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRfdGVhY2hlciUyMCUzRCUyMHRlYWNoZXJfbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjBvdXRwdXRfc3R1ZGVudCUyMCUzRCUyMHN0dWRlbnRfbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjAlMjMlMjBDb21iaW5lJTIwdGhlJTIwbG9zc2VzJTIwb3IlMjBtb2RpZnklMjBpdCUyMGluJTIwc29tZSUyMHdheSUwQSUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBvdXRwdXRfdGVhY2hlci5sb3NzJTIwJTJCJTIwb3V0cHV0X3N0dWRlbnQubG9zcyUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMHNjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCk=",highlighted:`teacher_model.<span class="hljs-built_in">eval</span>()
student_model.train()
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
    <span class="hljs-keyword">with</span> torch.no_grad():
        output_teacher = teacher_model(**batch)
    output_student = student_model(**batch)
    <span class="hljs-comment"># Combine the losses or modify it in some way</span>
    loss = output_teacher.loss + output_student.loss
    accelerator.backward(loss)
    optimizer.step()
    scheduler.step()
    optimizer.zero_grad()`,wrap:!1}}),le=new Te({props:{title:"Train multiple disjoint models",local:"train-multiple-disjoint-models",headingTag:"h2"}}),ie=new w({props:{code:"Zmlyc3RfYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihkZWVwc3BlZWRfcGx1Z2lucyUzRGRlZXBzcGVlZF9wbHVnaW5zKSUwQXNlY29uZF9hY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKCk=",highlighted:`first_accelerator = Accelerator(deepspeed_plugins=deepspeed_plugins)
second_accelerator = Accelerator()`,wrap:!1}}),oe=new w({props:{code:"JTIzJTIwY2FuJTIwYmUlMjAlNjBhY2NlbGVyYXRvcl8wJTYwJTJDJTIwJTYwYWNjZWxlcmF0b3JfMSU2MCUyQyUyMG9yJTIwYnklMjBjYWxsaW5nJTIwJTYwQWNjZWxlcmF0b3JTdGF0ZSgpLnNlbGVjdF9kZWVwc3BlZWRfcGx1Z2luKC4uLiklNjAlMEFmaXJzdF9hY2NlbGVyYXRvci5zdGF0ZS5zZWxlY3RfZGVlcHNwZWVkX3BsdWdpbiglMjJmaXJzdF9tb2RlbCUyMiklMEFmaXJzdF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoLi4uKSUwQSUyMyUyMEZvciUyMHRoaXMlMjBleGFtcGxlJTJDJTIwJTYwZ2V0X3RyYWluaW5nX2l0ZW1zJTYwJTIwaXMlMjBhJTIwbm9uZXhpc3RlbnQlMjBmdW5jdGlvbiUyMHRoYXQlMjBnZXRzJTIwdGhlJTIwc2V0dXAlMjB3ZSUyMG5lZWQlMjBmb3IlMjB0cmFpbmluZyUwQWZpcnN0X29wdGltaXplciUyQyUyMGZpcnN0X3NjaGVkdWxlciUyQyUyMHRyYWluX2RsJTJDJTIwZXZhbF9kbCUyMCUzRCUyMGdldF90cmFpbmluZ19pdGVtcyhtb2RlbDEpJTBBZmlyc3RfbW9kZWwlMkMlMjBmaXJzdF9vcHRpbWl6ZXIlMkMlMjBmaXJzdF9zY2hlZHVsZXIlMkMlMjB0cmFpbl9kbCUyQyUyMGV2YWxfZGwlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKCUwQSUyMCUyMCUyMCUyMGZpcnN0X21vZGVsJTJDJTIwZmlyc3Rfb3B0aW1pemVyJTJDJTIwZmlyc3Rfc2NoZWR1bGVyJTJDJTIwdHJhaW5fZGwlMkMlMjBldmFsX2RsJTBBKSUwQSUwQXNlY29uZF9hY2NlbGVyYXRvci5zdGF0ZS5zZWxlY3RfZGVlcHNwZWVkX3BsdWdpbiglMjJzZWNvbmRfbW9kZWwlMjIpJTBBc2Vjb25kX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCguLi4pJTBBJTIzJTIwRm9yJTIwdGhpcyUyMGV4YW1wbGUlMkMlMjAlNjBnZXRfdHJhaW5pbmdfaXRlbXMlNjAlMjBpcyUyMGElMjBub25leGlzdGVudCUyMGZ1bmN0aW9uJTIwdGhhdCUyMGdldHMlMjB0aGUlMjBzZXR1cCUyMHdlJTIwbmVlZCUyMGZvciUyMHRyYWluaW5nJTBBc2Vjb25kX29wdGltaXplciUyQyUyMHNlY29uZF9zY2hlZHVsZXIlMkMlMjBfJTJDJTIwXyUyMCUzRCUyMGdldF90cmFpbmluZ19pdGVtcyhtb2RlbDIpJTBBc2Vjb25kX21vZGVsJTJDJTIwc2Vjb25kX29wdGltaXplciUyQyUyMHNlY29uZF9zY2hlZHVsZXIlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKCUwQSUyMCUyMCUyMCUyMHNlY29uZF9tb2RlbCUyQyUyMHNlY29uZF9vcHRpbWl6ZXIlMkMlMjBzZWNvbmRfc2NoZWR1bGVyJTBBKQ==",highlighted:`<span class="hljs-comment"># can be \`accelerator_0\`, \`accelerator_1\`, or by calling \`AcceleratorState().select_deepspeed_plugin(...)\`</span>
first_accelerator.state.select_deepspeed_plugin(<span class="hljs-string">&quot;first_model&quot;</span>)
first_model = AutoModel.from_pretrained(...)
<span class="hljs-comment"># For this example, \`get_training_items\` is a nonexistent function that gets the setup we need for training</span>
first_optimizer, first_scheduler, train_dl, eval_dl = get_training_items(model1)
first_model, first_optimizer, first_scheduler, train_dl, eval_dl = accelerator.prepare(
    first_model, first_optimizer, first_scheduler, train_dl, eval_dl
)

second_accelerator.state.select_deepspeed_plugin(<span class="hljs-string">&quot;second_model&quot;</span>)
second_model = AutoModel.from_pretrained(...)
<span class="hljs-comment"># For this example, \`get_training_items\` is a nonexistent function that gets the setup we need for training</span>
second_optimizer, second_scheduler, _, _ = get_training_items(model2)
second_model, second_optimizer, second_scheduler = accelerator.prepare(
    second_model, second_optimizer, second_scheduler
)`,wrap:!1}}),re=new w({props:{code:"Zm9yJTIwYmF0Y2glMjBpbiUyMGRsJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0czElMjAlM0QlMjBmaXJzdF9tb2RlbCgqKmJhdGNoKSUwQSUyMCUyMCUyMCUyMGZpcnN0X2FjY2VsZXJhdG9yLmJhY2t3YXJkKG91dHB1dHMxLmxvc3MpJTBBJTIwJTIwJTIwJTIwZmlyc3Rfb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMGZpcnN0X3NjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjBmaXJzdF9vcHRpbWl6ZXIuemVyb19ncmFkKCklMEElMjAlMjAlMjAlMjAlMEElMjAlMjAlMjAlMjBvdXRwdXRzMiUyMCUzRCUyMG1vZGVsMigqKmJhdGNoKSUwQSUyMCUyMCUyMCUyMHNlY29uZF9hY2NlbGVyYXRvci5iYWNrd2FyZChvdXRwdXRzMi5sb3NzKSUwQSUyMCUyMCUyMCUyMHNlY29uZF9vcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwc2Vjb25kX3NjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjBzZWNvbmRfb3B0aW1pemVyLnplcm9fZ3JhZCgp",highlighted:`<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> dl:
    outputs1 = first_model(**batch)
    first_accelerator.backward(outputs1.loss)
    first_optimizer.step()
    first_scheduler.step()
    first_optimizer.zero_grad()
    
    outputs2 = model2(**batch)
    second_accelerator.backward(outputs2.loss)
    second_optimizer.step()
    second_scheduler.step()
    second_optimizer.zero_grad()`,wrap:!1}}),Me=new Te({props:{title:"Resources",local:"resources",headingTag:"h2"}}),Je=new Pt({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/deepspeed_multiple_model.md"}}),{c(){o=p("meta"),m=a(),y=p("p"),h=a(),u(I.$$.fragment),me=a(),u(j.$$.fragment),he=a(),U=p("p"),U.textContent=Jt,je=a(),b=p("ul"),b.innerHTML=Tt,fe=a(),g=p("p"),g.innerHTML=yt,Ie=a(),C=p("p"),C.textContent=wt,Ue=a(),_=p("ol"),_.innerHTML=mt,be=a(),u(Z.$$.fragment),ge=a(),$=p("p"),$.textContent=ht,Ce=a(),B=p("p"),B.innerHTML=jt,_e=a(),u(W.$$.fragment),Ze=a(),X=p("p"),X.innerHTML=ft,$e=a(),u(v.$$.fragment),Be=a(),G=p("p"),G.innerHTML=It,We=a(),u(R.$$.fragment),Xe=a(),u(f.$$.fragment),ve=a(),V=p("p"),V.innerHTML=Ut,Ge=a(),u(z.$$.fragment),Re=a(),Y=p("p"),Y.textContent=bt,Ve=a(),u(A.$$.fragment),ze=a(),q=p("p"),q.innerHTML=gt,Ye=a(),u(k.$$.fragment),Ae=a(),N=p("p"),N.innerHTML=Ct,qe=a(),u(H.$$.fragment),ke=a(),x=p("p"),x.innerHTML=_t,Ne=a(),u(Q.$$.fragment),He=a(),S=p("p"),S.textContent=Zt,xe=a(),u(F.$$.fragment),Qe=a(),E=p("p"),E.innerHTML=$t,Se=a(),u(L.$$.fragment),Fe=a(),D=p("p"),D.innerHTML=Bt,Ee=a(),u(P.$$.fragment),Le=a(),K=p("p"),K.innerHTML=Wt,De=a(),u(O.$$.fragment),Pe=a(),u(ee.$$.fragment),Ke=a(),te=p("p"),te.innerHTML=Xt,Oe=a(),u(se.$$.fragment),et=a(),u(le.$$.fragment),tt=a(),ae=p("p"),ae.innerHTML=vt,st=a(),ne=p("p"),ne.innerHTML=Gt,lt=a(),pe=p("p"),pe.innerHTML=Rt,at=a(),u(ie.$$.fragment),nt=a(),ce=p("p"),ce.innerHTML=Vt,pt=a(),u(oe.$$.fragment),it=a(),ue=p("p"),ue.textContent=zt,ct=a(),u(re.$$.fragment),ot=a(),u(Me.$$.fragment),ut=a(),de=p("p"),de.innerHTML=Yt,rt=a(),u(Je.$$.fragment),Mt=a(),we=p("p"),this.h()},l(e){const t=Lt("svelte-u9bgzb",document.head);o=i(t,"META",{name:!0,content:!0}),t.forEach(s),m=n(e),y=i(e,"P",{}),kt(y).forEach(s),h=n(e),r(I.$$.fragment,e),me=n(e),r(j.$$.fragment,e),he=n(e),U=i(e,"P",{"data-svelte-h":!0}),c(U)!=="svelte-1b1gxyz"&&(U.textContent=Jt),je=n(e),b=i(e,"UL",{"data-svelte-h":!0}),c(b)!=="svelte-7mvbd"&&(b.innerHTML=Tt),fe=n(e),g=i(e,"P",{"data-svelte-h":!0}),c(g)!=="svelte-e8xs1z"&&(g.innerHTML=yt),Ie=n(e),C=i(e,"P",{"data-svelte-h":!0}),c(C)!=="svelte-owdl16"&&(C.textContent=wt),Ue=n(e),_=i(e,"OL",{"data-svelte-h":!0}),c(_)!=="svelte-to0mxc"&&(_.innerHTML=mt),be=n(e),r(Z.$$.fragment,e),ge=n(e),$=i(e,"P",{"data-svelte-h":!0}),c($)!=="svelte-zc35yl"&&($.textContent=ht),Ce=n(e),B=i(e,"P",{"data-svelte-h":!0}),c(B)!=="svelte-jozejn"&&(B.innerHTML=jt),_e=n(e),r(W.$$.fragment,e),Ze=n(e),X=i(e,"P",{"data-svelte-h":!0}),c(X)!=="svelte-4l2obz"&&(X.innerHTML=ft),$e=n(e),r(v.$$.fragment,e),Be=n(e),G=i(e,"P",{"data-svelte-h":!0}),c(G)!=="svelte-2zuo2r"&&(G.innerHTML=It),We=n(e),r(R.$$.fragment,e),Xe=n(e),r(f.$$.fragment,e),ve=n(e),V=i(e,"P",{"data-svelte-h":!0}),c(V)!=="svelte-t7yg3i"&&(V.innerHTML=Ut),Ge=n(e),r(z.$$.fragment,e),Re=n(e),Y=i(e,"P",{"data-svelte-h":!0}),c(Y)!=="svelte-edf8im"&&(Y.textContent=bt),Ve=n(e),r(A.$$.fragment,e),ze=n(e),q=i(e,"P",{"data-svelte-h":!0}),c(q)!=="svelte-bgcpa4"&&(q.innerHTML=gt),Ye=n(e),r(k.$$.fragment,e),Ae=n(e),N=i(e,"P",{"data-svelte-h":!0}),c(N)!=="svelte-m0abi0"&&(N.innerHTML=Ct),qe=n(e),r(H.$$.fragment,e),ke=n(e),x=i(e,"P",{"data-svelte-h":!0}),c(x)!=="svelte-107qzzz"&&(x.innerHTML=_t),Ne=n(e),r(Q.$$.fragment,e),He=n(e),S=i(e,"P",{"data-svelte-h":!0}),c(S)!=="svelte-202ivx"&&(S.textContent=Zt),xe=n(e),r(F.$$.fragment,e),Qe=n(e),E=i(e,"P",{"data-svelte-h":!0}),c(E)!=="svelte-11n7i8s"&&(E.innerHTML=$t),Se=n(e),r(L.$$.fragment,e),Fe=n(e),D=i(e,"P",{"data-svelte-h":!0}),c(D)!=="svelte-171sb04"&&(D.innerHTML=Bt),Ee=n(e),r(P.$$.fragment,e),Le=n(e),K=i(e,"P",{"data-svelte-h":!0}),c(K)!=="svelte-14r7t5t"&&(K.innerHTML=Wt),De=n(e),r(O.$$.fragment,e),Pe=n(e),r(ee.$$.fragment,e),Ke=n(e),te=i(e,"P",{"data-svelte-h":!0}),c(te)!=="svelte-1843po4"&&(te.innerHTML=Xt),Oe=n(e),r(se.$$.fragment,e),et=n(e),r(le.$$.fragment,e),tt=n(e),ae=i(e,"P",{"data-svelte-h":!0}),c(ae)!=="svelte-16bzhhs"&&(ae.innerHTML=vt),st=n(e),ne=i(e,"P",{"data-svelte-h":!0}),c(ne)!=="svelte-1j3vqvo"&&(ne.innerHTML=Gt),lt=n(e),pe=i(e,"P",{"data-svelte-h":!0}),c(pe)!=="svelte-iouvgh"&&(pe.innerHTML=Rt),at=n(e),r(ie.$$.fragment,e),nt=n(e),ce=i(e,"P",{"data-svelte-h":!0}),c(ce)!=="svelte-5wgsij"&&(ce.innerHTML=Vt),pt=n(e),r(oe.$$.fragment,e),it=n(e),ue=i(e,"P",{"data-svelte-h":!0}),c(ue)!=="svelte-1oe955w"&&(ue.textContent=zt),ct=n(e),r(re.$$.fragment,e),ot=n(e),r(Me.$$.fragment,e),ut=n(e),de=i(e,"P",{"data-svelte-h":!0}),c(de)!=="svelte-1hqyy97"&&(de.innerHTML=Yt),rt=n(e),r(Je.$$.fragment,e),Mt=n(e),we=i(e,"P",{}),kt(we).forEach(s),this.h()},h(){Nt(o,"name","hf:doc:metadata"),Nt(o,"content",ts)},m(e,t){Dt(document.head,o),l(e,m,t),l(e,y,t),l(e,h,t),M(I,e,t),l(e,me,t),M(j,e,t),l(e,he,t),l(e,U,t),l(e,je,t),l(e,b,t),l(e,fe,t),l(e,g,t),l(e,Ie,t),l(e,C,t),l(e,Ue,t),l(e,_,t),l(e,be,t),M(Z,e,t),l(e,ge,t),l(e,$,t),l(e,Ce,t),l(e,B,t),l(e,_e,t),M(W,e,t),l(e,Ze,t),l(e,X,t),l(e,$e,t),M(v,e,t),l(e,Be,t),l(e,G,t),l(e,We,t),M(R,e,t),l(e,Xe,t),M(f,e,t),l(e,ve,t),l(e,V,t),l(e,Ge,t),M(z,e,t),l(e,Re,t),l(e,Y,t),l(e,Ve,t),M(A,e,t),l(e,ze,t),l(e,q,t),l(e,Ye,t),M(k,e,t),l(e,Ae,t),l(e,N,t),l(e,qe,t),M(H,e,t),l(e,ke,t),l(e,x,t),l(e,Ne,t),M(Q,e,t),l(e,He,t),l(e,S,t),l(e,xe,t),M(F,e,t),l(e,Qe,t),l(e,E,t),l(e,Se,t),M(L,e,t),l(e,Fe,t),l(e,D,t),l(e,Ee,t),M(P,e,t),l(e,Le,t),l(e,K,t),l(e,De,t),M(O,e,t),l(e,Pe,t),M(ee,e,t),l(e,Ke,t),l(e,te,t),l(e,Oe,t),M(se,e,t),l(e,et,t),M(le,e,t),l(e,tt,t),l(e,ae,t),l(e,st,t),l(e,ne,t),l(e,lt,t),l(e,pe,t),l(e,at,t),M(ie,e,t),l(e,nt,t),l(e,ce,t),l(e,pt,t),M(oe,e,t),l(e,it,t),l(e,ue,t),l(e,ct,t),M(re,e,t),l(e,ot,t),M(Me,e,t),l(e,ut,t),l(e,de,t),l(e,rt,t),M(Je,e,t),l(e,Mt,t),l(e,we,t),dt=!0},p(e,[t]){const At={};t&2&&(At.$$scope={dirty:t,ctx:e}),j.$set(At);const qt={};t&2&&(qt.$$scope={dirty:t,ctx:e}),f.$set(qt)},i(e){dt||(d(I.$$.fragment,e),d(j.$$.fragment,e),d(Z.$$.fragment,e),d(W.$$.fragment,e),d(v.$$.fragment,e),d(R.$$.fragment,e),d(f.$$.fragment,e),d(z.$$.fragment,e),d(A.$$.fragment,e),d(k.$$.fragment,e),d(H.$$.fragment,e),d(Q.$$.fragment,e),d(F.$$.fragment,e),d(L.$$.fragment,e),d(P.$$.fragment,e),d(O.$$.fragment,e),d(ee.$$.fragment,e),d(se.$$.fragment,e),d(le.$$.fragment,e),d(ie.$$.fragment,e),d(oe.$$.fragment,e),d(re.$$.fragment,e),d(Me.$$.fragment,e),d(Je.$$.fragment,e),dt=!0)},o(e){J(I.$$.fragment,e),J(j.$$.fragment,e),J(Z.$$.fragment,e),J(W.$$.fragment,e),J(v.$$.fragment,e),J(R.$$.fragment,e),J(f.$$.fragment,e),J(z.$$.fragment,e),J(A.$$.fragment,e),J(k.$$.fragment,e),J(H.$$.fragment,e),J(Q.$$.fragment,e),J(F.$$.fragment,e),J(L.$$.fragment,e),J(P.$$.fragment,e),J(O.$$.fragment,e),J(ee.$$.fragment,e),J(se.$$.fragment,e),J(le.$$.fragment,e),J(ie.$$.fragment,e),J(oe.$$.fragment,e),J(re.$$.fragment,e),J(Me.$$.fragment,e),J(Je.$$.fragment,e),dt=!1},d(e){e&&(s(m),s(y),s(h),s(me),s(he),s(U),s(je),s(b),s(fe),s(g),s(Ie),s(C),s(Ue),s(_),s(be),s(ge),s($),s(Ce),s(B),s(_e),s(Ze),s(X),s($e),s(Be),s(G),s(We),s(Xe),s(ve),s(V),s(Ge),s(Re),s(Y),s(Ve),s(ze),s(q),s(Ye),s(Ae),s(N),s(qe),s(ke),s(x),s(Ne),s(He),s(S),s(xe),s(Qe),s(E),s(Se),s(Fe),s(D),s(Ee),s(Le),s(K),s(De),s(Pe),s(Ke),s(te),s(Oe),s(et),s(tt),s(ae),s(st),s(ne),s(lt),s(pe),s(at),s(nt),s(ce),s(pt),s(it),s(ue),s(ct),s(ot),s(ut),s(de),s(rt),s(Mt),s(we)),s(o),T(I,e),T(j,e),T(Z,e),T(W,e),T(v,e),T(R,e),T(f,e),T(z,e),T(A,e),T(k,e),T(H,e),T(Q,e),T(F,e),T(L,e),T(P,e),T(O,e),T(ee,e),T(se,e),T(le,e),T(ie,e),T(oe,e),T(re,e),T(Me,e),T(Je,e)}}}const ts='{"title":"Using multiple models with DeepSpeed","local":"using-multiple-models-with-deepspeed","sections":[{"title":"Knowledge distillation","local":"knowledge-distillation","sections":[{"title":"Student model","local":"student-model","sections":[],"depth":3},{"title":"Teacher model","local":"teacher-model","sections":[],"depth":3},{"title":"Training","local":"training","sections":[],"depth":3}],"depth":2},{"title":"Train multiple disjoint models","local":"train-multiple-disjoint-models","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function ss(ye){return St(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class cs extends Ft{constructor(o){super(),Et(this,o,ss,es,Qt,{})}}export{cs as component};
