import{s as Fn,o as Hn,n as qn}from"../chunks/scheduler.b9285784.js";import{S as $n,i as Ln,e as p,s as a,c as o,q as Dn,h as Pn,a as i,d as e,b as n,f as $l,g as M,j as c,r as On,k as Xl,l as d,m as t,n as u,t as r,o as y,p as j}from"../chunks/index.26bc89a1.js";import{T as Kn}from"../chunks/Tip.e4eba3d6.js";import{C as sp,H as b,E as lp}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e364eac2.js";import{C as J}from"../chunks/CodeBlock.804615c8.js";function ep(Dl){let T,X='For those interested in the similarities and differences between FSDP and DeepSpeed, please check out the <a href="../concept_guides/fsdp_and_deepspeed">concept guide here</a>!';return{c(){T=p("p"),T.innerHTML=X},l(f){T=i(f,"P",{"data-svelte-h":!0}),c(T)!=="svelte-we3qam"&&(T.innerHTML=X)},m(f,Nl){t(f,T,Nl)},p:qn,d(f){f&&e(T)}}}function tp(Dl){let T,X,f,Nl,N,Pl,G,Ol,v,ha='<a href="https://github.com/deepspeedai/DeepSpeed" rel="nofollow">DeepSpeed</a> implements everything described in the <a href="https://arxiv.org/abs/1910.02054" rel="nofollow">ZeRO paper</a>. Some of the salient optimizations are:',Kl,V,wa="<li>Optimizer state partitioning (ZeRO stage 1)</li> <li>Gradient partitioning (ZeRO stage 2)</li> <li>Parameter partitioning (ZeRO stage 3)</li> <li>Custom mixed precision training handling</li> <li>A range of fast CUDA-extension-based optimizers</li> <li>ZeRO-Offload to CPU and Disk/NVMe</li> <li>Hierarchical partitioning of model parameters (ZeRO++)</li>",se,Y,Ta=`ZeRO-Offload has its own dedicated paper: <a href="https://arxiv.org/abs/2101.06840" rel="nofollow">ZeRO-Offload: Democratizing Billion-Scale Model Training</a>. And NVMe-support is described in the paper <a href="https://arxiv.org/abs/2104.07857" rel="nofollow">ZeRO-Infinity: Breaking the GPU
Memory Wall for Extreme Scale Deep Learning</a>.`,le,A,Ua="DeepSpeed ZeRO-2 is primarily used only for training, as its features are of no use to inference.",ee,z,ma=`DeepSpeed ZeRO-3 can be used for inference as well since it allows huge models to be loaded on multiple GPUs, which
won’t be possible on a single GPU.`,te,R,fa='Accelerate integrates <a href="https://github.com/deepspeedai/DeepSpeed" rel="nofollow">DeepSpeed</a> via 2 options:',ae,x,ba=`<li>Integration of the DeepSpeed features via <code>deepspeed config file</code> specification in <code>accelerate config</code> . You just supply your custom config file or use our template. Most of
this document is focused on this feature. This supports all the core features of DeepSpeed and gives user a lot of flexibility.
User may have to change a few lines of code depending on the config.</li> <li>Integration via <code>deepspeed_plugin</code>.This supports subset of the DeepSpeed features and uses default options for the rest of the configurations.
User need not change any code and is good for those who are fine with most of the default settings of DeepSpeed.</li>`,ne,E,pe,Q,Ia="Training:",ie,k,Za=`<li>Accelerate integrates all features of DeepSpeed ZeRO. This includes all the ZeRO stages 1, 2 and 3 as well as ZeRO-Offload, ZeRO-Infinity (which can offload to disk/NVMe) and ZeRO++.
Below is a short description of Data Parallelism using ZeRO - Zero Redundancy Optimizer along with diagram from this <a href="https://www.microsoft.com/en-us/research/blog/zero-deepspeed-new-system-optimizations-enable-training-models-with-over-100-billion-parameters/" rel="nofollow">blog post</a> <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/parallelism-zero.png" alt="ZeRO Data Parallelism"/></li>`,ce,F,Ca='(Source: <a href="https://www.microsoft.com/en-us/research/blog/zero-deepspeed-new-system-optimizations-enable-training-models-with-over-100-billion-parameters/" rel="nofollow">link</a>)',oe,H,_a="a. <strong>Stage 1</strong> : Shards optimizer states across data parallel workers/GPUs",Me,q,Ba="b. <strong>Stage 2</strong> : Shards optimizer states + gradients across data parallel workers/GPUs",ue,$,ga="c. <strong>Stage 3</strong>: Shards optimizer states + gradients + model parameters across data parallel workers/GPUs",re,L,Wa="d. <strong>Optimizer Offload</strong>: Offloads the gradients + optimizer states to CPU/Disk building on top of ZERO Stage 2",ye,D,Sa="e. <strong>Param Offload</strong>: Offloads the model parameters to CPU/Disk building on top of ZERO Stage 3",je,P,Xa="f. <strong>Hierarchical Partitioning</strong>: Enables efficient multi-node training with data-parallel training across nodes and ZeRO-3 sharding within a node, built on top of ZeRO Stage 3.",de,O,Na="Note",Je,K,Ga="Inference:",he,ss,va=`<li>DeepSpeed ZeRO Inference supports ZeRO stage 3 with ZeRO-Infinity. It uses the same ZeRO protocol as training, but
it doesn’t use an optimizer and a lr scheduler and only stage 3 is relevant. For more details see:
<a href="#deepspeed-zero-inference">deepspeed-zero-inference</a>.</li>`,we,ls,Te,es,Va=`<strong>Pre-Requisites</strong>: Install DeepSpeed version &gt;=0.6.5. Please refer to the <a href="https://github.com/deepspeedai/DeepSpeed#installation" rel="nofollow">DeepSpeed Installation details</a>
for more information.`,Ue,ts,Ya=`We will first look at easy to use integration via <code>accelerate config</code>.
Followed by more flexible and feature rich <code>deepspeed config file</code> integration.`,me,as,fe,ns,Aa="On your machine(s) just run:",be,ps,Ie,is,za=`and answer the questions asked. It will ask whether you want to use a config file for DeepSpeed to which you should answer no. Then answer the following questions to generate a basic DeepSpeed config.
This will generate a config file that will be used automatically to properly set the
default options when doing`,Ze,cs,Ce,os,Ra="For instance, here is how you would run the NLP example <code>examples/nlp_example.py</code> (from the root of the repo) with DeepSpeed Plugin:",_e,Ms,xa="<strong>ZeRO Stage-2 DeepSpeed Plugin Example</strong>",Be,us,ge,rs,We,ys,Ea="<strong>ZeRO Stage-3 with CPU Offload DeepSpeed Plugin Example</strong>",Se,js,Xe,ds,Ne,Js,Qa="Currently, <code>Accelerate</code> supports following config through the CLI:",Ge,hs,ve,ws,ka="To be able to tweak more options, you will need to use a DeepSpeed config file.",Ve,Ts,Ye,Us,Fa="On your machine(s) just run:",Ae,ms,ze,fs,Ha=`and answer the questions asked. It will ask whether you want to use a config file for deepspeed to which you answer yes
and provide the path to the deepspeed config file.
This will generate a config file that will be used automatically to properly set the
default options when doing`,Re,bs,xe,Is,qa="For instance, here is how you would run the NLP example <code>examples/by_feature/deepspeed_with_config_support.py</code> (from the root of the repo) with DeepSpeed Config File:",Ee,Zs,$a="<strong>ZeRO Stage-2 DeepSpeed Config File Example</strong>",Qe,Cs,ke,_s,La="with the contents of <code>zero_stage2_config.json</code> being:",Fe,Bs,He,gs,qe,Ws,Da="<strong>ZeRO Stage-3 with CPU offload DeepSpeed Config File Example</strong>",$e,Ss,Le,Xs,Pa="with the contents of <code>zero_stage3_offload_config.json</code> being:",De,Ns,Pe,Gs,Oe,vs,Oa=`<strong>ZeRO++ Config Example</strong>
You can use the features of ZeRO++ by using the appropriate config parameters. Note that ZeRO++ is an extension for ZeRO Stage 3. Here is how the config file can be modified, from <a href="https://www.deepspeed.ai/tutorials/zeropp/" rel="nofollow">DeepSpeed’s ZeRO++ tutorial</a>:`,Ke,Vs,st,Ys,Ka="For hierarchical partitioning, the partition size <code>zero_hpz_partition_size</code> should ideally be set to the number of GPUs per node. (For example, the above config file assumes 8 GPUs per node)",lt,As,sn="<strong>Important code changes when using DeepSpeed Config File</strong>",et,I,U,Gl,ln=`DeepSpeed Optimizers and Schedulers. For more information on these,
see the <a href="https://deepspeed.readthedocs.io/en/latest/optimizers.html" rel="nofollow">DeepSpeed Optimizers</a> and <a href="https://deepspeed.readthedocs.io/en/latest/schedulers.html" rel="nofollow">DeepSpeed Schedulers</a> documentation.
We will look at the changes needed in the code when using these.`,Kt,vl,en=`a. DS Optim + DS Scheduler: The case when both <code>optimizer</code> and <code>scheduler</code> keys are present in the DeepSpeed config file.
In this situation, those will be used and the user has to use <code>accelerate.utils.DummyOptim</code> and <code>accelerate.utils.DummyScheduler</code> to replace the PyTorch/Custom optimizers and schedulers in their code.
Below is the snippet from <code>examples/by_feature/deepspeed_with_config_support.py</code> showing this:`,sa,zs,la,Vl,tn=`b. Custom Optim + Custom Scheduler: The case when both <code>optimizer</code> and <code>scheduler</code> keys are absent in the DeepSpeed config file.
In this situation, no code changes are needed from the user and this is the case when using integration via DeepSpeed Plugin.
In the above example we can see that the code remains unchanged if the <code>optimizer</code> and <code>scheduler</code> keys are absent in the DeepSpeed config file.`,ea,Yl,an=`c. Custom Optim + DS Scheduler: The case when only <code>scheduler</code> key is present in the DeepSpeed config file.
In this situation, the user has to use <code>accelerate.utils.DummyScheduler</code> to replace the PyTorch/Custom scheduler in their code.`,ta,Al,nn=`d. DS Optim + Custom Scheduler: The case when only <code>optimizer</code> key is present in the DeepSpeed config file.
This will result in an error because you can only use DS Scheduler when using DS Optim.`,aa,zl,pn=`<p>Notice the <code>auto</code> values in the above example DeepSpeed config files. These are automatically handled by <code>prepare</code> method
based on model, dataloaders, dummy optimizer and dummy schedulers provided to <code>prepare</code> method.
Only the <code>auto</code> fields specified in above examples are handled by <code>prepare</code> method and the rest have to be explicitly specified by the user.</p>`,tt,Rs,cn="The <code>auto</code> values are calculated as:",at,xs,on="<li><code>reduce_bucket_size</code>: <code>hidden_size * hidden_size</code></li> <li><code>stage3_prefetch_bucket_size</code>: <code>int(0.9 * hidden_size * hidden_size)</code></li> <li><code>stage3_param_persistence_threshold</code>: <code>10 * hidden_size</code></li>",nt,Es,Mn="For the <code>auto</code> feature to work for these 3 config entries - Accelerate will use <code>model.config.hidden_size</code> or <code>max(model.config.hidden_sizes)</code> as <code>hidden_size</code>. If neither of these is available, the launching will fail and you will have to set these 3 config entries manually. Remember the first 2 config entries are the communication buffers - the larger they are the more efficient the comms will be, and the larger they are the more GPU memory they will consume, so it’s a tunable performance trade-off.",pt,Qs,un="<strong>Things to note when using DeepSpeed Config File</strong>",it,ks,rn="Below is a sample script using <code>deepspeed_config_file</code> in different scenarios.",ct,Fs,yn="Code <code>test.py</code>:",ot,Hs,Mt,qs,jn="<strong>Scenario 1</strong>: Manually tampered accelerate config file having <code>deepspeed_config_file</code> along with other entries.",ut,$s,dn="<li>Content of the <code>accelerate</code> config:</li>",rt,Ls,yt,Z,Jn="<li><code>ds_config.json</code>:</li>",jt,Ds,dt,C,hn="<li>Output of <code>accelerate launch test.py</code>:</li>",Jt,Ps,ht,Os,wn="<strong>Scenario 2</strong>: Use the solution of the error to create new accelerate config and check that no ambiguity error is now thrown.",wt,Ks,Tn="<li>Run <code>accelerate config</code>:</li>",Tt,sl,Ut,_,Un="<li>Content of the <code>accelerate</code> config:</li>",mt,ll,ft,B,mn="<li>Output of <code>accelerate launch test.py</code>:</li>",bt,el,It,tl,fn="<strong>Scenario 3</strong>: Setting the <code>accelerate launch</code> command arguments related to DeepSpeed as <code>&quot;auto&quot;</code> in the DeepSpeed` configuration file and check that things work as expected.",Zt,al,bn="<li>New <code>ds_config.json</code> with <code>&quot;auto&quot;</code> for the <code>accelerate launch</code> DeepSpeed command arguments:</li>",Ct,nl,_t,g,In="<li>Output of <code>accelerate launch --mixed_precision=&quot;fp16&quot; --zero_stage=3 --gradient_accumulation_steps=5 --gradient_clipping=1.0 --offload_param_device=&quot;cpu&quot; --offload_optimizer_device=&quot;nvme&quot; --zero3_save_16bit_model=&quot;true&quot; test.py</code>:</li>",Bt,pl,gt,il,Zn="<strong>Note</strong>:",Wt,cl,Cn=`<li>Remaining <code>&quot;auto&quot;</code> values are handled in <code>accelerator.prepare()</code> call as explained in point 2 of
<code>Important code changes when using DeepSpeed Config File</code>.</li> <li>Only when <code>gradient_accumulation_steps</code> is <code>auto</code>, the value passed while creating <code>Accelerator</code> object via <code>Accelerator(gradient_accumulation_steps=k)</code> will be used. When using DeepSpeed Plugin, the value from it will be used and it will overwrite the value passed while creating Accelerator object.</li>`,St,ol,Xt,W,Rl,_n="<p>Saving and loading of models is unchanged for ZeRO Stage-1 and Stage-2.</p>",na,h,xl,Bn=`under ZeRO Stage-3, <code>state_dict</code> contains just the placeholders since the model weights are partitioned across multiple GPUs.
ZeRO Stage-3 has 2 options:`,pa,El,gn=`a. Saving the entire 16bit model weights to directly load later on using <code>model.load_state_dict(torch.load(pytorch_model.bin))</code>.
For this, either set <code>zero_optimization.stage3_gather_16bit_weights_on_model_save</code> to True in DeepSpeed Config file or set
<code>zero3_save_16bit_model</code> to True in DeepSpeed Plugin.
<strong>Note that this option requires consolidation of the weights on one GPU it can be slow and memory demanding, so only use this feature when needed.</strong>
Below is the snippet from <code>examples/by_feature/deepspeed_with_config_support.py</code> showing this:`,ia,Ml,ca,Ql,Wn=`b. To get 32bit weights, first save the model using <code>model.save_checkpoint()</code>.
Below is the snippet from <code>examples/by_feature/deepspeed_with_config_support.py</code> showing this:`,oa,ul,Ma,kl,Sn=`This will create ZeRO model and optimizer partitions along with <code>zero_to_fp32.py</code> script in checkpoint directory.
You can use this script to do offline consolidation.
It requires no configuration files or GPUs. Here is an example of its usage:`,ua,rl,ra,Fl,Xn="To get 32bit model for saving/inference, you can perform:",ya,yl,ja,Hl,Nn="If you are only interested in the <code>state_dict</code>, you can do the following:",da,jl,Ja,ql,Gn="Note that all these functions require ~2x memory (general RAM) of the size of the final checkpoint.",Nt,dl,Gt,Jl,vn=`DeepSpeed ZeRO Inference supports ZeRO stage 3 with ZeRO-Infinity.
It uses the same ZeRO protocol as training, but it doesn’t use an optimizer and a lr scheduler and only stage 3 is relevant.
With accelerate integration, you just need to prepare the model and dataloader as shown below:`,vt,hl,Vt,wl,Yt,Tl,Vn="<li>Current integration doesn’t support Pipeline Parallelism of DeepSpeed.</li> <li>Current integration doesn’t support <code>mpu</code>, limiting the tensor parallelism which is supported in Megatron-LM.</li> <li>Current integration doesn’t support multiple models.</li>",At,Ul,zt,ml,Yn="DeepSpeed supports multi-node inference and training over a variety of different launchers. You can specify a different launcher by setting the <code>deepspeed_multinode_launcher</code> config in the CLI or in the DeepSpeed config file.",Rt,fl,An="Currently, accelerate supports passing configuration for the following DeepSpeed multi-node launchers: <code>pdsh</code> (default), <code>standard</code>, <code>openmpi</code>, <code>mvapich</code>, <code>mpich</code>, <code>slurm</code>, <code>nossh</code> (requires DeepSpeed &gt;= 0.14.5).",xt,bl,zn='Please read the <a href="https://www.deepspeed.ai/getting-started/#resource-configuration-multi-node" rel="nofollow">DeepSpeed documentation</a> for more information on the different launchers. By default, DeepSpeed will attempt to use passwordless SSH from the main machine node to the other nodes to perform the launcher command. In this configuration, the accelerate launch command only needs to be run on the main node. If using the <code>nossh</code> launcher, you will need to run the accelerate launch command on every node using copied configuration.',Et,Il,Qt,Zl,Rn='The documentation for the internals related to deepspeed can be found <a href="../package_reference/deepspeed">here</a>.',kt,Cl,xn='<li><a href="https://github.com/deepspeedai/DeepSpeed" rel="nofollow">Project’s github</a></li> <li><a href="https://www.deepspeed.ai/getting-started/" rel="nofollow">Usage docs</a></li> <li><a href="https://deepspeed.readthedocs.io/en/latest/index.html" rel="nofollow">API docs</a></li> <li><a href="https://www.microsoft.com/en-us/research/search/?q=deepspeed" rel="nofollow">Blog posts</a></li>',Ft,_l,En="Papers:",Ht,Bl,Qn='<li><a href="https://arxiv.org/abs/1910.02054" rel="nofollow">ZeRO: Memory Optimizations Toward Training Trillion Parameter Models</a></li> <li><a href="https://arxiv.org/abs/2101.06840" rel="nofollow">ZeRO-Offload: Democratizing Billion-Scale Model Training</a></li> <li><a href="https://arxiv.org/abs/2104.07857" rel="nofollow">ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning</a></li> <li><a href="https://arxiv.org/abs/2306.10209" rel="nofollow">ZeRO++: Extremely Efficient Collective Communication for Giant Model Training</a></li>',qt,gl,kn=`Finally, please, remember that <code>Accelerate</code> only integrates DeepSpeed, therefore if you
have any problems or questions with regards to DeepSpeed usage, please, file an issue with <a href="https://github.com/deepspeedai/DeepSpeed/issues" rel="nofollow">DeepSpeed GitHub</a>.`,$t,S,Lt,Wl,Dt,Ll,Pt;return N=new sp({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),G=new b({props:{title:"DeepSpeed",local:"deepspeed",headingTag:"h1"}}),E=new b({props:{title:"What is integrated?",local:"what-is-integrated",headingTag:"h2"}}),ls=new b({props:{title:"How it works?",local:"how-it-works",headingTag:"h2"}}),as=new b({props:{title:"Accelerate DeepSpeed Plugin",local:"accelerate-deepspeed-plugin",headingTag:"h3"}}),ps=new J({props:{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZw==",highlighted:"accelerate config",wrap:!1}}),cs=new J({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMG15X3NjcmlwdC5weSUyMC0tYXJnc190b19teV9zY3JpcHQ=",highlighted:"accelerate launch my_script.py --args_to_my_script",wrap:!1}}),us=new J({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNBJTIwMSUwQSUyMGdyYWRpZW50X2NsaXBwaW5nJTNBJTIwMS4wJTBBJTIwb2ZmbG9hZF9vcHRpbWl6ZXJfZGV2aWNlJTNBJTIwbm9uZSUwQSUyMG9mZmxvYWRfcGFyYW1fZGV2aWNlJTNBJTIwbm9uZSUwQSUyMHplcm8zX2luaXRfZmxhZyUzQSUyMHRydWUlMEElMjB6ZXJvX3N0YWdlJTNBJTIwMiUwQWRpc3RyaWJ1dGVkX3R5cGUlM0ElMjBERUVQU1BFRUQlMEFmc2RwX2NvbmZpZyUzQSUyMCU3QiU3RCUwQW1hY2hpbmVfcmFuayUzQSUyMDAlMEFtYWluX3Byb2Nlc3NfaXAlM0ElMjBudWxsJTBBbWFpbl9wcm9jZXNzX3BvcnQlM0ElMjBudWxsJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBmcDE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjAyJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`compute_environment: LOCAL_MACHINE
deepspeed_config:
 gradient_accumulation_steps: 1
 gradient_clipping: 1.0
 offload_optimizer_device: none
 offload_param_device: none
 zero3_init_flag: <span class="hljs-literal">true</span>
 zero_stage: 2
distributed_type: DEEPSPEED
fsdp_config: {}
machine_rank: 0
main_process_ip: null
main_process_port: null
main_training_function: main
mixed_precision: fp16
num_machines: 1
num_processes: 2
use_cpu: <span class="hljs-literal">false</span>`,wrap:!1}}),rs=new J({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMGV4YW1wbGVzJTJGbmxwX2V4YW1wbGUucHklMjAtLW1peGVkX3ByZWNpc2lvbiUyMGZwMTY=",highlighted:"accelerate launch examples/nlp_example.py --mixed_precision fp16",wrap:!1}}),js=new J({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwJTIwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTNBJTIwMSUwQSUyMCUyMGdyYWRpZW50X2NsaXBwaW5nJTNBJTIwMS4wJTBBJTIwJTIwb2ZmbG9hZF9vcHRpbWl6ZXJfZGV2aWNlJTNBJTIwY3B1JTBBJTIwJTIwb2ZmbG9hZF9wYXJhbV9kZXZpY2UlM0ElMjBjcHUlMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBJTIwJTIwemVybzNfc2F2ZV8xNmJpdF9tb2RlbCUzQSUyMHRydWUlMEElMjAlMjB6ZXJvX3N0YWdlJTNBJTIwMyUwQWRpc3RyaWJ1dGVkX3R5cGUlM0ElMjBERUVQU1BFRUQlMEFmc2RwX2NvbmZpZyUzQSUyMCU3QiU3RCUwQW1hY2hpbmVfcmFuayUzQSUyMDAlMEFtYWluX3Byb2Nlc3NfaXAlM0ElMjBudWxsJTBBbWFpbl9wcm9jZXNzX3BvcnQlM0ElMjBudWxsJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBmcDE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjAyJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`compute_environment: LOCAL_MACHINE
deepspeed_config:
  gradient_accumulation_steps: 1
  gradient_clipping: 1.0
  offload_optimizer_device: cpu
  offload_param_device: cpu
  zero3_init_flag: <span class="hljs-literal">true</span>
  zero3_save_16bit_model: <span class="hljs-literal">true</span>
  zero_stage: 3
distributed_type: DEEPSPEED
fsdp_config: {}
machine_rank: 0
main_process_ip: null
main_process_port: null
main_training_function: main
mixed_precision: fp16
num_machines: 1
num_processes: 2
use_cpu: <span class="hljs-literal">false</span>`,wrap:!1}}),ds=new J({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMGV4YW1wbGVzJTJGbmxwX2V4YW1wbGUucHklMjAtLW1peGVkX3ByZWNpc2lvbiUyMGZwMTY=",highlighted:"accelerate launch examples/nlp_example.py --mixed_precision fp16",wrap:!1}}),hs=new J({props:{code:"JTYwemVyb19zdGFnZSU2MCUzQSUyMCU1QjAlNUQlMjBEaXNhYmxlZCUyQyUyMCU1QjElNUQlMjBvcHRpbWl6ZXIlMjBzdGF0ZSUyMHBhcnRpdGlvbmluZyUyQyUyMCU1QjIlNUQlMjBvcHRpbWl6ZXIlMkJncmFkaWVudCUyMHN0YXRlJTIwcGFydGl0aW9uaW5nJTIwYW5kJTIwJTVCMyU1RCUyMG9wdGltaXplciUyQmdyYWRpZW50JTJCcGFyYW1ldGVyJTIwcGFydGl0aW9uaW5nJTBBJTYwZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTYwJTNBJTIwTnVtYmVyJTIwb2YlMjB0cmFpbmluZyUyMHN0ZXBzJTIwdG8lMjBhY2N1bXVsYXRlJTIwZ3JhZGllbnRzJTIwYmVmb3JlJTIwYXZlcmFnaW5nJTIwYW5kJTIwYXBwbHlpbmclMjB0aGVtLiUwQSU2MGdyYWRpZW50X2NsaXBwaW5nJTYwJTNBJTIwRW5hYmxlJTIwZ3JhZGllbnQlMjBjbGlwcGluZyUyMHdpdGglMjB2YWx1ZS4lMEElNjBvZmZsb2FkX29wdGltaXplcl9kZXZpY2UlNjAlM0ElMjAlNUJub25lJTVEJTIwRGlzYWJsZSUyMG9wdGltaXplciUyMG9mZmxvYWRpbmclMkMlMjAlNUJjcHUlNUQlMjBvZmZsb2FkJTIwb3B0aW1pemVyJTIwdG8lMjBDUFUlMkMlMjAlNUJudm1lJTVEJTIwb2ZmbG9hZCUyMG9wdGltaXplciUyMHRvJTIwTlZNZSUyMFNTRC4lMjBPbmx5JTIwYXBwbGljYWJsZSUyMHdpdGglMjBaZVJPJTIwJTNFJTNEJTIwU3RhZ2UtMi4lMEElNjBvZmZsb2FkX29wdGltaXplcl9udm1lX3BhdGglNjAlM0ElMjBEZWNpZGVzJTIwTnZtZSUyMFBhdGglMjB0byUyMG9mZmxvYWQlMjBvcHRpbWl6ZXIlMjBzdGF0ZXMuJTIwSWYlMjB1bnNwZWNpZmllZCUyQyUyMHdpbGwlMjBkZWZhdWx0JTIwdG8lMjAnbm9uZScuJTBBJTYwb2ZmbG9hZF9wYXJhbV9kZXZpY2UlNjAlM0ElMjAlNUJub25lJTVEJTIwRGlzYWJsZSUyMHBhcmFtZXRlciUyMG9mZmxvYWRpbmclMkMlMjAlNUJjcHUlNUQlMjBvZmZsb2FkJTIwcGFyYW1ldGVycyUyMHRvJTIwQ1BVJTJDJTIwJTVCbnZtZSU1RCUyMG9mZmxvYWQlMjBwYXJhbWV0ZXJzJTIwdG8lMjBOVk1lJTIwU1NELiUyME9ubHklMjBhcHBsaWNhYmxlJTIwd2l0aCUyMFplUk8lMjBTdGFnZS0zLiUwQSU2MG9mZmxvYWRfcGFyYW1fbnZtZV9wYXRoJTYwJTNBJTIwRGVjaWRlcyUyME52bWUlMjBQYXRoJTIwdG8lMjBvZmZsb2FkJTIwcGFyYW1ldGVycy4lMjBJZiUyMHVuc3BlY2lmaWVkJTJDJTIwd2lsbCUyMGRlZmF1bHQlMjB0byUyMCdub25lJy4lMEElNjB6ZXJvM19pbml0X2ZsYWclNjAlM0ElMjBEZWNpZGVzJTIwd2hldGhlciUyMHRvJTIwZW5hYmxlJTIwJTYwZGVlcHNwZWVkLnplcm8uSW5pdCU2MCUyMGZvciUyMGNvbnN0cnVjdGluZyUyMG1hc3NpdmUlMjBtb2RlbHMuJTIwT25seSUyMGFwcGxpY2FibGUlMjB3aXRoJTIwWmVSTyUyMFN0YWdlLTMuJTBBJTYwemVybzNfc2F2ZV8xNmJpdF9tb2RlbCU2MCUzQSUyMERlY2lkZXMlMjB3aGV0aGVyJTIwdG8lMjBzYXZlJTIwMTYtYml0JTIwbW9kZWwlMjB3ZWlnaHRzJTIwd2hlbiUyMHVzaW5nJTIwWmVSTyUyMFN0YWdlLTMuJTBBJTYwbWl4ZWRfcHJlY2lzaW9uJTYwJTNBJTIwJTYwbm8lNjAlMjBmb3IlMjBGUDMyJTIwdHJhaW5pbmclMkMlMjAlNjBmcDE2JTYwJTIwZm9yJTIwRlAxNiUyMG1peGVkLXByZWNpc2lvbiUyMHRyYWluaW5nJTIwYW5kJTIwJTYwYmYxNiU2MCUyMGZvciUyMEJGMTYlMjBtaXhlZC1wcmVjaXNpb24lMjB0cmFpbmluZy4lMEElNjBkZWVwc3BlZWRfbW9lX2xheWVyX2Nsc19uYW1lcyU2MCUzQSUyMENvbW1hLXNlcGFyYXRlZCUyMGxpc3QlMjBvZiUyMHRyYW5zZm9ybWVyJTIwTWl4dHVyZS1vZi1FeHBlcnRzJTIwKE1vRSklMjBsYXllciUyMGNsYXNzJTIwbmFtZXMlMjAoY2FzZS1zZW5zaXRpdmUpJTIwdG8lMjB3cmFwJTIwJTJDZS5nJTJDJTIwJTYwTWl4dHJhbFNwYXJzZU1vZUJsb2NrJTYwJTJDJTIwJTYwUXdlbjJNb2VTcGFyc2VNb2VCbG9jayU2MCUyQyUyMCU2MEpldE1vRUF0dGVudGlvbiUyQ0pldE1vRUJsb2NrJTYwJTIwLi4uJTBBJTYwZGVlcHNwZWVkX2hvc3RmaWxlJTYwJTNBJTIwRGVlcFNwZWVkJTIwaG9zdGZpbGUlMjBmb3IlMjBjb25maWd1cmluZyUyMG11bHRpLW5vZGUlMjBjb21wdXRlJTIwcmVzb3VyY2VzLiUwQSU2MGRlZXBzcGVlZF9leGNsdXNpb25fZmlsdGVyJTYwJTNBJTIwRGVlcFNwZWVkJTIwZXhjbHVzaW9uJTIwZmlsdGVyJTIwc3RyaW5nJTIwd2hlbiUyMHVzaW5nJTIwbXV0bGktbm9kZSUyMHNldHVwLiUwQSU2MGRlZXBzcGVlZF9pbmNsdXNpb25fZmlsdGVyJTYwJTNBJTIwRGVlcFNwZWVkJTIwaW5jbHVzaW9uJTIwZmlsdGVyJTIwc3RyaW5nJTIwd2hlbiUyMHVzaW5nJTIwbXV0bGktbm9kZSUyMHNldHVwLiUwQSU2MGRlZXBzcGVlZF9tdWx0aW5vZGVfbGF1bmNoZXIlNjAlM0ElMjBEZWVwU3BlZWQlMjBtdWx0aS1ub2RlJTIwbGF1bmNoZXIlMjB0byUyMHVzZSUyQyUyMGUuZy4lMjAlNjBwZHNoJTYwJTJDJTIwJTYwc3RhbmRhcmQlNjAlMkMlMjAlNjBvcGVubXBpJTYwJTJDJTIwJTYwbXZhcGljaCU2MCUyQyUyMCU2MG1waWNoJTYwJTJDJTIwJTYwc2x1cm0lNjAlMkMlMjAlNjBub3NzaCU2MCUyMChyZXF1aXJlcyUyMERlZXBTcGVlZCUyMCUzRSUzRCUyMDAuMTQuNSkuJTIwSWYlMjB1bnNwZWNpZmllZCUyQyUyMHdpbGwlMjBkZWZhdWx0JTIwdG8lMjAlNjBwZHNoJTYwLiUwQSU2MGRlZXBzcGVlZF9jb25maWdfZmlsZSU2MCUzQSUyMHBhdGglMjB0byUyMHRoZSUyMERlZXBTcGVlZCUyMGNvbmZpZyUyMGZpbGUlMjBpbiUyMCU2MGpzb24lNjAlMjBmb3JtYXQuJTIwU2VlJTIwdGhlJTIwbmV4dCUyMHNlY3Rpb24lMjBmb3IlMjBtb3JlJTIwZGV0YWlscyUyMG9uJTIwdGhpcy4=",highlighted:'`zero_stage`: [0] Disabled, [1] optimizer state partitioning, [2] optimizer+gradient state partitioning and [3] optimizer+gradient+parameter partitioning\n`gradient_accumulation_steps`: Number of training steps to accumulate gradients before averaging and applying them.\n`gradient_clipping`: Enable gradient clipping with value.\n`offload_optimizer_device`: [none] Disable optimizer offloading, [cpu] offload optimizer to CPU, [nvme] offload optimizer to NVMe SSD. Only applicable with ZeRO &gt;= Stage-2.\n`offload_optimizer_nvme_path`: Decides Nvme Path to offload optimizer states. If unspecified, will default to <span class="hljs-string">&#x27;none&#x27;</span>.\n`offload_param_device`: [none] Disable parameter offloading, [cpu] offload parameters to CPU, [nvme] offload parameters to NVMe SSD. Only applicable with ZeRO Stage-3.\n`offload_param_nvme_path`: Decides Nvme Path to offload parameters. If unspecified, will default to <span class="hljs-string">&#x27;none&#x27;</span>.\n`zero3_init_flag`: Decides whether to <span class="hljs-built_in">enable</span> `deepspeed.zero.Init` <span class="hljs-keyword">for</span> constructing massive models. Only applicable with ZeRO Stage-3.\n`zero3_save_16bit_model`: Decides whether to save 16-bit model weights when using ZeRO Stage-3.\n`mixed_precision`: `no` <span class="hljs-keyword">for</span> FP32 training, `fp16` <span class="hljs-keyword">for</span> FP16 mixed-precision training and `bf16` <span class="hljs-keyword">for</span> BF16 mixed-precision training.\n`deepspeed_moe_layer_cls_names`: Comma-separated list of transformer Mixture-of-Experts (MoE) layer class names (case-sensitive) to wrap ,e.g, `MixtralSparseMoeBlock`, `Qwen2MoeSparseMoeBlock`, `JetMoEAttention,JetMoEBlock` ...\n`deepspeed_hostfile`: DeepSpeed hostfile <span class="hljs-keyword">for</span> configuring multi-node compute resources.\n`deepspeed_exclusion_filter`: DeepSpeed exclusion filter string when using mutli-node setup.\n`deepspeed_inclusion_filter`: DeepSpeed inclusion filter string when using mutli-node setup.\n`deepspeed_multinode_launcher`: DeepSpeed multi-node launcher to use, e.g. `pdsh`, `standard`, `openmpi`, `mvapich`, `mpich`, `slurm`, `nossh` (requires DeepSpeed &gt;= 0.14.5). If unspecified, will default to `pdsh`.\n`deepspeed_config_file`: path to the DeepSpeed config file <span class="hljs-keyword">in</span> `json` format. See the next section <span class="hljs-keyword">for</span> more details on this.',wrap:!1}}),Ts=new b({props:{title:"DeepSpeed Config File",local:"deepspeed-config-file",headingTag:"h3"}}),ms=new J({props:{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZw==",highlighted:"accelerate config",wrap:!1}}),bs=new J({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMG15X3NjcmlwdC5weSUyMC0tYXJnc190b19teV9zY3JpcHQ=",highlighted:"accelerate launch my_script.py --args_to_my_script",wrap:!1}}),Cs=new J({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwZGVlcHNwZWVkX2NvbmZpZ19maWxlJTNBJTIwJTJGaG9tZSUyRnVidW50dSUyRmFjY2VsZXJhdGUlMkZleGFtcGxlcyUyRmRlZXBzcGVlZF9jb25maWdfdGVtcGxhdGVzJTJGemVyb19zdGFnZTJfY29uZmlnLmpzb24lMEElMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBZGlzdHJpYnV0ZWRfdHlwZSUzQSUyMERFRVBTUEVFRCUwQWZzZHBfY29uZmlnJTNBJTIwJTdCJTdEJTBBbWFjaGluZV9yYW5rJTNBJTIwMCUwQW1haW5fcHJvY2Vzc19pcCUzQSUyMG51bGwlMEFtYWluX3Byb2Nlc3NfcG9ydCUzQSUyMG51bGwlMEFtYWluX3RyYWluaW5nX2Z1bmN0aW9uJTNBJTIwbWFpbiUwQW1peGVkX3ByZWNpc2lvbiUzQSUyMGZwMTYlMEFudW1fbWFjaGluZXMlM0ElMjAxJTBBbnVtX3Byb2Nlc3NlcyUzQSUyMDIlMEF1c2VfY3B1JTNBJTIwZmFsc2U=",highlighted:`compute_environment: LOCAL_MACHINE
deepspeed_config:
 deepspeed_config_file: /home/ubuntu/accelerate/examples/deepspeed_config_templates/zero_stage2_config.json
 zero3_init_flag: <span class="hljs-literal">true</span>
distributed_type: DEEPSPEED
fsdp_config: {}
machine_rank: 0
main_process_ip: null
main_process_port: null
main_training_function: main
mixed_precision: fp16
num_machines: 1
num_processes: 2
use_cpu: <span class="hljs-literal">false</span>`,wrap:!1}}),Bs=new J({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyZnAxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybG9zc19zY2FsZSUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJsb3NzX3NjYWxlX3dpbmRvdyUyMiUzQSUyMDEwMDAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJpbml0aWFsX3NjYWxlX3Bvd2VyJTIyJTNBJTIwMTYlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJoeXN0ZXJlc2lzJTIyJTNBJTIwMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1pbl9sb3NzX3NjYWxlJTIyJTNBJTIwMSUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMm9wdGltaXplciUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnR5cGUlMjIlM0ElMjAlMjJBZGFtVyUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBhcmFtcyUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmxyJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMndlaWdodF9kZWNheSUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b3JjaF9hZGFtJTIyJTNBJTIwdHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmFkYW1fd19tb2RlJTIyJTNBJTIwdHJ1ZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMnNjaGVkdWxlciUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnR5cGUlMjIlM0ElMjAlMjJXYXJtdXBEZWNheUxSJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGFyYW1zJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyd2FybXVwX21pbl9sciUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ3YXJtdXBfbWF4X2xyJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMndhcm11cF9udW1fc3RlcHMlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydG90YWxfbnVtX3N0ZXBzJTIyJTNBJTIwJTIyYXV0byUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMnplcm9fb3B0aW1pemF0aW9uJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyc3RhZ2UlMjIlM0ElMjAyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWxsZ2F0aGVyX3BhcnRpdGlvbnMlMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWxsZ2F0aGVyX2J1Y2tldF9zaXplJTIyJTNBJTIwMmU4JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyb3ZlcmxhcF9jb21tJTIyJTNBJTIwdHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJlZHVjZV9zY2F0dGVyJTIyJTNBJTIwdHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJlZHVjZV9idWNrZXRfc2l6ZSUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJjb250aWd1b3VzX2dyYWRpZW50cyUyMiUzQSUyMHRydWUlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjJncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMlMjIlM0ElMjAxJTJDJTBBJTIwJTIwJTIwJTIwJTIyZ3JhZGllbnRfY2xpcHBpbmclMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyc3RlcHNfcGVyX3ByaW50JTIyJTNBJTIwMjAwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYWluX2JhdGNoX3NpemUlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMndhbGxfY2xvY2tfYnJlYWtkb3duJTIyJTNBJTIwZmFsc2UlMEElN0Q=",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;fp16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;loss_scale&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">0</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;loss_scale_window&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1000</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;initial_scale_power&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">16</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;hysteresis&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;min_loss_scale&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;AdamW&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;weight_decay&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;torch_adam&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;adam_w_mode&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;scheduler&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;WarmupDecayLR&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;warmup_min_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_max_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_num_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;total_num_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;allgather_partitions&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;allgather_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2e8</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;overlap_comm&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;reduce_scatter&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;reduce_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;contiguous_gradients&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_accumulation_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_clipping&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;steps_per_print&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2000</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_batch_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;wall_clock_breakdown&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">false</span></span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),gs=new J({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMGV4YW1wbGVzJTJGYnlfZmVhdHVyZSUyRmRlZXBzcGVlZF93aXRoX2NvbmZpZ19zdXBwb3J0LnB5JTIwJTVDJTBBLS1jb25maWdfbmFtZSUyMCUyMmdwdDItbGFyZ2UlMjIlMjAlNUMlMEEtLXRva2VuaXplcl9uYW1lJTIwJTIyZ3B0Mi1sYXJnZSUyMiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwJTIyd2lraXRleHQlMjIlMjAlNUMlMEEtLWRhdGFzZXRfY29uZmlnX25hbWUlMjAlMjJ3aWtpdGV4dC0yLXJhdy12MSUyMiUyMCU1QyUwQS0tYmxvY2tfc2l6ZSUyMDEyOCUyMCU1QyUwQS0tb3V0cHV0X2RpciUyMCUyMi4lMkZjbG0lMkZjbG1fZGVlcHNwZWVkX3N0YWdlMl9hY2NlbGVyYXRlJTIyJTIwJTVDJTBBLS1sZWFybmluZ19yYXRlJTIwNWUtNCUyMCU1QyUwQS0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwMjQlMjAlNUMlMEEtLXBlcl9kZXZpY2VfZXZhbF9iYXRjaF9zaXplJTIwMjQlMjAlNUMlMEEtLW51bV90cmFpbl9lcG9jaHMlMjAzJTIwJTVDJTBBLS13aXRoX3RyYWNraW5nJTIwJTVDJTBBLS1yZXBvcnRfdG8lMjAlMjJ3YW5kYiUyMiU1Qw==",highlighted:`accelerate launch examples/by_feature/deepspeed_with_config_support.py \\
--config_name <span class="hljs-string">&quot;gpt2-large&quot;</span> \\
--tokenizer_name <span class="hljs-string">&quot;gpt2-large&quot;</span> \\
--dataset_name <span class="hljs-string">&quot;wikitext&quot;</span> \\
--dataset_config_name <span class="hljs-string">&quot;wikitext-2-raw-v1&quot;</span> \\
--block_size 128 \\
--output_dir <span class="hljs-string">&quot;./clm/clm_deepspeed_stage2_accelerate&quot;</span> \\
--learning_rate 5e-4 \\
--per_device_train_batch_size 24 \\
--per_device_eval_batch_size 24 \\
--num_train_epochs 3 \\
--with_tracking \\
--report_to <span class="hljs-string">&quot;wandb&quot;</span>\\`,wrap:!1}}),Ss=new J({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwZGVlcHNwZWVkX2NvbmZpZ19maWxlJTNBJTIwJTJGaG9tZSUyRnVidW50dSUyRmFjY2VsZXJhdGUlMkZleGFtcGxlcyUyRmRlZXBzcGVlZF9jb25maWdfdGVtcGxhdGVzJTJGemVyb19zdGFnZTNfb2ZmbG9hZF9jb25maWcuanNvbiUwQSUyMHplcm8zX2luaXRfZmxhZyUzQSUyMHRydWUlMEFkaXN0cmlidXRlZF90eXBlJTNBJTIwREVFUFNQRUVEJTBBZnNkcF9jb25maWclM0ElMjAlN0IlN0QlMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl9wcm9jZXNzX2lwJTNBJTIwbnVsbCUwQW1haW5fcHJvY2Vzc19wb3J0JTNBJTIwbnVsbCUwQW1haW5fdHJhaW5pbmdfZnVuY3Rpb24lM0ElMjBtYWluJTBBbWl4ZWRfcHJlY2lzaW9uJTNBJTIwZnAxNiUwQW51bV9tYWNoaW5lcyUzQSUyMDElMEFudW1fcHJvY2Vzc2VzJTNBJTIwMiUwQXVzZV9jcHUlM0ElMjBmYWxzZQ==",highlighted:`compute_environment: LOCAL_MACHINE
deepspeed_config:
 deepspeed_config_file: /home/ubuntu/accelerate/examples/deepspeed_config_templates/zero_stage3_offload_config.json
 zero3_init_flag: <span class="hljs-literal">true</span>
distributed_type: DEEPSPEED
fsdp_config: {}
machine_rank: 0
main_process_ip: null
main_process_port: null
main_training_function: main
mixed_precision: fp16
num_machines: 1
num_processes: 2
use_cpu: <span class="hljs-literal">false</span>`,wrap:!1}}),Ns=new J({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyZnAxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybG9zc19zY2FsZSUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJsb3NzX3NjYWxlX3dpbmRvdyUyMiUzQSUyMDEwMDAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJpbml0aWFsX3NjYWxlX3Bvd2VyJTIyJTNBJTIwMTYlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJoeXN0ZXJlc2lzJTIyJTNBJTIwMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1pbl9sb3NzX3NjYWxlJTIyJTNBJTIwMSUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMm9wdGltaXplciUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnR5cGUlMjIlM0ElMjAlMjJBZGFtVyUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBhcmFtcyUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmxyJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMndlaWdodF9kZWNheSUyMiUzQSUyMCUyMmF1dG8lMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjJzY2hlZHVsZXIlMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0eXBlJTIyJTNBJTIwJTIyV2FybXVwRGVjYXlMUiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBhcmFtcyUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMndhcm11cF9taW5fbHIlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyd2FybXVwX21heF9sciUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ3YXJtdXBfbnVtX3N0ZXBzJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRvdGFsX251bV9zdGVwcyUyMiUzQSUyMCUyMmF1dG8lMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjJ6ZXJvX29wdGltaXphdGlvbiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlJTIyJTNBJTIwMyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm9mZmxvYWRfb3B0aW1pemVyJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZGV2aWNlJTIyJTNBJTIwJTIyY3B1JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGluX21lbW9yeSUyMiUzQSUyMHRydWUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvZmZsb2FkX3BhcmFtJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZGV2aWNlJTIyJTNBJTIwJTIyY3B1JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGluX21lbW9yeSUyMiUzQSUyMHRydWUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvdmVybGFwX2NvbW0lMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGlndW91c19ncmFkaWVudHMlMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycmVkdWNlX2J1Y2tldF9zaXplJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19wcmVmZXRjaF9idWNrZXRfc2l6ZSUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZTNfcGFyYW1fcGVyc2lzdGVuY2VfdGhyZXNob2xkJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN1Yl9ncm91cF9zaXplJTIyJTNBJTIwMWU5JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyc3RhZ2UzX21heF9saXZlX3BhcmFtZXRlcnMlMjIlM0ElMjAxZTklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZTNfbWF4X3JldXNlX2Rpc3RhbmNlJTIyJTNBJTIwMWU5JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyc3RhZ2UzX2dhdGhlcl8xNmJpdF93ZWlnaHRzX29uX21vZGVsX3NhdmUlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIyJTNBJTIwMSUyQyUwQSUyMCUyMCUyMCUyMCUyMmdyYWRpZW50X2NsaXBwaW5nJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnN0ZXBzX3Blcl9wcmludCUyMiUzQSUyMDIwMDAlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFpbl9iYXRjaF9zaXplJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYWluX21pY3JvX2JhdGNoX3NpemVfcGVyX2dwdSUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJ3YWxsX2Nsb2NrX2JyZWFrZG93biUyMiUzQSUyMGZhbHNlJTBBJTdE",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;fp16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;loss_scale&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">0</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;loss_scale_window&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1000</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;initial_scale_power&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">16</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;hysteresis&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;min_loss_scale&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;AdamW&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;weight_decay&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;scheduler&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;WarmupDecayLR&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;warmup_min_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_max_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_num_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;total_num_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">3</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;offload_optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;device&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;cpu&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;pin_memory&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
        <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;offload_param&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;device&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;cpu&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;pin_memory&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
        <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;overlap_comm&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;contiguous_gradients&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;reduce_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_prefetch_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_param_persistence_threshold&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;sub_group_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1e9</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_max_live_parameters&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1e9</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_max_reuse_distance&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1e9</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_gather_16bit_weights_on_model_save&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_accumulation_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_clipping&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;steps_per_print&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2000</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_batch_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;wall_clock_breakdown&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">false</span></span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),Gs=new J({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMGV4YW1wbGVzJTJGYnlfZmVhdHVyZSUyRmRlZXBzcGVlZF93aXRoX2NvbmZpZ19zdXBwb3J0LnB5JTIwJTVDJTBBLS1jb25maWdfbmFtZSUyMCUyMmdwdDItbGFyZ2UlMjIlMjAlNUMlMEEtLXRva2VuaXplcl9uYW1lJTIwJTIyZ3B0Mi1sYXJnZSUyMiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwJTIyd2lraXRleHQlMjIlMjAlNUMlMEEtLWRhdGFzZXRfY29uZmlnX25hbWUlMjAlMjJ3aWtpdGV4dC0yLXJhdy12MSUyMiUyMCU1QyUwQS0tYmxvY2tfc2l6ZSUyMDEyOCUyMCU1QyUwQS0tb3V0cHV0X2RpciUyMCUyMi4lMkZjbG0lMkZjbG1fZGVlcHNwZWVkX3N0YWdlM19vZmZsb2FkX2FjY2VsZXJhdGUlMjIlMjAlNUMlMEEtLWxlYXJuaW5nX3JhdGUlMjA1ZS00JTIwJTVDJTBBLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjAzMiUyMCU1QyUwQS0tcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlMjAzMiUyMCU1QyUwQS0tbnVtX3RyYWluX2Vwb2NocyUyMDMlMjAlNUMlMEEtLXdpdGhfdHJhY2tpbmclMjAlNUMlMEEtLXJlcG9ydF90byUyMCUyMndhbmRiJTIyJTVD",highlighted:`accelerate launch examples/by_feature/deepspeed_with_config_support.py \\
--config_name <span class="hljs-string">&quot;gpt2-large&quot;</span> \\
--tokenizer_name <span class="hljs-string">&quot;gpt2-large&quot;</span> \\
--dataset_name <span class="hljs-string">&quot;wikitext&quot;</span> \\
--dataset_config_name <span class="hljs-string">&quot;wikitext-2-raw-v1&quot;</span> \\
--block_size 128 \\
--output_dir <span class="hljs-string">&quot;./clm/clm_deepspeed_stage3_offload_accelerate&quot;</span> \\
--learning_rate 5e-4 \\
--per_device_train_batch_size 32 \\
--per_device_eval_batch_size 32 \\
--num_train_epochs 3 \\
--with_tracking \\
--report_to <span class="hljs-string">&quot;wandb&quot;</span>\\`,wrap:!1}}),Vs=new J({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyemVyb19vcHRpbWl6YXRpb24lMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZSUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZWR1Y2VfYnVja2V0X3NpemUlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyemVyb19xdWFudGl6ZWRfd2VpZ2h0cyUyMiUzQSUyMHRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ6ZXJvX2hwel9wYXJ0aXRpb25fc2l6ZSUyMiUzQSUyMDglMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ6ZXJvX3F1YW50aXplZF9ncmFkaWVudHMlMjIlM0ElMjB0cnVlJTJDJTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGlndW91c19ncmFkaWVudHMlMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyb3ZlcmxhcF9jb21tJTIyJTNBJTIwdHJ1ZSUwQSUyMCUyMCUyMCUyMCU3RCUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">3</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;reduce_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>

        <span class="hljs-attr">&quot;zero_quantized_weights&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;zero_hpz_partition_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">8</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;zero_quantized_gradients&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>

        <span class="hljs-attr">&quot;contiguous_gradients&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;overlap_comm&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
    <span class="hljs-punctuation">}</span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),zs=new J({props:{code:"JTIwJTIzJTIwQ3JlYXRlcyUyMER1bW15JTIwT3B0aW1pemVyJTIwaWYlMjAlNjBvcHRpbWl6ZXIlNjAlMjB3YXMlMjBzcGVjaWZpZWQlMjBpbiUyMHRoZSUyMGNvbmZpZyUyMGZpbGUlMjBlbHNlJTIwY3JlYXRlcyUyMEFkYW0lMjBPcHRpbWl6ZXIlMEElMjBvcHRpbWl6ZXJfY2xzJTIwJTNEJTIwKCUwQSUyMCUyMCUyMCUyMCUyMHRvcmNoLm9wdGltLkFkYW1XJTBBJTIwJTIwJTIwJTIwJTIwaWYlMjBhY2NlbGVyYXRvci5zdGF0ZS5kZWVwc3BlZWRfcGx1Z2luJTIwaXMlMjBOb25lJTBBJTIwJTIwJTIwJTIwJTIwb3IlMjAlMjJvcHRpbWl6ZXIlMjIlMjBub3QlMjBpbiUyMGFjY2VsZXJhdG9yLnN0YXRlLmRlZXBzcGVlZF9wbHVnaW4uZGVlcHNwZWVkX2NvbmZpZyUwQSUyMCUyMCUyMCUyMCUyMGVsc2UlMjBEdW1teU9wdGltJTBBJTIwKSUwQSUyMG9wdGltaXplciUyMCUzRCUyMG9wdGltaXplcl9jbHMob3B0aW1pemVyX2dyb3VwZWRfcGFyYW1ldGVycyUyQyUyMGxyJTNEYXJncy5sZWFybmluZ19yYXRlKSUwQSUwQSUyMCUyMyUyMENyZWF0ZXMlMjBEdW1teSUyMFNjaGVkdWxlciUyMGlmJTIwJTYwc2NoZWR1bGVyJTYwJTIwd2FzJTIwc3BlY2lmaWVkJTIwaW4lMjB0aGUlMjBjb25maWclMjBmaWxlJTIwZWxzZSUyMGNyZWF0ZXMlMjAlNjBhcmdzLmxyX3NjaGVkdWxlcl90eXBlJTYwJTIwU2NoZWR1bGVyJTBBJTIwaWYlMjAoJTBBJTIwJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3Iuc3RhdGUuZGVlcHNwZWVkX3BsdWdpbiUyMGlzJTIwTm9uZSUwQSUyMCUyMCUyMCUyMCUyMG9yJTIwJTIyc2NoZWR1bGVyJTIyJTIwbm90JTIwaW4lMjBhY2NlbGVyYXRvci5zdGF0ZS5kZWVwc3BlZWRfcGx1Z2luLmRlZXBzcGVlZF9jb25maWclMEElMjApJTNBJTBBJTIwJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyJTIwJTNEJTIwZ2V0X3NjaGVkdWxlciglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBuYW1lJTNEYXJncy5scl9zY2hlZHVsZXJfdHlwZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplciUzRG9wdGltaXplciUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG51bV93YXJtdXBfc3RlcHMlM0RhcmdzLm51bV93YXJtdXBfc3RlcHMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBudW1fdHJhaW5pbmdfc3RlcHMlM0RhcmdzLm1heF90cmFpbl9zdGVwcyUyQyUwQSUyMCUyMCUyMCUyMCUyMCklMEElMjBlbHNlJTNBJTBBJTIwJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyJTIwJTNEJTIwRHVtbXlTY2hlZHVsZXIoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyJTJDJTIwdG90YWxfbnVtX3N0ZXBzJTNEYXJncy5tYXhfdHJhaW5fc3RlcHMlMkMlMjB3YXJtdXBfbnVtX3N0ZXBzJTNEYXJncy5udW1fd2FybXVwX3N0ZXBzJTBBJTIwJTIwJTIwJTIwJTIwKQ==",highlighted:` <span class="hljs-comment"># Creates Dummy Optimizer if \`optimizer\` was specified in the config file else creates Adam Optimizer</span>
 optimizer_cls = (
     torch.optim.AdamW
     <span class="hljs-keyword">if</span> accelerator.state.deepspeed_plugin <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>
     <span class="hljs-keyword">or</span> <span class="hljs-string">&quot;optimizer&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> accelerator.state.deepspeed_plugin.deepspeed_config
     <span class="hljs-keyword">else</span> DummyOptim
 )
 optimizer = optimizer_cls(optimizer_grouped_parameters, lr=args.learning_rate)

 <span class="hljs-comment"># Creates Dummy Scheduler if \`scheduler\` was specified in the config file else creates \`args.lr_scheduler_type\` Scheduler</span>
 <span class="hljs-keyword">if</span> (
     accelerator.state.deepspeed_plugin <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>
     <span class="hljs-keyword">or</span> <span class="hljs-string">&quot;scheduler&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> accelerator.state.deepspeed_plugin.deepspeed_config
 ):
     lr_scheduler = get_scheduler(
         name=args.lr_scheduler_type,
         optimizer=optimizer,
         num_warmup_steps=args.num_warmup_steps,
         num_training_steps=args.max_train_steps,
     )
 <span class="hljs-keyword">else</span>:
     lr_scheduler = DummyScheduler(
         optimizer, total_num_steps=args.max_train_steps, warmup_num_steps=args.num_warmup_steps
     )`,wrap:!1}}),Hs=new J({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQWZyb20lMjBhY2NlbGVyYXRlLnN0YXRlJTIwaW1wb3J0JTIwQWNjZWxlcmF0b3JTdGF0ZSUwQSUwQSUwQWRlZiUyMG1haW4oKSUzQSUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoKSUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLnByaW50KGYlMjIlN0JBY2NlbGVyYXRvclN0YXRlKCklN0QlMjIpJTBBJTBBJTBBaWYlMjBfX25hbWVfXyUyMCUzRCUzRCUyMCUyMl9fbWFpbl9fJTIyJTNBJTBBJTIwJTIwJTIwJTIwbWFpbigp",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator
<span class="hljs-keyword">from</span> accelerate.state <span class="hljs-keyword">import</span> AcceleratorState


<span class="hljs-keyword">def</span> <span class="hljs-title function_">main</span>():
    accelerator = Accelerator()
    accelerator.<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;<span class="hljs-subst">{AcceleratorState()}</span>&quot;</span>)


<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    main()`,wrap:!1}}),Ls=new J({props:{code:"Y29tbWFuZF9maWxlJTNBJTIwbnVsbCUwQWNvbW1hbmRzJTNBJTIwbnVsbCUwQWNvbXB1dGVfZW52aXJvbm1lbnQlM0ElMjBMT0NBTF9NQUNISU5FJTBBZGVlcHNwZWVkX2NvbmZpZyUzQSUwQSUyMCUyMGdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUzQSUyMDElMEElMjAlMjBncmFkaWVudF9jbGlwcGluZyUzQSUyMDEuMCUwQSUyMCUyMG9mZmxvYWRfb3B0aW1pemVyX2RldmljZSUzQSUyMCdjcHUnJTBBJTIwJTIwb2ZmbG9hZF9wYXJhbV9kZXZpY2UlM0ElMjAnY3B1JyUwQSUyMCUyMHplcm8zX2luaXRfZmxhZyUzQSUyMHRydWUlMEElMjAlMjB6ZXJvM19zYXZlXzE2Yml0X21vZGVsJTNBJTIwdHJ1ZSUwQSUyMCUyMHplcm9fc3RhZ2UlM0ElMjAzJTBBJTIwJTIwZGVlcHNwZWVkX2NvbmZpZ19maWxlJTNBJTIwJ2RzX2NvbmZpZy5qc29uJyUwQWRpc3RyaWJ1dGVkX3R5cGUlM0ElMjBERUVQU1BFRUQlMEFkb3duY2FzdF9iZjE2JTNBJTIwJ25vJyUwQWR5bmFtb19iYWNrZW5kJTNBJTIwJ05PJyUwQWZzZHBfY29uZmlnJTNBJTIwJTdCJTdEJTBBZ3B1X2lkcyUzQSUyMG51bGwlMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl9wcm9jZXNzX2lwJTNBJTIwbnVsbCUwQW1haW5fcHJvY2Vzc19wb3J0JTNBJTIwbnVsbCUwQW1haW5fdHJhaW5pbmdfZnVuY3Rpb24lM0ElMjBtYWluJTBBbWVnYXRyb25fbG1fY29uZmlnJTNBJTIwJTdCJTdEJTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjAyJTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9uYW1lJTNBJTIwbnVsbCUwQXRwdV96b25lJTNBJTIwbnVsbCUwQXVzZV9jcHUlM0ElMjBmYWxzZQ==",highlighted:`<span class="hljs-attr">command_file:</span> <span class="hljs-literal">null</span>
<span class="hljs-attr">commands:</span> <span class="hljs-literal">null</span>
<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">deepspeed_config:</span>
  <span class="hljs-attr">gradient_accumulation_steps:</span> <span class="hljs-number">1</span>
  <span class="hljs-attr">gradient_clipping:</span> <span class="hljs-number">1.0</span>
  <span class="hljs-attr">offload_optimizer_device:</span> <span class="hljs-string">&#x27;cpu&#x27;</span>
  <span class="hljs-attr">offload_param_device:</span> <span class="hljs-string">&#x27;cpu&#x27;</span>
  <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
  <span class="hljs-attr">zero3_save_16bit_model:</span> <span class="hljs-literal">true</span>
  <span class="hljs-attr">zero_stage:</span> <span class="hljs-number">3</span>
  <span class="hljs-attr">deepspeed_config_file:</span> <span class="hljs-string">&#x27;ds_config.json&#x27;</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">dynamo_backend:</span> <span class="hljs-string">&#x27;NO&#x27;</span>
<span class="hljs-attr">fsdp_config:</span> {}
<span class="hljs-attr">gpu_ids:</span> <span class="hljs-literal">null</span>
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_process_ip:</span> <span class="hljs-literal">null</span>
<span class="hljs-attr">main_process_port:</span> <span class="hljs-literal">null</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">megatron_lm_config:</span> {}
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">2</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">tpu_name:</span> <span class="hljs-literal">null</span>
<span class="hljs-attr">tpu_zone:</span> <span class="hljs-literal">null</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),Ds=new J({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyYmYxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjB0cnVlJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19vcHRpbWl6YXRpb24lMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZSUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZTNfZ2F0aGVyXzE2Yml0X3dlaWdodHNfb25fbW9kZWxfc2F2ZSUyMiUzQSUyMGZhbHNlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyb2ZmbG9hZF9vcHRpbWl6ZXIlMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJkZXZpY2UlMjIlM0ElMjAlMjJub25lJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyb2ZmbG9hZF9wYXJhbSUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmRldmljZSUyMiUzQSUyMCUyMm5vbmUlMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjJncmFkaWVudF9jbGlwcGluZyUyMiUzQSUyMDEuMCUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYWluX2JhdGNoX3NpemUlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUyMiUzQSUyMDEwJTJDJTBBJTIwJTIwJTIwJTIwJTIyc3RlcHNfcGVyX3ByaW50JTIyJTNBJTIwMjAwMDAwMCUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;bf16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">3</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_gather_16bit_weights_on_model_save&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">false</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;offload_optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;device&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;none&quot;</span>
        <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;offload_param&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;device&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;none&quot;</span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_clipping&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1.0</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_batch_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_accumulation_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">10</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;steps_per_print&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2000000</span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),Ps=new J({props:{code:"VmFsdWVFcnJvciUzQSUyMFdoZW4lMjB1c2luZyUyMCU2MGRlZXBzcGVlZF9jb25maWdfZmlsZSU2MCUyQyUyMHRoZSUyMGZvbGxvd2luZyUyMGFjY2VsZXJhdGUlMjBjb25maWclMjB2YXJpYWJsZXMlMjB3aWxsJTIwYmUlMjBpZ25vcmVkJTNBJTBBJTVCJ2dyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyclMkMlMjAnZ3JhZGllbnRfY2xpcHBpbmcnJTJDJTIwJ3plcm9fc3RhZ2UnJTJDJTIwJ29mZmxvYWRfb3B0aW1pemVyX2RldmljZSclMkMlMjAnb2ZmbG9hZF9wYXJhbV9kZXZpY2UnJTJDJTBBJ3plcm8zX3NhdmVfMTZiaXRfbW9kZWwnJTJDJTIwJ21peGVkX3ByZWNpc2lvbiclNUQuJTBBUGxlYXNlJTIwc3BlY2lmeSUyMHRoZW0lMjBhcHByb3ByaWF0ZWx5JTIwaW4lMjB0aGUlMjBEZWVwU3BlZWQlMjBjb25maWclMjBmaWxlLiUwQUlmJTIweW91JTIwYXJlJTIwdXNpbmclMjBhbiUyMGFjY2VsZXJhdGUlMjBjb25maWclMjBmaWxlJTJDJTIwcmVtb3ZlJTIwb3RoZXIlMjBjb25maWclMjB2YXJpYWJsZXMlMjBtZW50aW9uZWQlMjBpbiUyMHRoZSUyMGFib3ZlJTIwc3BlY2lmaWVkJTIwbGlzdC4lMEFUaGUlMjBlYXNpZXN0JTIwbWV0aG9kJTIwaXMlMjB0byUyMGNyZWF0ZSUyMGElMjBuZXclMjBjb25maWclMjBmb2xsb3dpbmclMjB0aGUlMjBxdWVzdGlvbm5haXJlJTIwdmlhJTIwJTYwYWNjZWxlcmF0ZSUyMGNvbmZpZyU2MC4lMEFJdCUyMHdpbGwlMjBvbmx5JTIwYXNrJTIwZm9yJTIwdGhlJTIwbmVjZXNzYXJ5JTIwY29uZmlnJTIwdmFyaWFibGVzJTIwd2hlbiUyMHVzaW5nJTIwJTYwZGVlcHNwZWVkX2NvbmZpZ19maWxlJTYwLg==",highlighted:'ValueError: When using `deepspeed_config_file`, the following accelerate config variables will be ignored:\n[<span class="hljs-string">&#x27;gradient_accumulation_steps&#x27;</span>, <span class="hljs-string">&#x27;gradient_clipping&#x27;</span>, <span class="hljs-string">&#x27;zero_stage&#x27;</span>, <span class="hljs-string">&#x27;offload_optimizer_device&#x27;</span>, <span class="hljs-string">&#x27;offload_param_device&#x27;</span>,\n<span class="hljs-string">&#x27;zero3_save_16bit_model&#x27;</span>, <span class="hljs-string">&#x27;mixed_precision&#x27;</span>].\nPlease specify them appropriately <span class="hljs-keyword">in</span> the DeepSpeed config file.\nIf you are using an accelerate config file, remove other config variables mentioned <span class="hljs-keyword">in</span> the above specified list.\nThe easiest method is to create a new config following the questionnaire via `accelerate config`.\nIt will only ask <span class="hljs-keyword">for</span> the necessary config variables when using `deepspeed_config_file`.',wrap:!1}}),sl=new J({props:{code:"JTI0JTIwYWNjZWxlcmF0ZSUyMGNvbmZpZyUwQS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0lMEFJbiUyMHdoaWNoJTIwY29tcHV0ZSUyMGVudmlyb25tZW50JTIwYXJlJTIweW91JTIwcnVubmluZyUzRiUwQVRoaXMlMjBtYWNoaW5lJTBBLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSUwQVdoaWNoJTIwdHlwZSUyMG9mJTIwbWFjaGluZSUyMGFyZSUyMHlvdSUyMHVzaW5nJTNGJTBBbXVsdGktR1BVJTBBSG93JTIwbWFueSUyMGRpZmZlcmVudCUyMG1hY2hpbmVzJTIwd2lsbCUyMHlvdSUyMHVzZSUyMCh1c2UlMjBtb3JlJTIwdGhhbiUyMDElMjBmb3IlMjBtdWx0aS1ub2RlJTIwdHJhaW5pbmcpJTNGJTIwJTVCMSU1RCUzQSUwQURvJTIweW91JTIwd2lzaCUyMHRvJTIwb3B0aW1pemUlMjB5b3VyJTIwc2NyaXB0JTIwd2l0aCUyMHRvcmNoJTIwZHluYW1vJTNGJTVCeWVzJTJGTk8lNUQlM0ElMEFEbyUyMHlvdSUyMHdhbnQlMjB0byUyMHVzZSUyMERlZXBTcGVlZCUzRiUyMCU1QnllcyUyRk5PJTVEJTNBJTIweWVzJTBBRG8lMjB5b3UlMjB3YW50JTIwdG8lMjBzcGVjaWZ5JTIwYSUyMGpzb24lMjBmaWxlJTIwdG8lMjBhJTIwRGVlcFNwZWVkJTIwY29uZmlnJTNGJTIwJTVCeWVzJTJGTk8lNUQlM0ElMjB5ZXMlMEFQbGVhc2UlMjBlbnRlciUyMHRoZSUyMHBhdGglMjB0byUyMHRoZSUyMGpzb24lMjBEZWVwU3BlZWQlMjBjb25maWclMjBmaWxlJTNBJTIwZHNfY29uZmlnLmpzb24lMEFEbyUyMHlvdSUyMHdhbnQlMjB0byUyMGVuYWJsZSUyMCU2MGRlZXBzcGVlZC56ZXJvLkluaXQlNjAlMjB3aGVuJTIwdXNpbmclMjBaZVJPJTIwU3RhZ2UtMyUyMGZvciUyMGNvbnN0cnVjdGluZyUyMG1hc3NpdmUlMjBtb2RlbHMlM0YlMjAlNUJ5ZXMlMkZOTyU1RCUzQSUyMHllcyUwQUhvdyUyMG1hbnklMjBHUFUocyklMjBzaG91bGQlMjBiZSUyMHVzZWQlMjBmb3IlMjBkaXN0cmlidXRlZCUyMHRyYWluaW5nJTNGJTIwJTVCMSU1RCUzQTQlMEFhY2NlbGVyYXRlJTIwY29uZmlndXJhdGlvbiUyMHNhdmVkJTIwYXQlMjBkc19jb25maWdfc2FtcGxlLnlhbWw=",highlighted:`$ accelerate config
-------------------------------------------------------------------------------------------------------------------------------
In <span class="hljs-built_in">which</span> compute environment are you running?
This machine
-------------------------------------------------------------------------------------------------------------------------------
Which <span class="hljs-built_in">type</span> of machine are you using?
multi-GPU
How many different machines will you use (use more than 1 <span class="hljs-keyword">for</span> multi-node training)? [1]:
Do you wish to optimize your script with torch dynamo?[<span class="hljs-built_in">yes</span>/NO]:
Do you want to use DeepSpeed? [<span class="hljs-built_in">yes</span>/NO]: <span class="hljs-built_in">yes</span>
Do you want to specify a json file to a DeepSpeed config? [<span class="hljs-built_in">yes</span>/NO]: <span class="hljs-built_in">yes</span>
Please enter the path to the json DeepSpeed config file: ds_config.json
Do you want to <span class="hljs-built_in">enable</span> \`deepspeed.zero.Init\` when using ZeRO Stage-3 <span class="hljs-keyword">for</span> constructing massive models? [<span class="hljs-built_in">yes</span>/NO]: <span class="hljs-built_in">yes</span>
How many GPU(s) should be used <span class="hljs-keyword">for</span> distributed training? [1]:4
accelerate configuration saved at ds_config_sample.yaml`,wrap:!1}}),ll=new J({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWVwc3BlZWRfY29uZmlnJTNBJTBBJTIwJTIwZGVlcHNwZWVkX2NvbmZpZ19maWxlJTNBJTIwZHNfY29uZmlnLmpzb24lMEElMjAlMjB6ZXJvM19pbml0X2ZsYWclM0ElMjB0cnVlJTBBZGlzdHJpYnV0ZWRfdHlwZSUzQSUyMERFRVBTUEVFRCUwQWRvd25jYXN0X2JmMTYlM0ElMjAnbm8nJTBBZHluYW1vX2JhY2tlbmQlM0ElMjAnTk8nJTBBZnNkcF9jb25maWclM0ElMjAlN0IlN0QlMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtZWdhdHJvbl9sbV9jb25maWclM0ElMjAlN0IlN0QlMEFudW1fbWFjaGluZXMlM0ElMjAxJTBBbnVtX3Byb2Nlc3NlcyUzQSUyMDQlMEFyZHp2X2JhY2tlbmQlM0ElMjBzdGF0aWMlMEFzYW1lX25ldHdvcmslM0ElMjB0cnVlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">deepspeed_config:</span>
  <span class="hljs-attr">deepspeed_config_file:</span> <span class="hljs-string">ds_config.json</span>
  <span class="hljs-attr">zero3_init_flag:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">distributed_type:</span> <span class="hljs-string">DEEPSPEED</span>
<span class="hljs-attr">downcast_bf16:</span> <span class="hljs-string">&#x27;no&#x27;</span>
<span class="hljs-attr">dynamo_backend:</span> <span class="hljs-string">&#x27;NO&#x27;</span>
<span class="hljs-attr">fsdp_config:</span> {}
<span class="hljs-attr">machine_rank:</span> <span class="hljs-number">0</span>
<span class="hljs-attr">main_training_function:</span> <span class="hljs-string">main</span>
<span class="hljs-attr">megatron_lm_config:</span> {}
<span class="hljs-attr">num_machines:</span> <span class="hljs-number">1</span>
<span class="hljs-attr">num_processes:</span> <span class="hljs-number">4</span>
<span class="hljs-attr">rdzv_backend:</span> <span class="hljs-string">static</span>
<span class="hljs-attr">same_network:</span> <span class="hljs-literal">true</span>
<span class="hljs-attr">use_cpu:</span> <span class="hljs-literal">false</span>`,wrap:!1}}),el=new J({props:{code:"RGlzdHJpYnV0ZWQlMjBlbnZpcm9ubWVudCUzQSUyMERFRVBTUEVFRCUyMCUyMEJhY2tlbmQlM0ElMjBuY2NsJTBBTnVtJTIwcHJvY2Vzc2VzJTNBJTIwNCUwQVByb2Nlc3MlMjBpbmRleCUzQSUyMDAlMEFMb2NhbCUyMHByb2Nlc3MlMjBpbmRleCUzQSUyMDAlMEFEZXZpY2UlM0ElMjBjdWRhJTNBMCUwQU1peGVkJTIwcHJlY2lzaW9uJTIwdHlwZSUzQSUyMGJmMTYlMEFkc19jb25maWclM0ElMjAlN0InYmYxNiclM0ElMjAlN0InZW5hYmxlZCclM0ElMjBUcnVlJTdEJTJDJTIwJ3plcm9fb3B0aW1pemF0aW9uJyUzQSUyMCU3QidzdGFnZSclM0ElMjAzJTJDJTIwJ3N0YWdlM19nYXRoZXJfMTZiaXRfd2VpZ2h0c19vbl9tb2RlbF9zYXZlJyUzQSUyMEZhbHNlJTJDJTIwJ29mZmxvYWRfb3B0aW1pemVyJyUzQSUyMCU3QidkZXZpY2UnJTNBJTIwJ25vbmUnJTdEJTJDJTIwJ29mZmxvYWRfcGFyYW0nJTNBJTIwJTdCJ2RldmljZSclM0ElMjAnbm9uZSclN0QlN0QlMkMlMjAnZ3JhZGllbnRfY2xpcHBpbmcnJTNBJTIwMS4wJTJDJTIwJ3RyYWluX2JhdGNoX3NpemUnJTNBJTIwJ2F1dG8nJTJDJTIwJ3RyYWluX21pY3JvX2JhdGNoX3NpemVfcGVyX2dwdSclM0ElMjAnYXV0byclMkMlMjAnZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJyUzQSUyMDEwJTJDJTIwJ3N0ZXBzX3Blcl9wcmludCclM0ElMjBpbmYlMkMlMjAnZnAxNiclM0ElMjAlN0InZW5hYmxlZCclM0ElMjBGYWxzZSU3RCU3RA==",highlighted:`Distributed environment: DEEPSPEED  Backend: nccl
Num processes: 4
Process index: 0
Local process index: 0
Device: cuda:0
Mixed precision <span class="hljs-built_in">type</span>: bf16
ds_config: {<span class="hljs-string">&#x27;bf16&#x27;</span>: {<span class="hljs-string">&#x27;enabled&#x27;</span>: True}, <span class="hljs-string">&#x27;zero_optimization&#x27;</span>: {<span class="hljs-string">&#x27;stage&#x27;</span>: 3, <span class="hljs-string">&#x27;stage3_gather_16bit_weights_on_model_save&#x27;</span>: False, <span class="hljs-string">&#x27;offload_optimizer&#x27;</span>: {<span class="hljs-string">&#x27;device&#x27;</span>: <span class="hljs-string">&#x27;none&#x27;</span>}, <span class="hljs-string">&#x27;offload_param&#x27;</span>: {<span class="hljs-string">&#x27;device&#x27;</span>: <span class="hljs-string">&#x27;none&#x27;</span>}}, <span class="hljs-string">&#x27;gradient_clipping&#x27;</span>: 1.0, <span class="hljs-string">&#x27;train_batch_size&#x27;</span>: <span class="hljs-string">&#x27;auto&#x27;</span>, <span class="hljs-string">&#x27;train_micro_batch_size_per_gpu&#x27;</span>: <span class="hljs-string">&#x27;auto&#x27;</span>, <span class="hljs-string">&#x27;gradient_accumulation_steps&#x27;</span>: 10, <span class="hljs-string">&#x27;steps_per_print&#x27;</span>: inf, <span class="hljs-string">&#x27;fp16&#x27;</span>: {<span class="hljs-string">&#x27;enabled&#x27;</span>: False}}`,wrap:!1}}),nl=new J({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyYmYxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19vcHRpbWl6YXRpb24lMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZSUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZTNfZ2F0aGVyXzE2Yml0X3dlaWdodHNfb25fbW9kZWxfc2F2ZSUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvZmZsb2FkX29wdGltaXplciUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmRldmljZSUyMiUzQSUyMCUyMmF1dG8lMjIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvZmZsb2FkX3BhcmFtJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZGV2aWNlJTIyJTNBJTIwJTIyYXV0byUyMiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCUyMmdyYWRpZW50X2NsaXBwaW5nJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYWluX2JhdGNoX3NpemUlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMmdyYWRpZW50X2FjY3VtdWxhdGlvbl9zdGVwcyUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJzdGVwc19wZXJfcHJpbnQlMjIlM0ElMjAyMDAwMDAwJTBBJTdE",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;bf16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_gather_16bit_weights_on_model_save&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;offload_optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;device&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
        <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;offload_param&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;device&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_clipping&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_batch_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_accumulation_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;steps_per_print&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2000000</span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),pl=new J({props:{code:"RGlzdHJpYnV0ZWQlMjBlbnZpcm9ubWVudCUzQSUyMERFRVBTUEVFRCUyMCUyMEJhY2tlbmQlM0ElMjBuY2NsJTBBTnVtJTIwcHJvY2Vzc2VzJTNBJTIwNCUwQVByb2Nlc3MlMjBpbmRleCUzQSUyMDAlMEFMb2NhbCUyMHByb2Nlc3MlMjBpbmRleCUzQSUyMDAlMEFEZXZpY2UlM0ElMjBjdWRhJTNBMCUwQU1peGVkJTIwcHJlY2lzaW9uJTIwdHlwZSUzQSUyMGZwMTYlMEFkc19jb25maWclM0ElMjAlN0InYmYxNiclM0ElMjAlN0InZW5hYmxlZCclM0ElMjBGYWxzZSU3RCUyQyUyMCd6ZXJvX29wdGltaXphdGlvbiclM0ElMjAlN0Inc3RhZ2UnJTNBJTIwMyUyQyUyMCdzdGFnZTNfZ2F0aGVyXzE2Yml0X3dlaWdodHNfb25fbW9kZWxfc2F2ZSclM0ElMjBUcnVlJTJDJTIwJ29mZmxvYWRfb3B0aW1pemVyJyUzQSUyMCU3QidkZXZpY2UnJTNBJTIwJ252bWUnJTdEJTJDJTIwJ29mZmxvYWRfcGFyYW0nJTNBJTIwJTdCJ2RldmljZSclM0ElMjAnY3B1JyU3RCU3RCUyQyUyMCdncmFkaWVudF9jbGlwcGluZyclM0ElMjAxLjAlMkMlMjAndHJhaW5fYmF0Y2hfc2l6ZSclM0ElMjAnYXV0byclMkMlMjAndHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JyUzQSUyMCdhdXRvJyUyQyUyMCdncmFkaWVudF9hY2N1bXVsYXRpb25fc3RlcHMnJTNBJTIwNSUyQyUyMCdzdGVwc19wZXJfcHJpbnQnJTNBJTIwaW5mJTJDJTIwJ2ZwMTYnJTNBJTIwJTdCJ2VuYWJsZWQnJTNBJTIwVHJ1ZSUyQyUyMCdhdXRvX2Nhc3QnJTNBJTIwVHJ1ZSU3RCU3RA==",highlighted:`Distributed environment: DEEPSPEED  Backend: nccl
Num processes: 4
Process index: 0
Local process index: 0
Device: cuda:0
Mixed precision <span class="hljs-built_in">type</span>: fp16
ds_config: {<span class="hljs-string">&#x27;bf16&#x27;</span>: {<span class="hljs-string">&#x27;enabled&#x27;</span>: False}, <span class="hljs-string">&#x27;zero_optimization&#x27;</span>: {<span class="hljs-string">&#x27;stage&#x27;</span>: 3, <span class="hljs-string">&#x27;stage3_gather_16bit_weights_on_model_save&#x27;</span>: True, <span class="hljs-string">&#x27;offload_optimizer&#x27;</span>: {<span class="hljs-string">&#x27;device&#x27;</span>: <span class="hljs-string">&#x27;nvme&#x27;</span>}, <span class="hljs-string">&#x27;offload_param&#x27;</span>: {<span class="hljs-string">&#x27;device&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>}}, <span class="hljs-string">&#x27;gradient_clipping&#x27;</span>: 1.0, <span class="hljs-string">&#x27;train_batch_size&#x27;</span>: <span class="hljs-string">&#x27;auto&#x27;</span>, <span class="hljs-string">&#x27;train_micro_batch_size_per_gpu&#x27;</span>: <span class="hljs-string">&#x27;auto&#x27;</span>, <span class="hljs-string">&#x27;gradient_accumulation_steps&#x27;</span>: 5, <span class="hljs-string">&#x27;steps_per_print&#x27;</span>: inf, <span class="hljs-string">&#x27;fp16&#x27;</span>: {<span class="hljs-string">&#x27;enabled&#x27;</span>: True, <span class="hljs-string">&#x27;auto_cast&#x27;</span>: True}}`,wrap:!1}}),ol=new b({props:{title:"Saving and loading",local:"saving-and-loading",headingTag:"h2"}}),Ml=new J({props:{code:"dW53cmFwcGVkX21vZGVsJTIwJTNEJTIwYWNjZWxlcmF0b3IudW53cmFwX21vZGVsKG1vZGVsKSUwQSUwQSUyMyUyME5ldyUyMENvZGUlMjAlMjMlMEElMjMlMjBTYXZlcyUyMHRoZSUyMHdob2xlJTJGdW5wYXJ0aXRpb25lZCUyMGZwMTYlMjBtb2RlbCUyMHdoZW4lMjBpbiUyMFplUk8lMjBTdGFnZS0zJTIwdG8lMjB0aGUlMjBvdXRwdXQlMjBkaXJlY3RvcnklMjBpZiUwQSUyMyUyMCU2MHN0YWdlM19nYXRoZXJfMTZiaXRfd2VpZ2h0c19vbl9tb2RlbF9zYXZlJTYwJTIwaXMlMjBUcnVlJTIwaW4lMjBEZWVwU3BlZWQlMjBDb25maWclMjBmaWxlJTIwb3IlMEElMjMlMjAlNjB6ZXJvM19zYXZlXzE2Yml0X21vZGVsJTYwJTIwaXMlMjBUcnVlJTIwaW4lMjBEZWVwU3BlZWQlMjBQbHVnaW4uJTBBJTIzJTIwRm9yJTIwWmVybyUyMFN0YWdlcyUyMDElMjBhbmQlMjAyJTJDJTIwbW9kZWxzJTIwYXJlJTIwc2F2ZWQlMjBhcyUyMHVzdWFsJTIwaW4lMjB0aGUlMjBvdXRwdXQlMjBkaXJlY3RvcnkuJTBBJTIzJTIwVGhlJTIwbW9kZWwlMjBuYW1lJTIwc2F2ZWQlMjBpcyUyMCU2MHB5dG9yY2hfbW9kZWwuYmluJTYwJTBBdW53cmFwcGVkX21vZGVsLnNhdmVfcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBhcmdzLm91dHB1dF9kaXIlMkMlMEElMjAlMjAlMjAlMjBpc19tYWluX3Byb2Nlc3MlM0RhY2NlbGVyYXRvci5pc19tYWluX3Byb2Nlc3MlMkMlMEElMjAlMjAlMjAlMjBzYXZlX2Z1bmN0aW9uJTNEYWNjZWxlcmF0b3Iuc2F2ZSUyQyUwQSUyMCUyMCUyMCUyMHN0YXRlX2RpY3QlM0RhY2NlbGVyYXRvci5nZXRfc3RhdGVfZGljdChtb2RlbCklMkMlMEEp",highlighted:`unwrapped_model = accelerator.unwrap_model(model)

<span class="hljs-comment"># New Code #</span>
<span class="hljs-comment"># Saves the whole/unpartitioned fp16 model when in ZeRO Stage-3 to the output directory if</span>
<span class="hljs-comment"># \`stage3_gather_16bit_weights_on_model_save\` is True in DeepSpeed Config file or</span>
<span class="hljs-comment"># \`zero3_save_16bit_model\` is True in DeepSpeed Plugin.</span>
<span class="hljs-comment"># For Zero Stages 1 and 2, models are saved as usual in the output directory.</span>
<span class="hljs-comment"># The model name saved is \`pytorch_model.bin\`</span>
unwrapped_model.save_pretrained(
    args.output_dir,
    is_main_process=accelerator.is_main_process,
    save_function=accelerator.save,
    state_dict=accelerator.get_state_dict(model),
)`,wrap:!1}}),ul=new J({props:{code:"c3VjY2VzcyUyMCUzRCUyMG1vZGVsLnNhdmVfY2hlY2twb2ludChQQVRIJTJDJTIwY2twdF9pZCUyQyUyMGNoZWNrcG9pbnRfc3RhdGVfZGljdCklMEFzdGF0dXNfbXNnJTIwJTNEJTIwZiUyMmNoZWNrcG9pbnRpbmclM0ElMjBQQVRIJTNEJTdCUEFUSCU3RCUyQyUyMGNrcHRfaWQlM0QlN0Jja3B0X2lkJTdEJTIyJTBBaWYlMjBzdWNjZXNzJTNBJTBBJTIwJTIwJTIwJTIwbG9nZ2luZy5pbmZvKGYlMjJTdWNjZXNzJTIwJTdCc3RhdHVzX21zZyU3RCUyMiklMEFlbHNlJTNBJTBBJTIwJTIwJTIwJTIwbG9nZ2luZy53YXJuaW5nKGYlMjJGYWlsdXJlJTIwJTdCc3RhdHVzX21zZyU3RCUyMik=",highlighted:`success = model.save_checkpoint(PATH, ckpt_id, checkpoint_state_dict)
status_msg = <span class="hljs-string">f&quot;checkpointing: PATH=<span class="hljs-subst">{PATH}</span>, ckpt_id=<span class="hljs-subst">{ckpt_id}</span>&quot;</span>
<span class="hljs-keyword">if</span> success:
    logging.info(<span class="hljs-string">f&quot;Success <span class="hljs-subst">{status_msg}</span>&quot;</span>)
<span class="hljs-keyword">else</span>:
    logging.warning(<span class="hljs-string">f&quot;Failure <span class="hljs-subst">{status_msg}</span>&quot;</span>)`,wrap:!1}}),rl=new J({props:{code:"JTI0JTIwY2QlMjAlMkZwYXRoJTJGdG8lMkZjaGVja3BvaW50X2RpciUwQSUyNCUyMC4lMkZ6ZXJvX3RvX2ZwMzIucHklMjAuJTIwcHl0b3JjaF9tb2RlbC5iaW4lMEFQcm9jZXNzaW5nJTIwemVybyUyMGNoZWNrcG9pbnQlMjBhdCUyMGdsb2JhbF9zdGVwMSUwQURldGVjdGVkJTIwY2hlY2twb2ludCUyMG9mJTIwdHlwZSUyMHplcm8lMjBzdGFnZSUyMDMlMkMlMjB3b3JsZF9zaXplJTNBJTIwMiUwQVNhdmluZyUyMGZwMzIlMjBzdGF0ZSUyMGRpY3QlMjB0byUyMHB5dG9yY2hfbW9kZWwuYmluJTIwKHRvdGFsX251bWVsJTNENjA1MDY2MjQp",highlighted:`$ <span class="hljs-built_in">cd</span> /path/to/checkpoint_dir
$ ./zero_to_fp32.py . pytorch_model.bin
Processing zero checkpoint at global_step1
Detected checkpoint of <span class="hljs-built_in">type</span> zero stage 3, world_size: 2
Saving fp32 state dict to pytorch_model.bin (total_numel=60506624)`,wrap:!1}}),yl=new J({props:{code:"ZnJvbSUyMGRlZXBzcGVlZC51dGlscy56ZXJvX3RvX2ZwMzIlMjBpbXBvcnQlMjBsb2FkX3N0YXRlX2RpY3RfZnJvbV96ZXJvX2NoZWNrcG9pbnQlMEElMEF1bndyYXBwZWRfbW9kZWwlMjAlM0QlMjBhY2NlbGVyYXRvci51bndyYXBfbW9kZWwobW9kZWwpJTBBZnAzMl9tb2RlbCUyMCUzRCUyMGxvYWRfc3RhdGVfZGljdF9mcm9tX3plcm9fY2hlY2twb2ludCh1bndyYXBwZWRfbW9kZWwlMkMlMjBjaGVja3BvaW50X2Rpcik=",highlighted:`<span class="hljs-keyword">from</span> deepspeed.utils.zero_to_fp32 <span class="hljs-keyword">import</span> load_state_dict_from_zero_checkpoint

unwrapped_model = accelerator.unwrap_model(model)
fp32_model = load_state_dict_from_zero_checkpoint(unwrapped_model, checkpoint_dir)`,wrap:!1}}),jl=new J({props:{code:"ZnJvbSUyMGRlZXBzcGVlZC51dGlscy56ZXJvX3RvX2ZwMzIlMjBpbXBvcnQlMjBnZXRfZnAzMl9zdGF0ZV9kaWN0X2Zyb21femVyb19jaGVja3BvaW50JTBBJTBBc3RhdGVfZGljdCUyMCUzRCUyMGdldF9mcDMyX3N0YXRlX2RpY3RfZnJvbV96ZXJvX2NoZWNrcG9pbnQoY2hlY2twb2ludF9kaXIp",highlighted:`<span class="hljs-keyword">from</span> deepspeed.utils.zero_to_fp32 <span class="hljs-keyword">import</span> get_fp32_state_dict_from_zero_checkpoint

state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir)`,wrap:!1}}),dl=new b({props:{title:"ZeRO Inference",local:"zero-inference",headingTag:"h2"}}),hl=new J({props:{code:"bW9kZWwlMkMlMjBldmFsX2RhdGFsb2FkZXIlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKG1vZGVsJTJDJTIwZXZhbF9kYXRhbG9hZGVyKQ==",highlighted:"model, eval_dataloader = accelerator.prepare(model, eval_dataloader)",wrap:!1}}),wl=new b({props:{title:"Few caveats to be aware of",local:"few-caveats-to-be-aware-of",headingTag:"h2"}}),Ul=new b({props:{title:"Multi-node DeepSpeed",local:"multi-node-deepspeed",headingTag:"h2"}}),Il=new b({props:{title:"DeepSpeed Resources",local:"deepspeed-resources",headingTag:"h2"}}),S=new Kn({props:{$$slots:{default:[ep]},$$scope:{ctx:Dl}}}),Wl=new lp({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/deepspeed.md"}}),{c(){T=p("meta"),X=a(),f=p("p"),Nl=a(),o(N.$$.fragment),Pl=a(),o(G.$$.fragment),Ol=a(),v=p("p"),v.innerHTML=ha,Kl=a(),V=p("ol"),V.innerHTML=wa,se=a(),Y=p("p"),Y.innerHTML=Ta,le=a(),A=p("p"),A.textContent=Ua,ee=a(),z=p("p"),z.textContent=ma,te=a(),R=p("p"),R.innerHTML=fa,ae=a(),x=p("ol"),x.innerHTML=ba,ne=a(),o(E.$$.fragment),pe=a(),Q=p("p"),Q.textContent=Ia,ie=a(),k=p("ol"),k.innerHTML=Za,ce=a(),F=p("p"),F.innerHTML=Ca,oe=a(),H=p("p"),H.innerHTML=_a,Me=a(),q=p("p"),q.innerHTML=Ba,ue=a(),$=p("p"),$.innerHTML=ga,re=a(),L=p("p"),L.innerHTML=Wa,ye=a(),D=p("p"),D.innerHTML=Sa,je=a(),P=p("p"),P.innerHTML=Xa,de=a(),O=p("u"),O.textContent=Na,Je=Dn(`: With respect to Disk Offload, the disk should be an NVME for decent speed but it technically works on any Disk
`),K=p("p"),K.textContent=Ga,he=a(),ss=p("ol"),ss.innerHTML=va,we=a(),o(ls.$$.fragment),Te=a(),es=p("p"),es.innerHTML=Va,Ue=a(),ts=p("p"),ts.innerHTML=Ya,me=a(),o(as.$$.fragment),fe=a(),ns=p("p"),ns.textContent=Aa,be=a(),o(ps.$$.fragment),Ie=a(),is=p("p"),is.textContent=za,Ze=a(),o(cs.$$.fragment),Ce=a(),os=p("p"),os.innerHTML=Ra,_e=a(),Ms=p("p"),Ms.innerHTML=xa,Be=a(),o(us.$$.fragment),ge=a(),o(rs.$$.fragment),We=a(),ys=p("p"),ys.innerHTML=Ea,Se=a(),o(js.$$.fragment),Xe=a(),o(ds.$$.fragment),Ne=a(),Js=p("p"),Js.innerHTML=Qa,Ge=a(),o(hs.$$.fragment),ve=a(),ws=p("p"),ws.textContent=ka,Ve=a(),o(Ts.$$.fragment),Ye=a(),Us=p("p"),Us.textContent=Fa,Ae=a(),o(ms.$$.fragment),ze=a(),fs=p("p"),fs.textContent=Ha,Re=a(),o(bs.$$.fragment),xe=a(),Is=p("p"),Is.innerHTML=qa,Ee=a(),Zs=p("p"),Zs.innerHTML=$a,Qe=a(),o(Cs.$$.fragment),ke=a(),_s=p("p"),_s.innerHTML=La,Fe=a(),o(Bs.$$.fragment),He=a(),o(gs.$$.fragment),qe=a(),Ws=p("p"),Ws.innerHTML=Da,$e=a(),o(Ss.$$.fragment),Le=a(),Xs=p("p"),Xs.innerHTML=Pa,De=a(),o(Ns.$$.fragment),Pe=a(),o(Gs.$$.fragment),Oe=a(),vs=p("p"),vs.innerHTML=Oa,Ke=a(),o(Vs.$$.fragment),st=a(),Ys=p("p"),Ys.innerHTML=Ka,lt=a(),As=p("p"),As.innerHTML=sn,et=a(),I=p("ol"),U=p("li"),Gl=p("p"),Gl.innerHTML=ln,Kt=a(),vl=p("p"),vl.innerHTML=en,sa=a(),o(zs.$$.fragment),la=a(),Vl=p("p"),Vl.innerHTML=tn,ea=a(),Yl=p("p"),Yl.innerHTML=an,ta=a(),Al=p("p"),Al.innerHTML=nn,aa=a(),zl=p("li"),zl.innerHTML=pn,tt=a(),Rs=p("p"),Rs.innerHTML=cn,at=a(),xs=p("ul"),xs.innerHTML=on,nt=a(),Es=p("p"),Es.innerHTML=Mn,pt=a(),Qs=p("p"),Qs.innerHTML=un,it=a(),ks=p("p"),ks.innerHTML=rn,ct=a(),Fs=p("p"),Fs.innerHTML=yn,ot=a(),o(Hs.$$.fragment),Mt=a(),qs=p("p"),qs.innerHTML=jn,ut=a(),$s=p("ol"),$s.innerHTML=dn,rt=a(),o(Ls.$$.fragment),yt=a(),Z=p("ol"),Z.innerHTML=Jn,jt=a(),o(Ds.$$.fragment),dt=a(),C=p("ol"),C.innerHTML=hn,Jt=a(),o(Ps.$$.fragment),ht=a(),Os=p("p"),Os.innerHTML=wn,wt=a(),Ks=p("ol"),Ks.innerHTML=Tn,Tt=a(),o(sl.$$.fragment),Ut=a(),_=p("ol"),_.innerHTML=Un,mt=a(),o(ll.$$.fragment),ft=a(),B=p("ol"),B.innerHTML=mn,bt=a(),o(el.$$.fragment),It=a(),tl=p("p"),tl.innerHTML=fn,Zt=a(),al=p("ol"),al.innerHTML=bn,Ct=a(),o(nl.$$.fragment),_t=a(),g=p("ol"),g.innerHTML=In,Bt=a(),o(pl.$$.fragment),gt=a(),il=p("p"),il.innerHTML=Zn,Wt=a(),cl=p("ol"),cl.innerHTML=Cn,St=a(),o(ol.$$.fragment),Xt=a(),W=p("ol"),Rl=p("li"),Rl.innerHTML=_n,na=a(),h=p("li"),xl=p("p"),xl.innerHTML=Bn,pa=a(),El=p("p"),El.innerHTML=gn,ia=a(),o(Ml.$$.fragment),ca=a(),Ql=p("p"),Ql.innerHTML=Wn,oa=a(),o(ul.$$.fragment),Ma=a(),kl=p("p"),kl.innerHTML=Sn,ua=a(),o(rl.$$.fragment),ra=a(),Fl=p("p"),Fl.textContent=Xn,ya=a(),o(yl.$$.fragment),ja=a(),Hl=p("p"),Hl.innerHTML=Nn,da=a(),o(jl.$$.fragment),Ja=a(),ql=p("p"),ql.textContent=Gn,Nt=a(),o(dl.$$.fragment),Gt=a(),Jl=p("p"),Jl.textContent=vn,vt=a(),o(hl.$$.fragment),Vt=a(),o(wl.$$.fragment),Yt=a(),Tl=p("ol"),Tl.innerHTML=Vn,At=a(),o(Ul.$$.fragment),zt=a(),ml=p("p"),ml.innerHTML=Yn,Rt=a(),fl=p("p"),fl.innerHTML=An,xt=a(),bl=p("p"),bl.innerHTML=zn,Et=a(),o(Il.$$.fragment),Qt=a(),Zl=p("p"),Zl.innerHTML=Rn,kt=a(),Cl=p("ul"),Cl.innerHTML=xn,Ft=a(),_l=p("p"),_l.textContent=En,Ht=a(),Bl=p("ul"),Bl.innerHTML=Qn,qt=a(),gl=p("p"),gl.innerHTML=kn,$t=a(),o(S.$$.fragment),Lt=a(),o(Wl.$$.fragment),Dt=a(),Ll=p("p"),this.h()},l(s){const l=Pn("svelte-u9bgzb",document.head);T=i(l,"META",{name:!0,content:!0}),l.forEach(e),X=n(s),f=i(s,"P",{}),$l(f).forEach(e),Nl=n(s),M(N.$$.fragment,s),Pl=n(s),M(G.$$.fragment,s),Ol=n(s),v=i(s,"P",{"data-svelte-h":!0}),c(v)!=="svelte-142dpqv"&&(v.innerHTML=ha),Kl=n(s),V=i(s,"OL",{"data-svelte-h":!0}),c(V)!=="svelte-3z90ra"&&(V.innerHTML=wa),se=n(s),Y=i(s,"P",{"data-svelte-h":!0}),c(Y)!=="svelte-1ipy5tt"&&(Y.innerHTML=Ta),le=n(s),A=i(s,"P",{"data-svelte-h":!0}),c(A)!=="svelte-iaccrf"&&(A.textContent=Ua),ee=n(s),z=i(s,"P",{"data-svelte-h":!0}),c(z)!=="svelte-l1xxpu"&&(z.textContent=ma),te=n(s),R=i(s,"P",{"data-svelte-h":!0}),c(R)!=="svelte-a6ic94"&&(R.innerHTML=fa),ae=n(s),x=i(s,"OL",{"data-svelte-h":!0}),c(x)!=="svelte-15nzigp"&&(x.innerHTML=ba),ne=n(s),M(E.$$.fragment,s),pe=n(s),Q=i(s,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-1igpel8"&&(Q.textContent=Ia),ie=n(s),k=i(s,"OL",{"data-svelte-h":!0}),c(k)!=="svelte-1dxmiag"&&(k.innerHTML=Za),ce=n(s),F=i(s,"P",{"data-svelte-h":!0}),c(F)!=="svelte-134sd43"&&(F.innerHTML=Ca),oe=n(s),H=i(s,"P",{"data-svelte-h":!0}),c(H)!=="svelte-1d4qqra"&&(H.innerHTML=_a),Me=n(s),q=i(s,"P",{"data-svelte-h":!0}),c(q)!=="svelte-n0e1lo"&&(q.innerHTML=Ba),ue=n(s),$=i(s,"P",{"data-svelte-h":!0}),c($)!=="svelte-1ob4qwq"&&($.innerHTML=ga),re=n(s),L=i(s,"P",{"data-svelte-h":!0}),c(L)!=="svelte-f1y4d9"&&(L.innerHTML=Wa),ye=n(s),D=i(s,"P",{"data-svelte-h":!0}),c(D)!=="svelte-1sgbcs5"&&(D.innerHTML=Sa),je=n(s),P=i(s,"P",{"data-svelte-h":!0}),c(P)!=="svelte-yiw0au"&&(P.innerHTML=Xa),de=n(s),O=i(s,"U",{"data-svelte-h":!0}),c(O)!=="svelte-248d1k"&&(O.textContent=Na),Je=On(s,`: With respect to Disk Offload, the disk should be an NVME for decent speed but it technically works on any Disk
`),K=i(s,"P",{"data-svelte-h":!0}),c(K)!=="svelte-1i9b1jx"&&(K.textContent=Ga),he=n(s),ss=i(s,"OL",{"data-svelte-h":!0}),c(ss)!=="svelte-yjfbl7"&&(ss.innerHTML=va),we=n(s),M(ls.$$.fragment,s),Te=n(s),es=i(s,"P",{"data-svelte-h":!0}),c(es)!=="svelte-1k0k3hd"&&(es.innerHTML=Va),Ue=n(s),ts=i(s,"P",{"data-svelte-h":!0}),c(ts)!=="svelte-yplg2o"&&(ts.innerHTML=Ya),me=n(s),M(as.$$.fragment,s),fe=n(s),ns=i(s,"P",{"data-svelte-h":!0}),c(ns)!=="svelte-3pof6s"&&(ns.textContent=Aa),be=n(s),M(ps.$$.fragment,s),Ie=n(s),is=i(s,"P",{"data-svelte-h":!0}),c(is)!=="svelte-1ik26hy"&&(is.textContent=za),Ze=n(s),M(cs.$$.fragment,s),Ce=n(s),os=i(s,"P",{"data-svelte-h":!0}),c(os)!=="svelte-10u1bf7"&&(os.innerHTML=Ra),_e=n(s),Ms=i(s,"P",{"data-svelte-h":!0}),c(Ms)!=="svelte-vp1nym"&&(Ms.innerHTML=xa),Be=n(s),M(us.$$.fragment,s),ge=n(s),M(rs.$$.fragment,s),We=n(s),ys=i(s,"P",{"data-svelte-h":!0}),c(ys)!=="svelte-12x21g8"&&(ys.innerHTML=Ea),Se=n(s),M(js.$$.fragment,s),Xe=n(s),M(ds.$$.fragment,s),Ne=n(s),Js=i(s,"P",{"data-svelte-h":!0}),c(Js)!=="svelte-lh95an"&&(Js.innerHTML=Qa),Ge=n(s),M(hs.$$.fragment,s),ve=n(s),ws=i(s,"P",{"data-svelte-h":!0}),c(ws)!=="svelte-1spmm13"&&(ws.textContent=ka),Ve=n(s),M(Ts.$$.fragment,s),Ye=n(s),Us=i(s,"P",{"data-svelte-h":!0}),c(Us)!=="svelte-3pof6s"&&(Us.textContent=Fa),Ae=n(s),M(ms.$$.fragment,s),ze=n(s),fs=i(s,"P",{"data-svelte-h":!0}),c(fs)!=="svelte-1vp75xs"&&(fs.textContent=Ha),Re=n(s),M(bs.$$.fragment,s),xe=n(s),Is=i(s,"P",{"data-svelte-h":!0}),c(Is)!=="svelte-17th5u9"&&(Is.innerHTML=qa),Ee=n(s),Zs=i(s,"P",{"data-svelte-h":!0}),c(Zs)!=="svelte-1l59g75"&&(Zs.innerHTML=$a),Qe=n(s),M(Cs.$$.fragment,s),ke=n(s),_s=i(s,"P",{"data-svelte-h":!0}),c(_s)!=="svelte-1iv8u1c"&&(_s.innerHTML=La),Fe=n(s),M(Bs.$$.fragment,s),He=n(s),M(gs.$$.fragment,s),qe=n(s),Ws=i(s,"P",{"data-svelte-h":!0}),c(Ws)!=="svelte-3yuze5"&&(Ws.innerHTML=Da),$e=n(s),M(Ss.$$.fragment,s),Le=n(s),Xs=i(s,"P",{"data-svelte-h":!0}),c(Xs)!=="svelte-15nfwtv"&&(Xs.innerHTML=Pa),De=n(s),M(Ns.$$.fragment,s),Pe=n(s),M(Gs.$$.fragment,s),Oe=n(s),vs=i(s,"P",{"data-svelte-h":!0}),c(vs)!=="svelte-1cyo8oc"&&(vs.innerHTML=Oa),Ke=n(s),M(Vs.$$.fragment,s),st=n(s),Ys=i(s,"P",{"data-svelte-h":!0}),c(Ys)!=="svelte-14gqfge"&&(Ys.innerHTML=Ka),lt=n(s),As=i(s,"P",{"data-svelte-h":!0}),c(As)!=="svelte-1y80l2"&&(As.innerHTML=sn),et=n(s),I=i(s,"OL",{});var Sl=$l(I);U=i(Sl,"LI",{});var m=$l(U);Gl=i(m,"P",{"data-svelte-h":!0}),c(Gl)!=="svelte-19wri7m"&&(Gl.innerHTML=ln),Kt=n(m),vl=i(m,"P",{"data-svelte-h":!0}),c(vl)!=="svelte-1w1lp0n"&&(vl.innerHTML=en),sa=n(m),M(zs.$$.fragment,m),la=n(m),Vl=i(m,"P",{"data-svelte-h":!0}),c(Vl)!=="svelte-ce0g29"&&(Vl.innerHTML=tn),ea=n(m),Yl=i(m,"P",{"data-svelte-h":!0}),c(Yl)!=="svelte-fgwfk1"&&(Yl.innerHTML=an),ta=n(m),Al=i(m,"P",{"data-svelte-h":!0}),c(Al)!=="svelte-jof0o2"&&(Al.innerHTML=nn),m.forEach(e),aa=n(Sl),zl=i(Sl,"LI",{"data-svelte-h":!0}),c(zl)!=="svelte-1k2nlfe"&&(zl.innerHTML=pn),Sl.forEach(e),tt=n(s),Rs=i(s,"P",{"data-svelte-h":!0}),c(Rs)!=="svelte-1nruzcp"&&(Rs.innerHTML=cn),at=n(s),xs=i(s,"UL",{"data-svelte-h":!0}),c(xs)!=="svelte-12xd0n5"&&(xs.innerHTML=on),nt=n(s),Es=i(s,"P",{"data-svelte-h":!0}),c(Es)!=="svelte-1d6bbd1"&&(Es.innerHTML=Mn),pt=n(s),Qs=i(s,"P",{"data-svelte-h":!0}),c(Qs)!=="svelte-nnfxti"&&(Qs.innerHTML=un),it=n(s),ks=i(s,"P",{"data-svelte-h":!0}),c(ks)!=="svelte-2eh81g"&&(ks.innerHTML=rn),ct=n(s),Fs=i(s,"P",{"data-svelte-h":!0}),c(Fs)!=="svelte-1r2ikm1"&&(Fs.innerHTML=yn),ot=n(s),M(Hs.$$.fragment,s),Mt=n(s),qs=i(s,"P",{"data-svelte-h":!0}),c(qs)!=="svelte-23a28o"&&(qs.innerHTML=jn),ut=n(s),$s=i(s,"OL",{"data-svelte-h":!0}),c($s)!=="svelte-835v9i"&&($s.innerHTML=dn),rt=n(s),M(Ls.$$.fragment,s),yt=n(s),Z=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(Z)!=="svelte-1f7op2x"&&(Z.innerHTML=Jn),jt=n(s),M(Ds.$$.fragment,s),dt=n(s),C=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(C)!=="svelte-czp5cr"&&(C.innerHTML=hn),Jt=n(s),M(Ps.$$.fragment,s),ht=n(s),Os=i(s,"P",{"data-svelte-h":!0}),c(Os)!=="svelte-57upna"&&(Os.innerHTML=wn),wt=n(s),Ks=i(s,"OL",{"data-svelte-h":!0}),c(Ks)!=="svelte-jop1mq"&&(Ks.innerHTML=Tn),Tt=n(s),M(sl.$$.fragment,s),Ut=n(s),_=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(_)!=="svelte-w41hdb"&&(_.innerHTML=Un),mt=n(s),M(ll.$$.fragment,s),ft=n(s),B=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(B)!=="svelte-czp5cr"&&(B.innerHTML=mn),bt=n(s),M(el.$$.fragment,s),It=n(s),tl=i(s,"P",{"data-svelte-h":!0}),c(tl)!=="svelte-s6le15"&&(tl.innerHTML=fn),Zt=n(s),al=i(s,"OL",{"data-svelte-h":!0}),c(al)!=="svelte-q77lu7"&&(al.innerHTML=bn),Ct=n(s),M(nl.$$.fragment,s),_t=n(s),g=i(s,"OL",{start:!0,"data-svelte-h":!0}),c(g)!=="svelte-18b0p9"&&(g.innerHTML=In),Bt=n(s),M(pl.$$.fragment,s),gt=n(s),il=i(s,"P",{"data-svelte-h":!0}),c(il)!=="svelte-pxb0rv"&&(il.innerHTML=Zn),Wt=n(s),cl=i(s,"OL",{"data-svelte-h":!0}),c(cl)!=="svelte-1x5f67y"&&(cl.innerHTML=Cn),St=n(s),M(ol.$$.fragment,s),Xt=n(s),W=i(s,"OL",{});var Ot=$l(W);Rl=i(Ot,"LI",{"data-svelte-h":!0}),c(Rl)!=="svelte-1ix3dy3"&&(Rl.innerHTML=_n),na=n(Ot),h=i(Ot,"LI",{});var w=$l(h);xl=i(w,"P",{"data-svelte-h":!0}),c(xl)!=="svelte-1vo2i9p"&&(xl.innerHTML=Bn),pa=n(w),El=i(w,"P",{"data-svelte-h":!0}),c(El)!=="svelte-hk8fhr"&&(El.innerHTML=gn),ia=n(w),M(Ml.$$.fragment,w),ca=n(w),Ql=i(w,"P",{"data-svelte-h":!0}),c(Ql)!=="svelte-1ddq7v6"&&(Ql.innerHTML=Wn),oa=n(w),M(ul.$$.fragment,w),Ma=n(w),kl=i(w,"P",{"data-svelte-h":!0}),c(kl)!=="svelte-hmvopk"&&(kl.innerHTML=Sn),ua=n(w),M(rl.$$.fragment,w),ra=n(w),Fl=i(w,"P",{"data-svelte-h":!0}),c(Fl)!=="svelte-1hbu42f"&&(Fl.textContent=Xn),ya=n(w),M(yl.$$.fragment,w),ja=n(w),Hl=i(w,"P",{"data-svelte-h":!0}),c(Hl)!=="svelte-h6u0m6"&&(Hl.innerHTML=Nn),da=n(w),M(jl.$$.fragment,w),Ja=n(w),ql=i(w,"P",{"data-svelte-h":!0}),c(ql)!=="svelte-13dj8kw"&&(ql.textContent=Gn),w.forEach(e),Ot.forEach(e),Nt=n(s),M(dl.$$.fragment,s),Gt=n(s),Jl=i(s,"P",{"data-svelte-h":!0}),c(Jl)!=="svelte-5ua48b"&&(Jl.textContent=vn),vt=n(s),M(hl.$$.fragment,s),Vt=n(s),M(wl.$$.fragment,s),Yt=n(s),Tl=i(s,"OL",{"data-svelte-h":!0}),c(Tl)!=="svelte-rz68fo"&&(Tl.innerHTML=Vn),At=n(s),M(Ul.$$.fragment,s),zt=n(s),ml=i(s,"P",{"data-svelte-h":!0}),c(ml)!=="svelte-1fkk50y"&&(ml.innerHTML=Yn),Rt=n(s),fl=i(s,"P",{"data-svelte-h":!0}),c(fl)!=="svelte-1fjh6gc"&&(fl.innerHTML=An),xt=n(s),bl=i(s,"P",{"data-svelte-h":!0}),c(bl)!=="svelte-lzjsvg"&&(bl.innerHTML=zn),Et=n(s),M(Il.$$.fragment,s),Qt=n(s),Zl=i(s,"P",{"data-svelte-h":!0}),c(Zl)!=="svelte-1aiu2sl"&&(Zl.innerHTML=Rn),kt=n(s),Cl=i(s,"UL",{"data-svelte-h":!0}),c(Cl)!=="svelte-9racni"&&(Cl.innerHTML=xn),Ft=n(s),_l=i(s,"P",{"data-svelte-h":!0}),c(_l)!=="svelte-1pnpyer"&&(_l.textContent=En),Ht=n(s),Bl=i(s,"UL",{"data-svelte-h":!0}),c(Bl)!=="svelte-b1iyxy"&&(Bl.innerHTML=Qn),qt=n(s),gl=i(s,"P",{"data-svelte-h":!0}),c(gl)!=="svelte-1feo4z0"&&(gl.innerHTML=kn),$t=n(s),M(S.$$.fragment,s),Lt=n(s),M(Wl.$$.fragment,s),Dt=n(s),Ll=i(s,"P",{}),$l(Ll).forEach(e),this.h()},h(){Xl(T,"name","hf:doc:metadata"),Xl(T,"content",ap),Xl(Z,"start","2"),Xl(C,"start","3"),Xl(_,"start","2"),Xl(B,"start","3"),Xl(g,"start","2")},m(s,l){d(document.head,T),t(s,X,l),t(s,f,l),t(s,Nl,l),u(N,s,l),t(s,Pl,l),u(G,s,l),t(s,Ol,l),t(s,v,l),t(s,Kl,l),t(s,V,l),t(s,se,l),t(s,Y,l),t(s,le,l),t(s,A,l),t(s,ee,l),t(s,z,l),t(s,te,l),t(s,R,l),t(s,ae,l),t(s,x,l),t(s,ne,l),u(E,s,l),t(s,pe,l),t(s,Q,l),t(s,ie,l),t(s,k,l),t(s,ce,l),t(s,F,l),t(s,oe,l),t(s,H,l),t(s,Me,l),t(s,q,l),t(s,ue,l),t(s,$,l),t(s,re,l),t(s,L,l),t(s,ye,l),t(s,D,l),t(s,je,l),t(s,P,l),t(s,de,l),t(s,O,l),t(s,Je,l),t(s,K,l),t(s,he,l),t(s,ss,l),t(s,we,l),u(ls,s,l),t(s,Te,l),t(s,es,l),t(s,Ue,l),t(s,ts,l),t(s,me,l),u(as,s,l),t(s,fe,l),t(s,ns,l),t(s,be,l),u(ps,s,l),t(s,Ie,l),t(s,is,l),t(s,Ze,l),u(cs,s,l),t(s,Ce,l),t(s,os,l),t(s,_e,l),t(s,Ms,l),t(s,Be,l),u(us,s,l),t(s,ge,l),u(rs,s,l),t(s,We,l),t(s,ys,l),t(s,Se,l),u(js,s,l),t(s,Xe,l),u(ds,s,l),t(s,Ne,l),t(s,Js,l),t(s,Ge,l),u(hs,s,l),t(s,ve,l),t(s,ws,l),t(s,Ve,l),u(Ts,s,l),t(s,Ye,l),t(s,Us,l),t(s,Ae,l),u(ms,s,l),t(s,ze,l),t(s,fs,l),t(s,Re,l),u(bs,s,l),t(s,xe,l),t(s,Is,l),t(s,Ee,l),t(s,Zs,l),t(s,Qe,l),u(Cs,s,l),t(s,ke,l),t(s,_s,l),t(s,Fe,l),u(Bs,s,l),t(s,He,l),u(gs,s,l),t(s,qe,l),t(s,Ws,l),t(s,$e,l),u(Ss,s,l),t(s,Le,l),t(s,Xs,l),t(s,De,l),u(Ns,s,l),t(s,Pe,l),u(Gs,s,l),t(s,Oe,l),t(s,vs,l),t(s,Ke,l),u(Vs,s,l),t(s,st,l),t(s,Ys,l),t(s,lt,l),t(s,As,l),t(s,et,l),t(s,I,l),d(I,U),d(U,Gl),d(U,Kt),d(U,vl),d(U,sa),u(zs,U,null),d(U,la),d(U,Vl),d(U,ea),d(U,Yl),d(U,ta),d(U,Al),d(I,aa),d(I,zl),t(s,tt,l),t(s,Rs,l),t(s,at,l),t(s,xs,l),t(s,nt,l),t(s,Es,l),t(s,pt,l),t(s,Qs,l),t(s,it,l),t(s,ks,l),t(s,ct,l),t(s,Fs,l),t(s,ot,l),u(Hs,s,l),t(s,Mt,l),t(s,qs,l),t(s,ut,l),t(s,$s,l),t(s,rt,l),u(Ls,s,l),t(s,yt,l),t(s,Z,l),t(s,jt,l),u(Ds,s,l),t(s,dt,l),t(s,C,l),t(s,Jt,l),u(Ps,s,l),t(s,ht,l),t(s,Os,l),t(s,wt,l),t(s,Ks,l),t(s,Tt,l),u(sl,s,l),t(s,Ut,l),t(s,_,l),t(s,mt,l),u(ll,s,l),t(s,ft,l),t(s,B,l),t(s,bt,l),u(el,s,l),t(s,It,l),t(s,tl,l),t(s,Zt,l),t(s,al,l),t(s,Ct,l),u(nl,s,l),t(s,_t,l),t(s,g,l),t(s,Bt,l),u(pl,s,l),t(s,gt,l),t(s,il,l),t(s,Wt,l),t(s,cl,l),t(s,St,l),u(ol,s,l),t(s,Xt,l),t(s,W,l),d(W,Rl),d(W,na),d(W,h),d(h,xl),d(h,pa),d(h,El),d(h,ia),u(Ml,h,null),d(h,ca),d(h,Ql),d(h,oa),u(ul,h,null),d(h,Ma),d(h,kl),d(h,ua),u(rl,h,null),d(h,ra),d(h,Fl),d(h,ya),u(yl,h,null),d(h,ja),d(h,Hl),d(h,da),u(jl,h,null),d(h,Ja),d(h,ql),t(s,Nt,l),u(dl,s,l),t(s,Gt,l),t(s,Jl,l),t(s,vt,l),u(hl,s,l),t(s,Vt,l),u(wl,s,l),t(s,Yt,l),t(s,Tl,l),t(s,At,l),u(Ul,s,l),t(s,zt,l),t(s,ml,l),t(s,Rt,l),t(s,fl,l),t(s,xt,l),t(s,bl,l),t(s,Et,l),u(Il,s,l),t(s,Qt,l),t(s,Zl,l),t(s,kt,l),t(s,Cl,l),t(s,Ft,l),t(s,_l,l),t(s,Ht,l),t(s,Bl,l),t(s,qt,l),t(s,gl,l),t(s,$t,l),u(S,s,l),t(s,Lt,l),u(Wl,s,l),t(s,Dt,l),t(s,Ll,l),Pt=!0},p(s,[l]){const Sl={};l&2&&(Sl.$$scope={dirty:l,ctx:s}),S.$set(Sl)},i(s){Pt||(r(N.$$.fragment,s),r(G.$$.fragment,s),r(E.$$.fragment,s),r(ls.$$.fragment,s),r(as.$$.fragment,s),r(ps.$$.fragment,s),r(cs.$$.fragment,s),r(us.$$.fragment,s),r(rs.$$.fragment,s),r(js.$$.fragment,s),r(ds.$$.fragment,s),r(hs.$$.fragment,s),r(Ts.$$.fragment,s),r(ms.$$.fragment,s),r(bs.$$.fragment,s),r(Cs.$$.fragment,s),r(Bs.$$.fragment,s),r(gs.$$.fragment,s),r(Ss.$$.fragment,s),r(Ns.$$.fragment,s),r(Gs.$$.fragment,s),r(Vs.$$.fragment,s),r(zs.$$.fragment,s),r(Hs.$$.fragment,s),r(Ls.$$.fragment,s),r(Ds.$$.fragment,s),r(Ps.$$.fragment,s),r(sl.$$.fragment,s),r(ll.$$.fragment,s),r(el.$$.fragment,s),r(nl.$$.fragment,s),r(pl.$$.fragment,s),r(ol.$$.fragment,s),r(Ml.$$.fragment,s),r(ul.$$.fragment,s),r(rl.$$.fragment,s),r(yl.$$.fragment,s),r(jl.$$.fragment,s),r(dl.$$.fragment,s),r(hl.$$.fragment,s),r(wl.$$.fragment,s),r(Ul.$$.fragment,s),r(Il.$$.fragment,s),r(S.$$.fragment,s),r(Wl.$$.fragment,s),Pt=!0)},o(s){y(N.$$.fragment,s),y(G.$$.fragment,s),y(E.$$.fragment,s),y(ls.$$.fragment,s),y(as.$$.fragment,s),y(ps.$$.fragment,s),y(cs.$$.fragment,s),y(us.$$.fragment,s),y(rs.$$.fragment,s),y(js.$$.fragment,s),y(ds.$$.fragment,s),y(hs.$$.fragment,s),y(Ts.$$.fragment,s),y(ms.$$.fragment,s),y(bs.$$.fragment,s),y(Cs.$$.fragment,s),y(Bs.$$.fragment,s),y(gs.$$.fragment,s),y(Ss.$$.fragment,s),y(Ns.$$.fragment,s),y(Gs.$$.fragment,s),y(Vs.$$.fragment,s),y(zs.$$.fragment,s),y(Hs.$$.fragment,s),y(Ls.$$.fragment,s),y(Ds.$$.fragment,s),y(Ps.$$.fragment,s),y(sl.$$.fragment,s),y(ll.$$.fragment,s),y(el.$$.fragment,s),y(nl.$$.fragment,s),y(pl.$$.fragment,s),y(ol.$$.fragment,s),y(Ml.$$.fragment,s),y(ul.$$.fragment,s),y(rl.$$.fragment,s),y(yl.$$.fragment,s),y(jl.$$.fragment,s),y(dl.$$.fragment,s),y(hl.$$.fragment,s),y(wl.$$.fragment,s),y(Ul.$$.fragment,s),y(Il.$$.fragment,s),y(S.$$.fragment,s),y(Wl.$$.fragment,s),Pt=!1},d(s){s&&(e(X),e(f),e(Nl),e(Pl),e(Ol),e(v),e(Kl),e(V),e(se),e(Y),e(le),e(A),e(ee),e(z),e(te),e(R),e(ae),e(x),e(ne),e(pe),e(Q),e(ie),e(k),e(ce),e(F),e(oe),e(H),e(Me),e(q),e(ue),e($),e(re),e(L),e(ye),e(D),e(je),e(P),e(de),e(O),e(Je),e(K),e(he),e(ss),e(we),e(Te),e(es),e(Ue),e(ts),e(me),e(fe),e(ns),e(be),e(Ie),e(is),e(Ze),e(Ce),e(os),e(_e),e(Ms),e(Be),e(ge),e(We),e(ys),e(Se),e(Xe),e(Ne),e(Js),e(Ge),e(ve),e(ws),e(Ve),e(Ye),e(Us),e(Ae),e(ze),e(fs),e(Re),e(xe),e(Is),e(Ee),e(Zs),e(Qe),e(ke),e(_s),e(Fe),e(He),e(qe),e(Ws),e($e),e(Le),e(Xs),e(De),e(Pe),e(Oe),e(vs),e(Ke),e(st),e(Ys),e(lt),e(As),e(et),e(I),e(tt),e(Rs),e(at),e(xs),e(nt),e(Es),e(pt),e(Qs),e(it),e(ks),e(ct),e(Fs),e(ot),e(Mt),e(qs),e(ut),e($s),e(rt),e(yt),e(Z),e(jt),e(dt),e(C),e(Jt),e(ht),e(Os),e(wt),e(Ks),e(Tt),e(Ut),e(_),e(mt),e(ft),e(B),e(bt),e(It),e(tl),e(Zt),e(al),e(Ct),e(_t),e(g),e(Bt),e(gt),e(il),e(Wt),e(cl),e(St),e(Xt),e(W),e(Nt),e(Gt),e(Jl),e(vt),e(Vt),e(Yt),e(Tl),e(At),e(zt),e(ml),e(Rt),e(fl),e(xt),e(bl),e(Et),e(Qt),e(Zl),e(kt),e(Cl),e(Ft),e(_l),e(Ht),e(Bl),e(qt),e(gl),e($t),e(Lt),e(Dt),e(Ll)),e(T),j(N,s),j(G,s),j(E,s),j(ls,s),j(as,s),j(ps,s),j(cs,s),j(us,s),j(rs,s),j(js,s),j(ds,s),j(hs,s),j(Ts,s),j(ms,s),j(bs,s),j(Cs,s),j(Bs,s),j(gs,s),j(Ss,s),j(Ns,s),j(Gs,s),j(Vs,s),j(zs),j(Hs,s),j(Ls,s),j(Ds,s),j(Ps,s),j(sl,s),j(ll,s),j(el,s),j(nl,s),j(pl,s),j(ol,s),j(Ml),j(ul),j(rl),j(yl),j(jl),j(dl,s),j(hl,s),j(wl,s),j(Ul,s),j(Il,s),j(S,s),j(Wl,s)}}}const ap='{"title":"DeepSpeed","local":"deepspeed","sections":[{"title":"What is integrated?","local":"what-is-integrated","sections":[],"depth":2},{"title":"How it works?","local":"how-it-works","sections":[{"title":"Accelerate DeepSpeed Plugin","local":"accelerate-deepspeed-plugin","sections":[],"depth":3},{"title":"DeepSpeed Config File","local":"deepspeed-config-file","sections":[],"depth":3}],"depth":2},{"title":"Saving and loading","local":"saving-and-loading","sections":[],"depth":2},{"title":"ZeRO Inference","local":"zero-inference","sections":[],"depth":2},{"title":"Few caveats to be aware of","local":"few-caveats-to-be-aware-of","sections":[],"depth":2},{"title":"Multi-node DeepSpeed","local":"multi-node-deepspeed","sections":[],"depth":2},{"title":"DeepSpeed Resources","local":"deepspeed-resources","sections":[],"depth":2}],"depth":1}';function np(Dl){return Hn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class up extends $n{constructor(T){super(),Ln(this,T,np,tp,Fn,{})}}export{up as component};
