import{s as Xe,n as Ye,o as Ze}from"../chunks/scheduler.b9285784.js";import{S as et,i as tt,e as s,s as r,c as l,h as at,a as i,d as a,b as o,f as _,g as c,j as P,k as f,l as p,m as n,n as d,t as m,o as g,p as u}from"../chunks/index.26bc89a1.js";import{C as nt,H as N,E as rt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e364eac2.js";import{D as T}from"../chunks/Docstring.43e9d871.js";function ot(Fe){let h,re,ae,oe,D,se,C,le,S,ie,v,O,ke,J,We=`Plugin for Megatron-LM to enable tensor, pipeline, sequence and data parallelism. Also to enable selective
activation recomputation and optimized fused kernels.`,ce,q,de,M,k,ze,K,je=`Dummy scheduler presents model parameters or param groups, this is primarily used to follow conventional training
loop when scheduler config is specified in the deepspeed config file.`,me,z,ge,$,E,Ee,Q,He="Dummy dataloader presents model parameters or param groups, this is primarily used to follow conventional training",ue,I,pe,b,A,Ie,X,Re="Abstract class for batching, forward pass and loss handler.",_e,G,fe,y,B,Ae,Y,Ue="GPT train step class.",he,V,ve,x,F,Ge,Z,Je="Bert train step class.",Me,W,$e,L,j,Be,ee,Ke="T5 train step class.",be,H,ye,w,R,Ve,te,Qe="Average losses across data parallel group.",xe,U,Le,ne,we;return D=new nt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),C=new N({props:{title:"Megatron-LM utilities",local:"megatron-lm-utilities",headingTag:"h1"}}),S=new N({props:{title:"MegatronLMPlugin",local:"accelerate.utils.MegatronLMPlugin",headingTag:"h2"}}),O=new T({props:{name:"class accelerate.utils.MegatronLMPlugin",anchor:"accelerate.utils.MegatronLMPlugin",parameters:[{name:"tp_degree",val:": int = None"},{name:"pp_degree",val:": int = None"},{name:"num_micro_batches",val:": int = None"},{name:"gradient_clipping",val:": float = None"},{name:"sequence_parallelism",val:": bool = None"},{name:"recompute_activations",val:": bool = None"},{name:"use_distributed_optimizer",val:": bool = None"},{name:"pipeline_model_parallel_split_rank",val:": int = None"},{name:"num_layers_per_virtual_pipeline_stage",val:": int = None"},{name:"is_train_batch_min",val:": str = True"},{name:"train_iters",val:": int = None"},{name:"train_samples",val:": int = None"},{name:"weight_decay_incr_style",val:": str = 'constant'"},{name:"start_weight_decay",val:": float = None"},{name:"end_weight_decay",val:": float = None"},{name:"lr_decay_style",val:": str = 'linear'"},{name:"lr_decay_iters",val:": int = None"},{name:"lr_decay_samples",val:": int = None"},{name:"lr_warmup_iters",val:": int = None"},{name:"lr_warmup_samples",val:": int = None"},{name:"lr_warmup_fraction",val:": float = None"},{name:"min_lr",val:": float = 0"},{name:"consumed_samples",val:": list = None"},{name:"no_wd_decay_cond",val:": typing.Optional[typing.Callable] = None"},{name:"scale_lr_cond",val:": typing.Optional[typing.Callable] = None"},{name:"lr_mult",val:": float = 1.0"},{name:"megatron_dataset_flag",val:": bool = False"},{name:"seq_length",val:": int = None"},{name:"encoder_seq_length",val:": int = None"},{name:"decoder_seq_length",val:": int = None"},{name:"tensorboard_dir",val:": str = None"},{name:"set_all_logging_options",val:": bool = False"},{name:"eval_iters",val:": int = 100"},{name:"eval_interval",val:": int = 1000"},{name:"return_logits",val:": bool = False"},{name:"custom_train_step_class",val:": typing.Optional[typing.Any] = None"},{name:"custom_train_step_kwargs",val:": typing.Optional[dict[str, typing.Any]] = None"},{name:"custom_model_provider_function",val:": typing.Optional[typing.Callable] = None"},{name:"custom_prepare_model_function",val:": typing.Optional[typing.Callable] = None"},{name:"custom_megatron_datasets_provider_function",val:": typing.Optional[typing.Callable] = None"},{name:"custom_get_batch_function",val:": typing.Optional[typing.Callable] = None"},{name:"custom_loss_function",val:": typing.Optional[typing.Callable] = None"},{name:"other_megatron_args",val:": typing.Optional[dict[str, typing.Any]] = None"}],parametersDescription:[{anchor:"accelerate.utils.MegatronLMPlugin.tp_degree",description:`<strong>tp_degree</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Tensor parallelism degree.`,name:"tp_degree"},{anchor:"accelerate.utils.MegatronLMPlugin.pp_degree",description:`<strong>pp_degree</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Pipeline parallelism degree.`,name:"pp_degree"},{anchor:"accelerate.utils.MegatronLMPlugin.num_micro_batches",description:`<strong>num_micro_batches</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of micro-batches.`,name:"num_micro_batches"},{anchor:"accelerate.utils.MegatronLMPlugin.gradient_clipping",description:`<strong>gradient_clipping</strong> (<code>float</code>, defaults to <code>None</code>) &#x2014;
Gradient clipping value based on global L2 Norm (0 to disable).`,name:"gradient_clipping"},{anchor:"accelerate.utils.MegatronLMPlugin.sequence_parallelism",description:`<strong>sequence_parallelism</strong> (<code>bool</code>, defaults to <code>None</code>) &#x2014;
Enable sequence parallelism.`,name:"sequence_parallelism"},{anchor:"accelerate.utils.MegatronLMPlugin.recompute_activations",description:`<strong>recompute_activations</strong> (<code>bool</code>, defaults to <code>None</code>) &#x2014;
Enable selective activation recomputation.`,name:"recompute_activations"},{anchor:"accelerate.utils.MegatronLMPlugin.use_distributed_optimizr",description:`<strong>use_distributed_optimizr</strong> (<code>bool</code>, defaults to <code>None</code>) &#x2014;
Enable distributed optimizer.`,name:"use_distributed_optimizr"},{anchor:"accelerate.utils.MegatronLMPlugin.pipeline_model_parallel_split_rank",description:`<strong>pipeline_model_parallel_split_rank</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Rank where encoder and decoder should be split.`,name:"pipeline_model_parallel_split_rank"},{anchor:"accelerate.utils.MegatronLMPlugin.num_layers_per_virtual_pipeline_stage",description:`<strong>num_layers_per_virtual_pipeline_stage</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of layers per virtual pipeline stage.`,name:"num_layers_per_virtual_pipeline_stage"},{anchor:"accelerate.utils.MegatronLMPlugin.is_train_batch_min",description:`<strong>is_train_batch_min</strong> (<code>str</code>, defaults to <code>True</code>) &#x2014;
If both tran &amp; eval dataloaders are specified, this will decide the <code>micro_batch_size</code>.`,name:"is_train_batch_min"},{anchor:"accelerate.utils.MegatronLMPlugin.train_iters",description:`<strong>train_iters</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Total number of samples to train over all training runs. Note that either train-iters or train-samples
should be provided when using <code>MegatronLMDummyScheduler</code>.`,name:"train_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.train_samples",description:`<strong>train_samples</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Total number of samples to train over all training runs. Note that either train-iters or train-samples
should be provided when using <code>MegatronLMDummyScheduler</code>.`,name:"train_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.weight_decay_incr_style",description:`<strong>weight_decay_incr_style</strong> (<code>str</code>, defaults to <code>&apos;constant&apos;</code>) &#x2014;
Weight decay increment function. choices=[&#x201C;constant&#x201D;, &#x201C;linear&#x201D;, &#x201C;cosine&#x201D;].`,name:"weight_decay_incr_style"},{anchor:"accelerate.utils.MegatronLMPlugin.start_weight_decay",description:`<strong>start_weight_decay</strong> (<code>float</code>, defaults to <code>None</code>) &#x2014;
Initial weight decay coefficient for L2 regularization.`,name:"start_weight_decay"},{anchor:"accelerate.utils.MegatronLMPlugin.end_weight_decay",description:`<strong>end_weight_decay</strong> (<code>float</code>, defaults to <code>None</code>) &#x2014;
End of run weight decay coefficient for L2 regularization.`,name:"end_weight_decay"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_decay_style",description:`<strong>lr_decay_style</strong> (<code>str</code>, defaults to <code>&apos;linear&apos;</code>) &#x2014;
Learning rate decay function. choices=[&#x2018;constant&#x2019;, &#x2018;linear&#x2019;, &#x2018;cosine&#x2019;].`,name:"lr_decay_style"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_decay_iters",description:`<strong>lr_decay_iters</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of iterations for learning rate decay. If None defaults to <code>train_iters</code>.`,name:"lr_decay_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_decay_samples",description:`<strong>lr_decay_samples</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of samples for learning rate decay. If None defaults to <code>train_samples</code>.`,name:"lr_decay_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_warmup_iters",description:`<strong>lr_warmup_iters</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of iterations to linearly warmup learning rate over.`,name:"lr_warmup_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_warmup_samples",description:`<strong>lr_warmup_samples</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Number of samples to linearly warmup learning rate over.`,name:"lr_warmup_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_warmup_fraction",description:`<strong>lr_warmup_fraction</strong> (<code>float</code>, defaults to <code>None</code>) &#x2014;
Fraction of lr-warmup-(iters/samples) to linearly warmup learning rate over.`,name:"lr_warmup_fraction"},{anchor:"accelerate.utils.MegatronLMPlugin.min_lr",description:`<strong>min_lr</strong> (<code>float</code>, defaults to <code>0</code>) &#x2014;
Minimum value for learning rate. The scheduler clip values below this threshold.`,name:"min_lr"},{anchor:"accelerate.utils.MegatronLMPlugin.consumed_samples",description:`<strong>consumed_samples</strong> (<code>List</code>, defaults to <code>None</code>) &#x2014;
Number of samples consumed in the same order as the dataloaders to <code>accelerator.prepare</code> call.`,name:"consumed_samples"},{anchor:"accelerate.utils.MegatronLMPlugin.no_wd_decay_cond",description:`<strong>no_wd_decay_cond</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Condition to disable weight decay.`,name:"no_wd_decay_cond"},{anchor:"accelerate.utils.MegatronLMPlugin.scale_lr_cond",description:`<strong>scale_lr_cond</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Condition to scale learning rate.`,name:"scale_lr_cond"},{anchor:"accelerate.utils.MegatronLMPlugin.lr_mult",description:`<strong>lr_mult</strong> (<code>float</code>, defaults to <code>1.0</code>) &#x2014;
Learning rate multiplier.`,name:"lr_mult"},{anchor:"accelerate.utils.MegatronLMPlugin.megatron_dataset_flag",description:`<strong>megatron_dataset_flag</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether the format of dataset follows Megatron-LM Indexed/Cached/MemoryMapped format.`,name:"megatron_dataset_flag"},{anchor:"accelerate.utils.MegatronLMPlugin.seq_length",description:`<strong>seq_length</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Maximum sequence length to process.`,name:"seq_length"},{anchor:"accelerate.utils.MegatronLMPlugin.encoder_seq_length",description:`<strong>encoder_seq_length</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Maximum sequence length to process for the encoder.`,name:"encoder_seq_length"},{anchor:"accelerate.utils.MegatronLMPlugin.decoder_seq_length",description:`<strong>decoder_seq_length</strong> (<code>int</code>, defaults to <code>None</code>) &#x2014;
Maximum sequence length to process for the decoder.`,name:"decoder_seq_length"},{anchor:"accelerate.utils.MegatronLMPlugin.tensorboard_dir",description:`<strong>tensorboard_dir</strong> (<code>str</code>, defaults to <code>None</code>) &#x2014;
Path to save tensorboard logs.`,name:"tensorboard_dir"},{anchor:"accelerate.utils.MegatronLMPlugin.set_all_logging_options",description:`<strong>set_all_logging_options</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to set all logging options.`,name:"set_all_logging_options"},{anchor:"accelerate.utils.MegatronLMPlugin.eval_iters",description:`<strong>eval_iters</strong> (<code>int</code>, defaults to <code>100</code>) &#x2014;
Number of iterations to run for evaluation validation/test for.`,name:"eval_iters"},{anchor:"accelerate.utils.MegatronLMPlugin.eval_interval",description:`<strong>eval_interval</strong> (<code>int</code>, defaults to <code>1000</code>) &#x2014;
Interval between running evaluation on validation set.`,name:"eval_interval"},{anchor:"accelerate.utils.MegatronLMPlugin.return_logits",description:`<strong>return_logits</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to return logits from the model.`,name:"return_logits"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_train_step_class",description:`<strong>custom_train_step_class</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom train step class.`,name:"custom_train_step_class"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_train_step_kwargs",description:`<strong>custom_train_step_kwargs</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom train step kwargs.`,name:"custom_train_step_kwargs"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_model_provider_function",description:`<strong>custom_model_provider_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom model provider function.`,name:"custom_model_provider_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_prepare_model_function",description:`<strong>custom_prepare_model_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom prepare model function.`,name:"custom_prepare_model_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_megatron_datasets_provider_function",description:`<strong>custom_megatron_datasets_provider_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom megatron train_valid_test datasets provider function.`,name:"custom_megatron_datasets_provider_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_get_batch_function",description:`<strong>custom_get_batch_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom get batch function.`,name:"custom_get_batch_function"},{anchor:"accelerate.utils.MegatronLMPlugin.custom_loss_function",description:`<strong>custom_loss_function</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Custom loss function.`,name:"custom_loss_function"},{anchor:"accelerate.utils.MegatronLMPlugin.other_megatron_args",description:`<strong>other_megatron_args</strong> (<code>Optional</code>, defaults to <code>None</code>) &#x2014;
Other Megatron-LM arguments. Please refer Megatron-LM.`,name:"other_megatron_args"}],source:"https://github.com/huggingface/accelerate/blob/main/src/accelerate/utils/dataclasses.py#L2215"}}),q=new N({props:{title:"MegatronLMDummyScheduler",local:"accelerate.utils.MegatronLMDummyScheduler",headingTag:"h2"}}),k=new T({props:{name:"class accelerate.utils.MegatronLMDummyScheduler",anchor:"accelerate.utils.MegatronLMDummyScheduler",parameters:[{name:"optimizer",val:""},{name:"total_num_steps",val:" = None"},{name:"warmup_num_steps",val:" = 0"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"accelerate.utils.MegatronLMDummyScheduler.optimizer",description:`<strong>optimizer</strong> (<code>torch.optim.optimizer.Optimizer</code>) &#x2014;
The optimizer to wrap.`,name:"optimizer"},{anchor:"accelerate.utils.MegatronLMDummyScheduler.total_num_steps",description:`<strong>total_num_steps</strong> (int) &#x2014;
Total number of steps.`,name:"total_num_steps"},{anchor:"accelerate.utils.MegatronLMDummyScheduler.warmup_num_steps",description:`<strong>warmup_num_steps</strong> (int) &#x2014;
Number of steps for warmup.`,name:"warmup_num_steps"},{anchor:"accelerate.utils.MegatronLMDummyScheduler.*kwargs",description:`*<strong>*kwargs</strong> (additional keyword arguments, <em>optional</em>) &#x2014;
Other arguments.`,name:"*kwargs"}],source:"https://github.com/huggingface/accelerate/blob/main/src/accelerate/utils/megatron_lm.py#L391"}}),z=new N({props:{title:"MegatronLMDummyDataLoader",local:"accelerate.utils.MegatronLMDummyDataLoader",headingTag:"h2"}}),E=new T({props:{name:"class accelerate.utils.MegatronLMDummyDataLoader",anchor:"accelerate.utils.MegatronLMDummyDataLoader",parameters:[{name:"**dataset_kwargs",val:""}],parametersDescription:[{anchor:"accelerate.utils.MegatronLMDummyDataLoader.*dataset_kwargs",description:"*<strong>*dataset_kwargs</strong> &#x2014; Megatron data arguments.",name:"*dataset_kwargs"}],source:"https://github.com/huggingface/accelerate/blob/main/src/accelerate/utils/megatron_lm.py#L175"}}),I=new N({props:{title:"AbstractTrainStep",local:"accelerate.utils.AbstractTrainStep",headingTag:"h2"}}),A=new T({props:{name:"class accelerate.utils.AbstractTrainStep",anchor:"accelerate.utils.AbstractTrainStep",parameters:[{name:"name",val:""}],source:"https://github.com/huggingface/accelerate/blob/main/src/accelerate/utils/megatron_lm.py#L428"}}),G=new N({props:{title:"GPTTrainStep",local:"accelerate.utils.GPTTrainStep",headingTag:"h2"}}),B=new T({props:{name:"class accelerate.utils.GPTTrainStep",anchor:"accelerate.utils.GPTTrainStep",parameters:[{name:"accelerator",val:""},{name:"args",val:""}],parametersDescription:[{anchor:"accelerate.utils.GPTTrainStep.args",description:"<strong>args</strong> (<code>argparse.Namespace</code>) &#x2014; Megatron-LM arguments.",name:"args"}],source:"https://github.com/huggingface/accelerate/blob/main/src/accelerate/utils/megatron_lm.py#L587"}}),V=new N({props:{title:"BertTrainStep",local:"accelerate.utils.BertTrainStep",headingTag:"h2"}}),F=new T({props:{name:"class accelerate.utils.BertTrainStep",anchor:"accelerate.utils.BertTrainStep",parameters:[{name:"accelerator",val:""},{name:"args",val:""}],parametersDescription:[{anchor:"accelerate.utils.BertTrainStep.args",description:"<strong>args</strong> (<code>argparse.Namespace</code>) &#x2014; Megatron-LM arguments.",name:"args"}],source:"https://github.com/huggingface/accelerate/blob/main/src/accelerate/utils/megatron_lm.py#L445"}}),W=new N({props:{title:"T5TrainStep",local:"accelerate.utils.T5TrainStep",headingTag:"h2"}}),j=new T({props:{name:"class accelerate.utils.T5TrainStep",anchor:"accelerate.utils.T5TrainStep",parameters:[{name:"accelerator",val:""},{name:"args",val:""}],parametersDescription:[{anchor:"accelerate.utils.T5TrainStep.args",description:"<strong>args</strong> (<code>argparse.Namespace</code>) &#x2014; Megatron-LM arguments.",name:"args"}],source:"https://github.com/huggingface/accelerate/blob/main/src/accelerate/utils/megatron_lm.py#L719"}}),H=new N({props:{title:"avg_losses_across_data_parallel_group",local:"accelerate.utils.avg_losses_across_data_parallel_group",headingTag:"h2"}}),R=new T({props:{name:"accelerate.utils.avg_losses_across_data_parallel_group",anchor:"accelerate.utils.avg_losses_across_data_parallel_group",parameters:[{name:"losses",val:""}],parametersDescription:[{anchor:"accelerate.utils.avg_losses_across_data_parallel_group.losses",description:"<strong>losses</strong> (List[Tensor]) &#x2014; List of losses to average across data parallel group.",name:"losses"}],source:"https://github.com/huggingface/accelerate/blob/main/src/accelerate/utils/megatron_lm.py#L1393"}}),U=new rt({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/package_reference/megatron_lm.md"}}),{c(){h=s("meta"),re=r(),ae=s("p"),oe=r(),l(D.$$.fragment),se=r(),l(C.$$.fragment),le=r(),l(S.$$.fragment),ie=r(),v=s("div"),l(O.$$.fragment),ke=r(),J=s("p"),J.textContent=We,ce=r(),l(q.$$.fragment),de=r(),M=s("div"),l(k.$$.fragment),ze=r(),K=s("p"),K.textContent=je,me=r(),l(z.$$.fragment),ge=r(),$=s("div"),l(E.$$.fragment),Ee=r(),Q=s("p"),Q.textContent=He,ue=r(),l(I.$$.fragment),pe=r(),b=s("div"),l(A.$$.fragment),Ie=r(),X=s("p"),X.textContent=Re,_e=r(),l(G.$$.fragment),fe=r(),y=s("div"),l(B.$$.fragment),Ae=r(),Y=s("p"),Y.textContent=Ue,he=r(),l(V.$$.fragment),ve=r(),x=s("div"),l(F.$$.fragment),Ge=r(),Z=s("p"),Z.textContent=Je,Me=r(),l(W.$$.fragment),$e=r(),L=s("div"),l(j.$$.fragment),Be=r(),ee=s("p"),ee.textContent=Ke,be=r(),l(H.$$.fragment),ye=r(),w=s("div"),l(R.$$.fragment),Ve=r(),te=s("p"),te.textContent=Qe,xe=r(),l(U.$$.fragment),Le=r(),ne=s("p"),this.h()},l(e){const t=at("svelte-u9bgzb",document.head);h=i(t,"META",{name:!0,content:!0}),t.forEach(a),re=o(e),ae=i(e,"P",{}),_(ae).forEach(a),oe=o(e),c(D.$$.fragment,e),se=o(e),c(C.$$.fragment,e),le=o(e),c(S.$$.fragment,e),ie=o(e),v=i(e,"DIV",{class:!0});var Ne=_(v);c(O.$$.fragment,Ne),ke=o(Ne),J=i(Ne,"P",{"data-svelte-h":!0}),P(J)!=="svelte-51ubws"&&(J.textContent=We),Ne.forEach(a),ce=o(e),c(q.$$.fragment,e),de=o(e),M=i(e,"DIV",{class:!0});var Pe=_(M);c(k.$$.fragment,Pe),ze=o(Pe),K=i(Pe,"P",{"data-svelte-h":!0}),P(K)!=="svelte-mxz6g6"&&(K.textContent=je),Pe.forEach(a),me=o(e),c(z.$$.fragment,e),ge=o(e),$=i(e,"DIV",{class:!0});var Te=_($);c(E.$$.fragment,Te),Ee=o(Te),Q=i(Te,"P",{"data-svelte-h":!0}),P(Q)!=="svelte-1r57fhg"&&(Q.textContent=He),Te.forEach(a),ue=o(e),c(I.$$.fragment,e),pe=o(e),b=i(e,"DIV",{class:!0});var De=_(b);c(A.$$.fragment,De),Ie=o(De),X=i(De,"P",{"data-svelte-h":!0}),P(X)!=="svelte-7ith7r"&&(X.textContent=Re),De.forEach(a),_e=o(e),c(G.$$.fragment,e),fe=o(e),y=i(e,"DIV",{class:!0});var Ce=_(y);c(B.$$.fragment,Ce),Ae=o(Ce),Y=i(Ce,"P",{"data-svelte-h":!0}),P(Y)!=="svelte-8o1bfn"&&(Y.textContent=Ue),Ce.forEach(a),he=o(e),c(V.$$.fragment,e),ve=o(e),x=i(e,"DIV",{class:!0});var Se=_(x);c(F.$$.fragment,Se),Ge=o(Se),Z=i(Se,"P",{"data-svelte-h":!0}),P(Z)!=="svelte-1af56j7"&&(Z.textContent=Je),Se.forEach(a),Me=o(e),c(W.$$.fragment,e),$e=o(e),L=i(e,"DIV",{class:!0});var Oe=_(L);c(j.$$.fragment,Oe),Be=o(Oe),ee=i(Oe,"P",{"data-svelte-h":!0}),P(ee)!=="svelte-1w9r8gv"&&(ee.textContent=Ke),Oe.forEach(a),be=o(e),c(H.$$.fragment,e),ye=o(e),w=i(e,"DIV",{class:!0});var qe=_(w);c(R.$$.fragment,qe),Ve=o(qe),te=i(qe,"P",{"data-svelte-h":!0}),P(te)!=="svelte-39hqbl"&&(te.textContent=Qe),qe.forEach(a),xe=o(e),c(U.$$.fragment,e),Le=o(e),ne=i(e,"P",{}),_(ne).forEach(a),this.h()},h(){f(h,"name","hf:doc:metadata"),f(h,"content",st),f(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(b,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(w,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){p(document.head,h),n(e,re,t),n(e,ae,t),n(e,oe,t),d(D,e,t),n(e,se,t),d(C,e,t),n(e,le,t),d(S,e,t),n(e,ie,t),n(e,v,t),d(O,v,null),p(v,ke),p(v,J),n(e,ce,t),d(q,e,t),n(e,de,t),n(e,M,t),d(k,M,null),p(M,ze),p(M,K),n(e,me,t),d(z,e,t),n(e,ge,t),n(e,$,t),d(E,$,null),p($,Ee),p($,Q),n(e,ue,t),d(I,e,t),n(e,pe,t),n(e,b,t),d(A,b,null),p(b,Ie),p(b,X),n(e,_e,t),d(G,e,t),n(e,fe,t),n(e,y,t),d(B,y,null),p(y,Ae),p(y,Y),n(e,he,t),d(V,e,t),n(e,ve,t),n(e,x,t),d(F,x,null),p(x,Ge),p(x,Z),n(e,Me,t),d(W,e,t),n(e,$e,t),n(e,L,t),d(j,L,null),p(L,Be),p(L,ee),n(e,be,t),d(H,e,t),n(e,ye,t),n(e,w,t),d(R,w,null),p(w,Ve),p(w,te),n(e,xe,t),d(U,e,t),n(e,Le,t),n(e,ne,t),we=!0},p:Ye,i(e){we||(m(D.$$.fragment,e),m(C.$$.fragment,e),m(S.$$.fragment,e),m(O.$$.fragment,e),m(q.$$.fragment,e),m(k.$$.fragment,e),m(z.$$.fragment,e),m(E.$$.fragment,e),m(I.$$.fragment,e),m(A.$$.fragment,e),m(G.$$.fragment,e),m(B.$$.fragment,e),m(V.$$.fragment,e),m(F.$$.fragment,e),m(W.$$.fragment,e),m(j.$$.fragment,e),m(H.$$.fragment,e),m(R.$$.fragment,e),m(U.$$.fragment,e),we=!0)},o(e){g(D.$$.fragment,e),g(C.$$.fragment,e),g(S.$$.fragment,e),g(O.$$.fragment,e),g(q.$$.fragment,e),g(k.$$.fragment,e),g(z.$$.fragment,e),g(E.$$.fragment,e),g(I.$$.fragment,e),g(A.$$.fragment,e),g(G.$$.fragment,e),g(B.$$.fragment,e),g(V.$$.fragment,e),g(F.$$.fragment,e),g(W.$$.fragment,e),g(j.$$.fragment,e),g(H.$$.fragment,e),g(R.$$.fragment,e),g(U.$$.fragment,e),we=!1},d(e){e&&(a(re),a(ae),a(oe),a(se),a(le),a(ie),a(v),a(ce),a(de),a(M),a(me),a(ge),a($),a(ue),a(pe),a(b),a(_e),a(fe),a(y),a(he),a(ve),a(x),a(Me),a($e),a(L),a(be),a(ye),a(w),a(xe),a(Le),a(ne)),a(h),u(D,e),u(C,e),u(S,e),u(O),u(q,e),u(k),u(z,e),u(E),u(I,e),u(A),u(G,e),u(B),u(V,e),u(F),u(W,e),u(j),u(H,e),u(R),u(U,e)}}}const st='{"title":"Megatron-LM utilities","local":"megatron-lm-utilities","sections":[{"title":"MegatronLMPlugin","local":"accelerate.utils.MegatronLMPlugin","sections":[],"depth":2},{"title":"MegatronLMDummyScheduler","local":"accelerate.utils.MegatronLMDummyScheduler","sections":[],"depth":2},{"title":"MegatronLMDummyDataLoader","local":"accelerate.utils.MegatronLMDummyDataLoader","sections":[],"depth":2},{"title":"AbstractTrainStep","local":"accelerate.utils.AbstractTrainStep","sections":[],"depth":2},{"title":"GPTTrainStep","local":"accelerate.utils.GPTTrainStep","sections":[],"depth":2},{"title":"BertTrainStep","local":"accelerate.utils.BertTrainStep","sections":[],"depth":2},{"title":"T5TrainStep","local":"accelerate.utils.T5TrainStep","sections":[],"depth":2},{"title":"avg_losses_across_data_parallel_group","local":"accelerate.utils.avg_losses_across_data_parallel_group","sections":[],"depth":2}],"depth":1}';function lt(Fe){return Ze(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gt extends et{constructor(h){super(),tt(this,h,lt,ot,Xe,{})}}export{gt as component};
