import{s as jt,o as gt,n as Gt}from"../chunks/scheduler.defa9a21.js";import{S as It,i as vt,g as o,s as n,r as d,A as At,h as s,f as l,c as i,j as ft,u as y,x as c,k as bt,y as _t,a,v as M,d as p,t as m,w}from"../chunks/index.fe795e71.js";import{T as Bt}from"../chunks/Tip.179eb360.js";import{C as ie}from"../chunks/CodeBlock.204b6c34.js";import{H as oe,E as kt}from"../chunks/getInferenceSnippets.2234a8dd.js";function $t(se){let r,T="Disabling <code>no_sync</code> means there <em>will be slowdown</em> due the extra data syncs, as explained by the earlier sections of this guide.";return{c(){r=o("p"),r.innerHTML=T},l(u){r=s(u,"P",{"data-svelte-h":!0}),c(r)!=="svelte-15yoesw"&&(r.innerHTML=T)},m(u,ae){a(u,r,ae)},p:Gt,d(u){u&&l(r)}}}function xt(se){let r,T,u,ae,U,ce,J,Ee=`PyTorch’s distributed module operates by communicating back and forth between all of the GPUs in your system.
This communication takes time, and ensuring all processes know the states of each other happens at particular triggerpoints
when using the <code>ddp</code> module.`,re,C,Ne=`These triggerpoints are added to the PyTorch model, specifically their <code>forward()</code> and <code>backward()</code> methods.
This happens when the model is wrapped with <code>DistributedDataParallel</code>:`,de,f,ye,b,De='In Accelerate this conversion happens automatically when calling <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.prepare">prepare()</a> and passing in your model.',Me,j,pe,g,me,G,Ke=`You now understand that PyTorch adds hooks to the <code>forward</code> and <code>backward</code> method of your PyTorch model when
training in a distributed setup. But how does this risk slowing down your code?`,we,I,qe=`In DDP (distributed data parallel), the specific order in which processes are performed and ran are expected
at specific points and these must also occur at roughly the same time before moving on.`,ue,v,Oe=`The most direct example is when you update model parameters through
<code>optimizer.step()</code>.
Without gradient accumulation, all instances of the model need to have updated
their gradients computed, collated, and updated before moving on to the next
batch of data.
When performing gradient accumulation, you accumulate <code>n</code> loss gradients and
skip <code>optimizer.step()</code> until <code>n</code> batches have been reached. As all training
processes only need to synchronize by the time <code>optimizer.step()</code> is called,
without any modification to your training step, this needless inter-process
communication can cause a significant slowdown.`,he,A,et="How can you avoid this overhead?",Te,_,Ue,B,tt=`Since you are skipping model parameter updates when training on these batches, their gradients do not need to be synchronized until the point where <code>optimizer.step()</code> is actually called.
PyTorch cannot automagically tell when you need to do this, but they do provide a tool to help through the <a href="https://pytorch.org/docs/stable/generated/torch.nn.parallel.DistributedDataParallel.html#torch.nn.parallel.DistributedDataParallel.no_sync" rel="nofollow"><code>no_sync</code></a> context manager
that is added to your model after converting it to DDP.`,Je,k,lt=`Under this context manager, PyTorch will skip synchronizing the gradients when
<code>.backward()</code> is called, and the first call to <code>.backward()</code> outside this
context manager will trigger the synchronization. See an example below:`,Ce,$,fe,x,at=`In Accelerate to make this an API that can be called no matter the training device (though it may not do anything if you are not in a distributed system!),
<code>ddp_model.no_sync</code> gets replaced with <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.no_sync">no_sync()</a> and operates the same way:`,be,Z,je,z,nt=`As you may expect, the <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.accumulate">accumulate()</a> function wraps around this conditional check by keeping track of the current batch number, leaving you with the final
gradient accumulation API:`,ge,H,Ge,W,it="As a result, you should either use <em><code>accelerator.accumulate</code> or <code>accelerator.no_sync</code></em> when it comes to API choice.",Ie,V,ve,R,ot="To set up a realistic example, consider the following setup:",Ae,X,st='<li>Two single-GPU T4 nodes and one node with two GPUs</li> <li>Each GPU is a T4, and are hosted on GCP</li> <li>The script used is a modification of the <a href="https://github.com/muellerzr/timing_experiments/blob/main/baseline.py" rel="nofollow">NLP Example</a> script</li> <li>Batch size per GPU is 16, and gradients are accumulated every 4 steps</li>',_e,P,ct='All scripts are available in <a href="https://github.com/muellerzr/timing_experiments" rel="nofollow">this repository</a>.',Be,S,rt=`If not careful about gradient synchronization and GPU communication, a <em>large</em> amount of time can be wasted
from when these GPUs communicate to each other during unnecessary periods.`,ke,L,dt="By how much?",$e,Y,yt="Reference:",xe,F,Mt='<li>Baseline: uses no synchronization practices discussed here</li> <li><code>no_sync</code> improperly: <code>no_sync</code> only around the <code>backward</code> call, not the <code>forward</code></li> <li><code>no_sync</code>: using the <code>no_sync</code> pattern properly</li> <li><code>accumulate</code>: using <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.accumulate">accumulate()</a> properly</li>',Ze,Q,pt="Below are the average seconds per batch iterating over 29 batches of data for each setup on both a single node and on the dual-node setup:",ze,E,mt='<thead><tr><th align="center"></th> <th align="center">Baseline</th> <th align="center"><code>no_sync</code> improperly</th> <th align="center"><code>no_sync</code></th> <th align="center"><code>accumulate</code></th></tr></thead> <tbody><tr><td align="center">Multi-Node</td> <td align="center">2±0.01s</td> <td align="center">2.13±0.08s</td> <td align="center"><strong>0.91±0.11s</strong></td> <td align="center"><strong>0.91±0.11s</strong></td></tr> <tr><td align="center">Single Node</td> <td align="center">0.50±0.01s</td> <td align="center">0.50±0.01s</td> <td align="center"><strong>0.41±0.015s</strong></td> <td align="center"><strong>0.41±0.015s</strong></td></tr></tbody>',He,N,wt="As you can see, if you are not careful about how you set up your gradient synchronization, you can get upwards of more than a 2x slowdown during training!",We,D,ut=`If you are worried about making sure everything is done properly, we highly recommend utilizing the <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.accumulate">accumulate()</a> function and passing in
<code>gradient_accumulation_steps</code> or <code>gradient_accumulation_plugin</code> to the <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> object so Accelerate can handle this for you.`,Ve,K,Re,q,ht='Be aware that not syncing gradients can have adverse effects while performing FSDP training. As it has been warned in <code>torch</code>, the <a href="https://pytorch.org/docs/stable/fsdp.html#torch.distributed.fsdp.FullyShardedDataParallel.no_sync" rel="nofollow"><code>no_sync</code> context manager for FSDP</a> will require additional memory.',Xe,O,Tt='Therefore in memory intensive situations while using FSDP, we recommend to set <code>sync_each_batch</code> to <code>True</code> in the <a href="/docs/accelerate/main/en/package_reference/utilities#accelerate.utils.GradientAccumulationPlugin">GradientAccumulationPlugin</a> to disable <code>no_sync</code>.',Pe,ee,Ut="See the example below where we fine-tune Mixtral (47B parameters) on 8 A100-80GB GPUs. We see that even for a modest <code>gradient_accumulation_steps=2</code> we quickly go out-of-memory (OOM) if <code>no_sync</code> is enabled. Again, this is due to additional memory overheads due to FSDP’s <code>no_sync</code>. However, if <code>no_sync</code> is disabled via <code>sync_each_batch=True</code>, then the memory consumption for <code>gradient_accumulation_steps=16</code> reverts to that of <code>gradient_accumulation_steps=1</code>.",Se,te,Jt='<thead><tr><th align="center">Model</th> <th align="center"><code>no_sync</code> (accum=1)</th> <th align="center"><code>no_sync</code> (accum=2)</th> <th align="center"><code>no_sync</code> disabled (accum=16)</th></tr></thead> <tbody><tr><td align="center">mixtral 8x7B</td> <td align="center">69G</td> <td align="center">OOM</td> <td align="center">69G</td></tr></tbody>',Le,h,Ye,le,Fe,ne,Qe;return U=new oe({props:{title:"Gradient synchronization",local:"gradient-synchronization",headingTag:"h1"}}),f=new ie({props:{code:"aW1wb3J0JTIwdG9yY2gubm4lMjBhcyUyMG5uJTBBZnJvbSUyMHRvcmNoLm5uLnBhcmFsbGVsJTIwaW1wb3J0JTIwRGlzdHJpYnV0ZWREYXRhUGFyYWxsZWwlMEElMEFtb2RlbCUyMCUzRCUyMG5uLkxpbmVhcigxMCUyQyUyMDEwKSUwQWRkcF9tb2RlbCUyMCUzRCUyMERpc3RyaWJ1dGVkRGF0YVBhcmFsbGVsKG1vZGVsKQ==",highlighted:`<span class="hljs-keyword">import</span> torch.nn <span class="hljs-keyword">as</span> nn
<span class="hljs-keyword">from</span> torch.nn.parallel <span class="hljs-keyword">import</span> DistributedDataParallel

model = nn.Linear(<span class="hljs-number">10</span>, <span class="hljs-number">10</span>)
ddp_model = DistributedDataParallel(model)`,wrap:!1}}),j=new ie({props:{code:"JTJCJTIwZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUyQiUyMGFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoKSUwQSUyMCUyMGltcG9ydCUyMHRvcmNoLm5uJTIwYXMlMjBubiUwQS0lMjBmcm9tJTIwdG9yY2gubm4ucGFyYWxsZWwlMjBpbXBvcnQlMjBEaXN0cmlidXRlZERhdGFQYXJhbGxlbCUwQSUwQSUyMCUyMG1vZGVsJTIwJTNEJTIwbm4uTGluZWFyKDEwJTJDMTApJTBBJTJCJTIwbW9kZWwlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKG1vZGVsKQ==",highlighted:`<span class="hljs-addition">+ from accelerate import Accelerator</span>
<span class="hljs-addition">+ accelerator = Accelerator()</span>
  import torch.nn as nn
<span class="hljs-deletion">- from torch.nn.parallel import DistributedDataParallel</span>

  model = nn.Linear(10,10)
<span class="hljs-addition">+ model = accelerator.prepare(model)</span>`,wrap:!1}}),g=new oe({props:{title:"The slowdown in gradient accumulation",local:"the-slowdown-in-gradient-accumulation",headingTag:"h2"}}),_=new oe({props:{title:"Solving the slowdown problem",local:"solving-the-slowdown-problem",headingTag:"h2"}}),$=new ie({props:{code:"ZGRwX21vZGVsJTJDJTIwZGF0YWxvYWRlciUyQyUyMG9wdGltaXplciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUobW9kZWwlMkMlMjBkYXRhbG9hZGVyJTJDJTIwb3B0aW1pemVyKSUwQSUwQWZvciUyMGluZGV4JTJDJTIwYmF0Y2glMjBpbiUyMGVudW1lcmF0ZShkYXRhbG9hZGVyKSUzQSUwQSUyMCUyMCUyMCUyMGlucHV0cyUyQyUyMHRhcmdldHMlMjAlM0QlMjBiYXRjaCUwQSUyMCUyMCUyMCUyMCUyMyUyMFRyaWdnZXIlMjBncmFkaWVudCUyMHN5bmNocm9uaXphdGlvbiUyMG9uJTIwdGhlJTIwbGFzdCUyMGJhdGNoJTBBJTIwJTIwJTIwJTIwaWYlMjBpbmRleCUyMCElM0QlMjAobGVuKGRhdGFsb2FkZXIpJTIwLSUyMDEpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2l0aCUyMGRkcF9tb2RlbC5ub19zeW5jKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBHcmFkaWVudHMlMjBvbmx5JTIwYWNjdW11bGF0ZSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBkZHBfbW9kZWwoaW5wdXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBsb3NzX2Z1bmMob3V0cHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhY2NlbGVyYXRvci5iYWNrd2FyZChsb3NzKSUwQSUyMCUyMCUyMCUyMGVsc2UlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjMlMjBHcmFkaWVudHMlMjBmaW5hbGx5JTIwc3luYyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBkZHBfbW9kZWwoaW5wdXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBsb3NzX2Z1bmMob3V0cHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhY2NlbGVyYXRvci5iYWNrd2FyZChsb3NzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci5zdGVwKCk=",highlighted:`ddp_model, dataloader, optimizer = accelerator.prepare(model, dataloader, optimizer)

<span class="hljs-keyword">for</span> index, batch <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(dataloader):
    inputs, targets = batch
    <span class="hljs-comment"># Trigger gradient synchronization on the last batch</span>
    <span class="hljs-keyword">if</span> index != (<span class="hljs-built_in">len</span>(dataloader) - <span class="hljs-number">1</span>):
        <span class="hljs-keyword">with</span> ddp_model.no_sync():
            <span class="hljs-comment"># Gradients only accumulate</span>
            outputs = ddp_model(inputs)
            loss = loss_func(outputs)
            accelerator.backward(loss)
    <span class="hljs-keyword">else</span>:
        <span class="hljs-comment"># Gradients finally sync</span>
        outputs = ddp_model(inputs)
        loss = loss_func(outputs)
        accelerator.backward(loss)
        optimizer.step()`,wrap:!1}}),Z=new ie({props:{code:"JTIwJTIwZGRwX21vZGVsJTJDJTIwZGF0YWxvYWRlciUyQyUyMG9wdGltaXplciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUobW9kZWwlMkMlMjBkYXRhbG9hZGVyJTJDJTIwb3B0aW1pemVyKSUwQSUwQSUyMCUyMGZvciUyMGluZGV4JTJDJTIwYmF0Y2glMjBpbiUyMGVudW1lcmF0ZShkYXRhbG9hZGVyKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyUyQyUyMHRhcmdldHMlMjAlM0QlMjBiYXRjaCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMFRyaWdnZXIlMjBncmFkaWVudCUyMHN5bmNocm9uaXphdGlvbiUyMG9uJTIwdGhlJTIwbGFzdCUyMGJhdGNoJTBBJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBpbmRleCUyMCElM0QlMjAobGVuKGRhdGFsb2FkZXIpLTEpJTNBJTBBLSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHdpdGglMjBkZHBfbW9kZWwubm9fc3luYygpJTNBJTBBJTJCJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2l0aCUyMGFjY2VsZXJhdG9yLm5vX3N5bmMobW9kZWwpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIzJTIwR3JhZGllbnRzJTIwb25seSUyMGFjY3VtdWxhdGUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwZGRwX21vZGVsKGlucHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mdW5jKG91dHB1dHMlMkMlMjB0YXJnZXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMyUyMEdyYWRpZW50cyUyMGZpbmFsbHklMjBzeW5jJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMGRkcF9tb2RlbChpbnB1dHMpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG9zcyUyMCUzRCUyMGxvc3NfZnVuYyhvdXRwdXRzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci56ZXJvX2dyYWQoKQ==",highlighted:`  ddp_model, dataloader, optimizer = accelerator.prepare(model, dataloader, optimizer)

  for index, batch in enumerate(dataloader):
      inputs, targets = batch
      # Trigger gradient synchronization on the last batch
      if index != (len(dataloader)-1):
<span class="hljs-deletion">-         with ddp_model.no_sync():</span>
<span class="hljs-addition">+         with accelerator.no_sync(model):</span>
              # Gradients only accumulate
              outputs = ddp_model(inputs)
              loss = loss_func(outputs, targets)
              accelerator.backward(loss)
      else:
          # Gradients finally sync
          outputs = ddp_model(inputs)
          loss = loss_func(outputs)
          accelerator.backward(loss)
          optimizer.step()
          optimizer.zero_grad()`,wrap:!1}}),H=new ie({props:{code:"ZGRwX21vZGVsJTJDJTIwZGF0YWxvYWRlciUyQyUyMG9wdGltaXplciUyMCUzRCUyMGFjY2VsZXJhdG9yLnByZXBhcmUobW9kZWwlMkMlMjBkYXRhbG9hZGVyJTJDJTIwb3B0aW1pemVyKSUwQSUwQWZvciUyMGJhdGNoJTIwaW4lMjBkYXRhbG9hZGVyJTNBJTBBJTIwJTIwJTIwJTIwd2l0aCUyMGFjY2VsZXJhdG9yLmFjY3VtdWxhdGUobW9kZWwpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyLnplcm9fZ3JhZCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5wdXRzJTJDJTIwdGFyZ2V0cyUyMCUzRCUyMGJhdGNoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsKGlucHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsb3NzJTIwJTNEJTIwbG9zc19mdW5jdGlvbihvdXRwdXRzJTJDJTIwdGFyZ2V0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBhY2NlbGVyYXRvci5iYWNrd2FyZChsb3NzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9wdGltaXplci5zdGVwKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCk=",highlighted:`ddp_model, dataloader, optimizer = accelerator.prepare(model, dataloader, optimizer)

<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> dataloader:
    <span class="hljs-keyword">with</span> accelerator.accumulate(model):
        optimizer.zero_grad()
        inputs, targets = batch
        outputs = model(inputs)
        loss = loss_function(outputs, targets)
        accelerator.backward(loss)
        optimizer.step()
        optimizer.zero_grad()`,wrap:!1}}),V=new oe({props:{title:"Just how much of a slowdown is there, and easy mistakes you can make",local:"just-how-much-of-a-slowdown-is-there-and-easy-mistakes-you-can-make",headingTag:"h2"}}),K=new oe({props:{title:"no_sync requires additional GPU memory when using FSDP",local:"nosync-requires-additional-gpu-memory-when-using-fsdp",headingTag:"h3"}}),h=new Bt({props:{warning:!0,$$slots:{default:[$t]},$$scope:{ctx:se}}}),le=new kt({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/concept_guides/gradient_synchronization.md"}}),{c(){r=o("meta"),T=n(),u=o("p"),ae=n(),d(U.$$.fragment),ce=n(),J=o("p"),J.innerHTML=Ee,re=n(),C=o("p"),C.innerHTML=Ne,de=n(),d(f.$$.fragment),ye=n(),b=o("p"),b.innerHTML=De,Me=n(),d(j.$$.fragment),pe=n(),d(g.$$.fragment),me=n(),G=o("p"),G.innerHTML=Ke,we=n(),I=o("p"),I.textContent=qe,ue=n(),v=o("p"),v.innerHTML=Oe,he=n(),A=o("p"),A.textContent=et,Te=n(),d(_.$$.fragment),Ue=n(),B=o("p"),B.innerHTML=tt,Je=n(),k=o("p"),k.innerHTML=lt,Ce=n(),d($.$$.fragment),fe=n(),x=o("p"),x.innerHTML=at,be=n(),d(Z.$$.fragment),je=n(),z=o("p"),z.innerHTML=nt,ge=n(),d(H.$$.fragment),Ge=n(),W=o("p"),W.innerHTML=it,Ie=n(),d(V.$$.fragment),ve=n(),R=o("p"),R.textContent=ot,Ae=n(),X=o("ul"),X.innerHTML=st,_e=n(),P=o("p"),P.innerHTML=ct,Be=n(),S=o("p"),S.innerHTML=rt,ke=n(),L=o("p"),L.textContent=dt,$e=n(),Y=o("p"),Y.textContent=yt,xe=n(),F=o("ul"),F.innerHTML=Mt,Ze=n(),Q=o("p"),Q.textContent=pt,ze=n(),E=o("table"),E.innerHTML=mt,He=n(),N=o("p"),N.textContent=wt,We=n(),D=o("p"),D.innerHTML=ut,Ve=n(),d(K.$$.fragment),Re=n(),q=o("p"),q.innerHTML=ht,Xe=n(),O=o("p"),O.innerHTML=Tt,Pe=n(),ee=o("p"),ee.innerHTML=Ut,Se=n(),te=o("table"),te.innerHTML=Jt,Le=n(),d(h.$$.fragment),Ye=n(),d(le.$$.fragment),Fe=n(),ne=o("p"),this.h()},l(e){const t=At("svelte-u9bgzb",document.head);r=s(t,"META",{name:!0,content:!0}),t.forEach(l),T=i(e),u=s(e,"P",{}),ft(u).forEach(l),ae=i(e),y(U.$$.fragment,e),ce=i(e),J=s(e,"P",{"data-svelte-h":!0}),c(J)!=="svelte-1bq702f"&&(J.innerHTML=Ee),re=i(e),C=s(e,"P",{"data-svelte-h":!0}),c(C)!=="svelte-fqoolu"&&(C.innerHTML=Ne),de=i(e),y(f.$$.fragment,e),ye=i(e),b=s(e,"P",{"data-svelte-h":!0}),c(b)!=="svelte-2k6ktj"&&(b.innerHTML=De),Me=i(e),y(j.$$.fragment,e),pe=i(e),y(g.$$.fragment,e),me=i(e),G=s(e,"P",{"data-svelte-h":!0}),c(G)!=="svelte-byuygg"&&(G.innerHTML=Ke),we=i(e),I=s(e,"P",{"data-svelte-h":!0}),c(I)!=="svelte-w9p09z"&&(I.textContent=qe),ue=i(e),v=s(e,"P",{"data-svelte-h":!0}),c(v)!=="svelte-1qb001k"&&(v.innerHTML=Oe),he=i(e),A=s(e,"P",{"data-svelte-h":!0}),c(A)!=="svelte-14tkwrb"&&(A.textContent=et),Te=i(e),y(_.$$.fragment,e),Ue=i(e),B=s(e,"P",{"data-svelte-h":!0}),c(B)!=="svelte-olrkxe"&&(B.innerHTML=tt),Je=i(e),k=s(e,"P",{"data-svelte-h":!0}),c(k)!=="svelte-cn8x56"&&(k.innerHTML=lt),Ce=i(e),y($.$$.fragment,e),fe=i(e),x=s(e,"P",{"data-svelte-h":!0}),c(x)!=="svelte-j7c9ip"&&(x.innerHTML=at),be=i(e),y(Z.$$.fragment,e),je=i(e),z=s(e,"P",{"data-svelte-h":!0}),c(z)!=="svelte-12bvrws"&&(z.innerHTML=nt),ge=i(e),y(H.$$.fragment,e),Ge=i(e),W=s(e,"P",{"data-svelte-h":!0}),c(W)!=="svelte-1b3vx79"&&(W.innerHTML=it),Ie=i(e),y(V.$$.fragment,e),ve=i(e),R=s(e,"P",{"data-svelte-h":!0}),c(R)!=="svelte-1hdyapv"&&(R.textContent=ot),Ae=i(e),X=s(e,"UL",{"data-svelte-h":!0}),c(X)!=="svelte-1d218dd"&&(X.innerHTML=st),_e=i(e),P=s(e,"P",{"data-svelte-h":!0}),c(P)!=="svelte-5qwyq6"&&(P.innerHTML=ct),Be=i(e),S=s(e,"P",{"data-svelte-h":!0}),c(S)!=="svelte-1yxh0lm"&&(S.innerHTML=rt),ke=i(e),L=s(e,"P",{"data-svelte-h":!0}),c(L)!=="svelte-1n0bzbn"&&(L.textContent=dt),$e=i(e),Y=s(e,"P",{"data-svelte-h":!0}),c(Y)!=="svelte-1i6grvn"&&(Y.textContent=yt),xe=i(e),F=s(e,"UL",{"data-svelte-h":!0}),c(F)!=="svelte-10i8zn2"&&(F.innerHTML=Mt),Ze=i(e),Q=s(e,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-vdaq6p"&&(Q.textContent=pt),ze=i(e),E=s(e,"TABLE",{"data-svelte-h":!0}),c(E)!=="svelte-sjfg3j"&&(E.innerHTML=mt),He=i(e),N=s(e,"P",{"data-svelte-h":!0}),c(N)!=="svelte-yk2mxh"&&(N.textContent=wt),We=i(e),D=s(e,"P",{"data-svelte-h":!0}),c(D)!=="svelte-8cx020"&&(D.innerHTML=ut),Ve=i(e),y(K.$$.fragment,e),Re=i(e),q=s(e,"P",{"data-svelte-h":!0}),c(q)!=="svelte-rzu0c8"&&(q.innerHTML=ht),Xe=i(e),O=s(e,"P",{"data-svelte-h":!0}),c(O)!=="svelte-qdcsdn"&&(O.innerHTML=Tt),Pe=i(e),ee=s(e,"P",{"data-svelte-h":!0}),c(ee)!=="svelte-1s75xk0"&&(ee.innerHTML=Ut),Se=i(e),te=s(e,"TABLE",{"data-svelte-h":!0}),c(te)!=="svelte-iqaw47"&&(te.innerHTML=Jt),Le=i(e),y(h.$$.fragment,e),Ye=i(e),y(le.$$.fragment,e),Fe=i(e),ne=s(e,"P",{}),ft(ne).forEach(l),this.h()},h(){bt(r,"name","hf:doc:metadata"),bt(r,"content",Zt)},m(e,t){_t(document.head,r),a(e,T,t),a(e,u,t),a(e,ae,t),M(U,e,t),a(e,ce,t),a(e,J,t),a(e,re,t),a(e,C,t),a(e,de,t),M(f,e,t),a(e,ye,t),a(e,b,t),a(e,Me,t),M(j,e,t),a(e,pe,t),M(g,e,t),a(e,me,t),a(e,G,t),a(e,we,t),a(e,I,t),a(e,ue,t),a(e,v,t),a(e,he,t),a(e,A,t),a(e,Te,t),M(_,e,t),a(e,Ue,t),a(e,B,t),a(e,Je,t),a(e,k,t),a(e,Ce,t),M($,e,t),a(e,fe,t),a(e,x,t),a(e,be,t),M(Z,e,t),a(e,je,t),a(e,z,t),a(e,ge,t),M(H,e,t),a(e,Ge,t),a(e,W,t),a(e,Ie,t),M(V,e,t),a(e,ve,t),a(e,R,t),a(e,Ae,t),a(e,X,t),a(e,_e,t),a(e,P,t),a(e,Be,t),a(e,S,t),a(e,ke,t),a(e,L,t),a(e,$e,t),a(e,Y,t),a(e,xe,t),a(e,F,t),a(e,Ze,t),a(e,Q,t),a(e,ze,t),a(e,E,t),a(e,He,t),a(e,N,t),a(e,We,t),a(e,D,t),a(e,Ve,t),M(K,e,t),a(e,Re,t),a(e,q,t),a(e,Xe,t),a(e,O,t),a(e,Pe,t),a(e,ee,t),a(e,Se,t),a(e,te,t),a(e,Le,t),M(h,e,t),a(e,Ye,t),M(le,e,t),a(e,Fe,t),a(e,ne,t),Qe=!0},p(e,[t]){const Ct={};t&2&&(Ct.$$scope={dirty:t,ctx:e}),h.$set(Ct)},i(e){Qe||(p(U.$$.fragment,e),p(f.$$.fragment,e),p(j.$$.fragment,e),p(g.$$.fragment,e),p(_.$$.fragment,e),p($.$$.fragment,e),p(Z.$$.fragment,e),p(H.$$.fragment,e),p(V.$$.fragment,e),p(K.$$.fragment,e),p(h.$$.fragment,e),p(le.$$.fragment,e),Qe=!0)},o(e){m(U.$$.fragment,e),m(f.$$.fragment,e),m(j.$$.fragment,e),m(g.$$.fragment,e),m(_.$$.fragment,e),m($.$$.fragment,e),m(Z.$$.fragment,e),m(H.$$.fragment,e),m(V.$$.fragment,e),m(K.$$.fragment,e),m(h.$$.fragment,e),m(le.$$.fragment,e),Qe=!1},d(e){e&&(l(T),l(u),l(ae),l(ce),l(J),l(re),l(C),l(de),l(ye),l(b),l(Me),l(pe),l(me),l(G),l(we),l(I),l(ue),l(v),l(he),l(A),l(Te),l(Ue),l(B),l(Je),l(k),l(Ce),l(fe),l(x),l(be),l(je),l(z),l(ge),l(Ge),l(W),l(Ie),l(ve),l(R),l(Ae),l(X),l(_e),l(P),l(Be),l(S),l(ke),l(L),l($e),l(Y),l(xe),l(F),l(Ze),l(Q),l(ze),l(E),l(He),l(N),l(We),l(D),l(Ve),l(Re),l(q),l(Xe),l(O),l(Pe),l(ee),l(Se),l(te),l(Le),l(Ye),l(Fe),l(ne)),l(r),w(U,e),w(f,e),w(j,e),w(g,e),w(_,e),w($,e),w(Z,e),w(H,e),w(V,e),w(K,e),w(h,e),w(le,e)}}}const Zt='{"title":"Gradient synchronization","local":"gradient-synchronization","sections":[{"title":"The slowdown in gradient accumulation","local":"the-slowdown-in-gradient-accumulation","sections":[],"depth":2},{"title":"Solving the slowdown problem","local":"solving-the-slowdown-problem","sections":[],"depth":2},{"title":"Just how much of a slowdown is there, and easy mistakes you can make","local":"just-how-much-of-a-slowdown-is-there-and-easy-mistakes-you-can-make","sections":[{"title":"no_sync requires additional GPU memory when using FSDP","local":"nosync-requires-additional-gpu-memory-when-using-fsdp","sections":[],"depth":3}],"depth":2}],"depth":1}';function zt(se){return gt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends It{constructor(r){super(),vt(this,r,zt,xt,jt,{})}}export{Pt as component};
