import{s as Lt,o as Pt,n as mt}from"../chunks/scheduler.defa9a21.js";import{S as Qt,i as St,g as p,s as n,r as f,A as qt,h as u,f as l,c as a,j as Nt,u as y,x as d,k as Rt,y as Ot,a as s,v as M,d as g,t as b,w}from"../chunks/index.fe795e71.js";import{T as Dt}from"../chunks/Tip.179eb360.js";import{C as v}from"../chunks/CodeBlock.204b6c34.js";import{H as J,E as Kt}from"../chunks/getInferenceSnippets.2234a8dd.js";import{H as el,a as ht}from"../chunks/HfOption.3c290b0f.js";function tl(U){let o,T;return o=new v({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tZGVidWclMjAlN0JteV9zY3JpcHQucHklN0QlMjAtLWFyZzElMjAtLWFyZzI=",highlighted:"accelerate launch --debug {my_script.py} --arg1 --arg2",wrap:!1}}),{c(){f(o.$$.fragment)},l(r){y(o.$$.fragment,r)},m(r,m){M(o,r,m),T=!0},p:mt,i(r){T||(g(o.$$.fragment,r),T=!0)},o(r){b(o.$$.fragment,r),T=!1},d(r){w(o,r)}}}function ll(U){let o,T="If enabling debug mode as an environment variable, you don’t need to call <code>accelerate launch</code>.",r,m,h;return m=new v({props:{code:"QUNDRUxFUkFURV9ERUJVR19NT0RFJTNEJTIyMSUyMiUyMHRvcmNocnVuJTIwJTdCbXlfc2NyaXB0LnB5JTdEJTIwLS1hcmcxJTIwLS1hcmcy",highlighted:'ACCELERATE_DEBUG_MODE=<span class="hljs-string">&quot;1&quot;</span> torchrun {my_script.py} --arg1 --arg2',wrap:!1}}),{c(){o=p("p"),o.innerHTML=T,r=n(),f(m.$$.fragment)},l(c){o=u(c,"P",{"data-svelte-h":!0}),d(o)!=="svelte-onfw6c"&&(o.innerHTML=T),r=a(c),y(m.$$.fragment,c)},m(c,i){s(c,o,i),s(c,r,i),M(m,c,i),h=!0},p:mt,i(c){h||(g(m.$$.fragment,c),h=!0)},o(c){b(m.$$.fragment,c),h=!1},d(c){c&&(l(o),l(r)),w(m,c)}}}function sl(U){let o,T="Add <code>debug: true</code> to your <code>config.yaml</code> file.",r,m,h;return m=new v({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWJ1ZyUzQSUyMHRydWU=",highlighted:`<span class="hljs-attr">compute_environment:</span> <span class="hljs-string">LOCAL_MACHINE</span>
<span class="hljs-attr">debug:</span> <span class="hljs-literal">true</span>`,wrap:!1}}),{c(){o=p("p"),o.innerHTML=T,r=n(),f(m.$$.fragment)},l(c){o=u(c,"P",{"data-svelte-h":!0}),d(o)!=="svelte-10693yl"&&(o.innerHTML=T),r=a(c),y(m.$$.fragment,c)},m(c,i){s(c,o,i),s(c,r,i),M(m,c,i),h=!0},p:mt,i(c){h||(g(m.$$.fragment,c),h=!0)},o(c){b(m.$$.fragment,c),h=!1},d(c){c&&(l(o),l(r)),w(m,c)}}}function nl(U){let o,T,r,m,h,c;return o=new ht({props:{id:"mismatch",option:"CLI",$$slots:{default:[tl]},$$scope:{ctx:U}}}),r=new ht({props:{id:"mismatch",option:"environment variable",$$slots:{default:[ll]},$$scope:{ctx:U}}}),h=new ht({props:{id:"mismatch",option:"config.yaml",$$slots:{default:[sl]},$$scope:{ctx:U}}}),{c(){f(o.$$.fragment),T=n(),f(r.$$.fragment),m=n(),f(h.$$.fragment)},l(i){y(o.$$.fragment,i),T=a(i),y(r.$$.fragment,i),m=a(i),y(h.$$.fragment,i)},m(i,$){M(o,i,$),s(i,T,$),M(r,i,$),s(i,m,$),M(h,i,$),c=!0},p(i,$){const C={};$&2&&(C.$$scope={dirty:$,ctx:i}),o.$set(C);const j={};$&2&&(j.$$scope={dirty:$,ctx:i}),r.$set(j);const G={};$&2&&(G.$$scope={dirty:$,ctx:i}),h.$set(G)},i(i){c||(g(o.$$.fragment,i),g(r.$$.fragment,i),g(h.$$.fragment,i),c=!0)},o(i){b(o.$$.fragment,i),b(r.$$.fragment,i),b(h.$$.fragment,i),c=!1},d(i){i&&(l(T),l(m)),w(o,i),w(r,i),w(h,i)}}}function al(U){let o,T='The inner function <strong>must</strong> take batch size as the first parameter, but we do not pass one to it when called. The wrapper will handle this for you. Any object (models, optimizers) that consumes device memory and is passed to the <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> also <strong>must</strong> be declared inside the inner function.';return{c(){o=p("p"),o.innerHTML=T},l(r){o=u(r,"P",{"data-svelte-h":!0}),d(o)!=="svelte-1wj4aqf"&&(o.innerHTML=T)},m(r,m){s(r,o,m)},p:mt,d(r){r&&l(o)}}}function ol(U){let o,T,r,m,h,c,i,$='This guide provides solutions to some issues you might encounter when using Accelerate. Not all errors are covered because Accelerate is an active library that is continuously evolving and there are many different use cases and distributed training setups. If the solutions described here don’t help with your specific error, please take a look at the <a href="#ask-for-help">Ask for help</a> section to learn where and how to get help.',C,j,G,k,ft="Logging can help you identify where an error is coming from. In a distributed setup with multiple processes, logging can be a challenge, but Accelerate provides the <code>logging()</code> utility to ensure logs are synchronized.",Te,Z,yt='To troubleshoot an issue, use <code>logging()</code> instead of the standard Python <a href="https://docs.python.org/3/library/logging.html#module-logging" rel="nofollow"><code>logging</code></a> module. Set the verbosity level (<code>INFO</code>, <code>DEBUG</code>, <code>WARNING</code>, <code>ERROR</code>, <code>CRITICAL</code>) with the <code>log_level</code> parameter, and then you can either:',$e,x,Mt="<li>Export the <code>log_level</code> as the <code>ACCELERATE_LOG_LEVEL</code> environment variable.</li> <li>Pass the <code>log_level</code> directly to <code>get_logger</code>.</li>",Ue,A,gt="For example, to set <code>log_level=&quot;INFO&quot;</code>:",Je,W,je,V,bt=`By default, the log is called on main processes only. To call it on all processes, pass <code>main_process_only=False</code>.
If a log should be called on all processes and in order, also pass <code>in_order=True</code>.`,ve,B,_e,F,Ie,H,wt="There can be many reasons why your code is hanging. Let’s take a look at how to solve some of the most common issues that can cause your code to hang.",Ce,X,Ge,Y,Tt="Mismatched tensor shapes is a common issue that can cause your code to hang for a significant amount of time on a distributed setup.",ke,z,$t='When running scripts in a distributed setup, functions such as <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.gather">Accelerator.gather()</a> and <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.reduce">Accelerator.reduce()</a> are necessary to grab tensors across devices to collectively perform operations on them. These (and other) functions rely on <code>torch.distributed</code> to perform a <code>gather</code> operation, which requires tensors to have the <strong>exact same shape</strong> across all processes. When the tensor shapes don’t match, your code hangs and you’ll eventually hit a timeout exception.',Ze,E,Ut="You can use Accelerate’s operational debug mode to immediately catch this issue. We recommend enabling this mode during the <code>accelerate config</code> setup, but you can also enable it from the CLI, as an environment variable, or by manually editing the <code>config.yaml</code> file.",xe,_,Ae,N,Jt="Once you enable debug mode, you should get a traceback that points to the tensor shape mismatch issue.",We,R,Ve,L,Be,P,jt="For early stopping in distributed training, if each process has a specific stopping condition (e.g. validation loss), it may not be synchronized across all processes. As a result, a break can happen on process 0 but not on process 1 which will cause your code to hang indefinitely until a timeout occurs.",Fe,Q,vt=`If you have early stopping conditionals, use the <code>set_trigger</code> and <code>check_trigger</code> methods to make sure all the processes
are ended correctly.`,He,S,Xe,q,Ye,O,_t="On Linux with kernel version < 5.5, hanging processes have been reported. To avoid this problem, upgrade your system to a later kernel version.",ze,D,Ee,K,It=`If your distributed CPU training job using MPI is hanging, ensure that you have
<a href="https://www.open-mpi.org/faq/?category=rsh#ssh-keys" rel="nofollow">passwordless SSH</a> setup (using keys) between the nodes. This means
that for all nodes in your hostfile, you should to be able to SSH from one node to another without being prompted for a password.`,Ne,ee,Ct=`Next, try to run the <code>mpirun</code> command as a sanity check. For example, the command below should print out the
hostnames for each of the nodes.`,Re,te,Le,le,Pe,se,Gt="One of the most frustrating errors when it comes to running training scripts is hitting “Out-of-Memory” on devices like CUDA, XPU or CPU. The entire script needs to be restarted and any progress is lost.",Qe,ne,kt=`To address this problem, Accelerate provides the <a href="/docs/accelerate/main/en/package_reference/utilities#accelerate.find_executable_batch_size">find_executable_batch_size()</a> utility that is heavily based on <a href="https://github.com/BlackHC/toma" rel="nofollow">toma</a>.
This utility retries code that fails due to OOM (out-of-memory) conditions and automatically lowers batch sizes. For each OOM condition, the algorithm decreases the batch size by half and retries the code until it succeeds.`,Se,ae,Zt='To use <a href="/docs/accelerate/main/en/package_reference/utilities#accelerate.find_executable_batch_size">find_executable_batch_size()</a>, restructure your training function to include an inner function with <code>find_executable_batch_size</code> and build your dataloaders inside it. At a minimum, this only takes 4 new lines of code.',qe,I,Oe,oe,De,ie,Ke,re,xt="If you changed the device setup and observe different model performance, it is likely you didn’t update your script when moving from one setup to another. Even if you’re using the same script with the same batch size, the results will still be different on a TPU, multi-GPU, and single GPU.",et,ce,At="For example, if you were training on a single GPU with a batch size of 16 and you move to a dual GPU setup, you need to change the batch size to 8 to have the same effective batch size. This is because when training with Accelerate, the batch size passed to the dataloader is the <strong>batch size per GPU</strong>.",tt,pe,Wt="To make sure you can reproduce the results between the setups, make sure to use the same seed, adjust the batch size accordingly, and consider scaling the learning rate.",lt,ue,Vt='For more details and a quick reference for batch sizes, check out the <a href="../concept_guides/performance">Comparing performance between different device setups</a> guide.',st,de,nt,me,Bt="If your multi-GPU setup consists of different GPUs, you may encounter some performance issues:",at,he,Ft="<li>There may be an imbalance in GPU memory between the GPUs. In this case, the GPU with the smaller memory will limit the batch size or the size of the model that can be loaded onto the GPUs.</li> <li>If you are using GPUs with different performance profiles, the performance will be driven by the slowest GPU you are using because the other GPUs will have to wait for it to complete its workload.</li>",ot,fe,Ht="Vastly different GPUs within the same setup can lead to performance bottlenecks.",it,ye,rt,Me,Xt="If none of the solutions and advice here helped resolve your issue, you can always reach out to the community and Accelerate team for help.",ct,ge,Yt='<li><p>Ask for help on the Hugging Face forums by posting your question in the <a href="https://discuss.huggingface.co/c/accelerate/18" rel="nofollow">Accelerate category</a>. Make sure to write a descriptive post with relevant context about your setup and reproducible code to maximize the likelihood that your problem is solved!</p></li> <li><p>Post a question on <a href="http://hf.co/join/discord" rel="nofollow">Discord</a>, and let the team and the community help you.</p></li> <li><p>Create an Issue on the Accelerate <a href="https://github.com/huggingface/accelerate/issues" rel="nofollow">GitHub repository</a> if you think you’ve found a bug related to the library. Include context regarding the bug and details about your distributed setup to help us better figure out what’s wrong and how we can fix it.</p></li>',pt,be,ut,we,dt;return h=new J({props:{title:"Troubleshoot",local:"troubleshoot",headingTag:"h1"}}),j=new J({props:{title:"Logging",local:"logging",headingTag:"h2"}}),W=new v({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUubG9nZ2luZyUyMGltcG9ydCUyMGdldF9sb2dnZXIlMEElMEFsb2dnZXIlMjAlM0QlMjBnZXRfbG9nZ2VyKF9fbmFtZV9fJTJDJTIwbG9nX2xldmVsJTNEJTIyREVCVUclMjIp",highlighted:`<span class="hljs-keyword">from</span> accelerate.logging <span class="hljs-keyword">import</span> get_logger

logger = get_logger(__name__, log_level=<span class="hljs-string">&quot;DEBUG&quot;</span>)`,wrap:!1}}),B=new v({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUubG9nZ2luZyUyMGltcG9ydCUyMGdldF9sb2dnZXIlMEElMEFsb2dnZXIlMjAlM0QlMjBnZXRfbG9nZ2VyKF9fbmFtZV9fJTJDJTIwbG9nX2xldmVsJTNEJTIyREVCVUclMjIpJTBBJTIzJTIwbG9nJTIwYWxsJTIwcHJvY2Vzc2VzJTBBbG9nZ2VyLmRlYnVnKCUyMnRoaW5nX3RvX2xvZyUyMiUyQyUyMG1haW5fcHJvY2Vzc19vbmx5JTNERmFsc2UpJTBBJTIzJTIwbG9nJTIwYWxsJTIwcHJvY2Vzc2VzJTIwaW4lMjBvcmRlciUwQWxvZ2dlci5kZWJ1ZyglMjJ0aGluZ190b19sb2clMjIlMkMlMjBtYWluX3Byb2Nlc3Nfb25seSUzREZhbHNlJTJDJTIwaW5fb3JkZXIlM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate.logging <span class="hljs-keyword">import</span> get_logger

logger = get_logger(__name__, log_level=<span class="hljs-string">&quot;DEBUG&quot;</span>)
<span class="hljs-comment"># log all processes</span>
logger.debug(<span class="hljs-string">&quot;thing_to_log&quot;</span>, main_process_only=<span class="hljs-literal">False</span>)
<span class="hljs-comment"># log all processes in order</span>
logger.debug(<span class="hljs-string">&quot;thing_to_log&quot;</span>, main_process_only=<span class="hljs-literal">False</span>, in_order=<span class="hljs-literal">True</span>)`,wrap:!1}}),F=new J({props:{title:"Hanging code and timeout errors",local:"hanging-code-and-timeout-errors",headingTag:"h2"}}),X=new J({props:{title:"Mismatched tensor shapes",local:"mismatched-tensor-shapes",headingTag:"h3"}}),_=new el({props:{id:"mismatch",options:["CLI","environment variable","config.yaml"],$$slots:{default:[nl]},$$scope:{ctx:U}}}),R=new v({props:{code:"VHJhY2ViYWNrJTIwKG1vc3QlMjByZWNlbnQlMjBjYWxsJTIwbGFzdCklM0ElMEElMjAlMjBGaWxlJTIwJTIyJTJGaG9tZSUyRnphY2hfbXVlbGxlcl9odWdnaW5nZmFjZV9jbyUyRnRlc3QucHklMjIlMkMlMjBsaW5lJTIwMTglMkMlMjBpbiUyMCUzQ21vZHVsZSUzRSUwQSUyMCUyMCUyMCUyMG1haW4oKSUwQSUyMCUyMEZpbGUlMjAlMjIlMkZob21lJTJGemFjaF9tdWVsbGVyX2h1Z2dpbmdmYWNlX2NvJTJGdGVzdC5weSUyMiUyQyUyMGxpbmUlMjAxNSUyQyUyMGluJTIwbWFpbiUwQSUyMCUyMCUyMCUyMGJyb2FkY2FzdF90ZW5zb3IlMjAlM0QlMjBicm9hZGNhc3QodGVuc29yKSUwQSUyMCUyMEZpbGUlMjAlMjIlMkZob21lJTJGemFjaF9tdWVsbGVyX2h1Z2dpbmdmYWNlX2NvJTJGYWNjZWxlcmF0ZSUyRnNyYyUyRmFjY2VsZXJhdGUlMkZ1dGlscyUyRm9wZXJhdGlvbnMucHklMjIlMkMlMjBsaW5lJTIwMzAzJTJDJTIwaW4lMjB3cmFwcGVyJTBBYWNjZWxlcmF0ZS51dGlscy5vcGVyYXRpb25zLkRpc3RyaWJ1dGVkT3BlcmF0aW9uRXhjZXB0aW9uJTNBJTBBJTBBQ2Fubm90JTIwYXBwbHklMjBkZXNpcmVkJTIwb3BlcmF0aW9uJTIwZHVlJTIwdG8lMjBzaGFwZSUyMG1pc21hdGNoZXMuJTIwQWxsJTIwc2hhcGVzJTIwYWNyb3NzJTIwZGV2aWNlcyUyMG11c3QlMjBiZSUyMHZhbGlkLiUwQSUwQU9wZXJhdGlvbiUzQSUyMCU2MGFjY2VsZXJhdGUudXRpbHMub3BlcmF0aW9ucy5icm9hZGNhc3QlNjAlMEFJbnB1dCUyMHNoYXBlcyUzQSUwQSUyMCUyMC0lMjBQcm9jZXNzJTIwMCUzQSUyMCU1QjElMkMlMjA1JTVEJTBBJTIwJTIwLSUyMFByb2Nlc3MlMjAxJTNBJTIwJTVCMSUyQyUyMDIlMkMlMjA1JTVE",highlighted:`Traceback (most recent call last):
  File <span class="hljs-string">&quot;/home/zach_mueller_huggingface_co/test.py&quot;</span>, line <span class="hljs-number">18</span>, <span class="hljs-keyword">in</span> &lt;module&gt;
    main()
  File <span class="hljs-string">&quot;/home/zach_mueller_huggingface_co/test.py&quot;</span>, line <span class="hljs-number">15</span>, <span class="hljs-keyword">in</span> main
    broadcast_tensor = broadcast(tensor)
  File <span class="hljs-string">&quot;/home/zach_mueller_huggingface_co/accelerate/src/accelerate/utils/operations.py&quot;</span>, line <span class="hljs-number">303</span>, <span class="hljs-keyword">in</span> wrapper
accelerate.utils.operations.DistributedOperationException:

Cannot apply desired operation due to shape mismatches. All shapes across devices must be valid.

Operation: \`accelerate.utils.operations.broadcast\`
Input shapes:
  - Process <span class="hljs-number">0</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">5</span>]
  - Process <span class="hljs-number">1</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">5</span>]`,wrap:!1}}),L=new J({props:{title:"Early stopping",local:"early-stopping",headingTag:"h3"}}),S=new v({props:{code:"JTIzJTIwQXNzdW1lJTIwJTYwc2hvdWxkX2RvX2JyZWFrcG9pbnQlNjAlMjBpcyUyMGElMjBjdXN0b20lMjBkZWZpbmVkJTIwZnVuY3Rpb24lMjB0aGF0JTIwcmV0dXJucyUyMGElMjBjb25kaXRpb25hbCUyQyUyMCUwQSUyMyUyMGFuZCUyMHRoYXQlMjBjb25kaXRpb25hbCUyMG1pZ2h0JTIwYmUlMjB0cnVlJTIwb25seSUyMG9uJTIwcHJvY2VzcyUyMDElMEFpZiUyMHNob3VsZF9kb19icmVha3BvaW50KGxvc3MpJTNBJTBBJTIwJTIwJTIwJTIwYWNjZWxlcmF0b3Iuc2V0X3RyaWdnZXIoKSUwQSUwQSUyMyUyMExhdGVyJTIwaW4lMjB0aGUlMjB0cmFpbmluZyUyMHNjcmlwdCUyMHdoZW4lMjB3ZSUyMG5lZWQlMjB0byUyMGNoZWNrJTIwZm9yJTIwdGhlJTIwYnJlYWtwb2ludCUwQWlmJTIwYWNjZWxlcmF0b3IuY2hlY2tfdHJpZ2dlcigpJTNBJTBBJTIwJTIwJTIwJTIwYnJlYWs=",highlighted:`<span class="hljs-comment"># Assume \`should_do_breakpoint\` is a custom defined function that returns a conditional, </span>
<span class="hljs-comment"># and that conditional might be true only on process 1</span>
<span class="hljs-keyword">if</span> should_do_breakpoint(loss):
    accelerator.set_trigger()

<span class="hljs-comment"># Later in the training script when we need to check for the breakpoint</span>
<span class="hljs-keyword">if</span> accelerator.check_trigger():
    <span class="hljs-keyword">break</span>`,wrap:!1}}),q=new J({props:{title:"Low kernel versions on Linux",local:"low-kernel-versions-on-linux",headingTag:"h3"}}),D=new J({props:{title:"MPI",local:"mpi",headingTag:"h3"}}),te=new v({props:{code:"bXBpcnVuJTIwLWYlMjBob3N0ZmlsZSUyMC1uJTIwJTdCbnVtYmVyJTIwb2YlMjBub2RlcyU3RCUyMC1wcG4lMjAxJTIwaG9zdG5hbWU=",highlighted:"mpirun -f hostfile -n {number of nodes} -ppn 1 hostname",wrap:!1}}),le=new J({props:{title:"Out-of-Memory",local:"out-of-memory",headingTag:"h2"}}),I=new Dt({props:{warning:!0,$$slots:{default:[al]},$$scope:{ctx:U}}}),oe=new v({props:{code:"ZGVmJTIwdHJhaW5pbmdfZnVuY3Rpb24oYXJncyklM0ElMEElMjAlMjAlMjAlMjBhY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKCklMEElMEElMkIlMjAlMjAlMjAlNDBmaW5kX2V4ZWN1dGFibGVfYmF0Y2hfc2l6ZShzdGFydGluZ19iYXRjaF9zaXplJTNEYXJncy5iYXRjaF9zaXplKSUwQSUyQiUyMCUyMCUyMGRlZiUyMGlubmVyX3RyYWluaW5nX2xvb3AoYmF0Y2hfc2l6ZSklM0ElMEElMkIlMjAlMjAlMjAlMjAlMjAlMjAlMjBub25sb2NhbCUyMGFjY2VsZXJhdG9yJTIwJTIzJTIwRW5zdXJlJTIwdGhleSUyMGNhbiUyMGJlJTIwdXNlZCUyMGluJTIwb3VyJTIwY29udGV4dCUwQSUyQiUyMCUyMCUyMCUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmZyZWVfbWVtb3J5KCklMjAlMjMlMjBGcmVlJTIwYWxsJTIwbGluZ2VyaW5nJTIwcmVmZXJlbmNlcyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsJTIwJTNEJTIwZ2V0X21vZGVsKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbC50byhhY2NlbGVyYXRvci5kZXZpY2UpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwb3B0aW1pemVyJTIwJTNEJTIwZ2V0X29wdGltaXplcigpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHJhaW5fZGF0YWxvYWRlciUyQyUyMGV2YWxfZGF0YWxvYWRlciUyMCUzRCUyMGdldF9kYXRhbG9hZGVycyhhY2NlbGVyYXRvciUyQyUyMGJhdGNoX3NpemUpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbHJfc2NoZWR1bGVyJTIwJTNEJTIwZ2V0X3NjaGVkdWxlciglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcHRpbWl6ZXIlMkMlMjAlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBudW1fdHJhaW5pbmdfc3RlcHMlM0RsZW4odHJhaW5fZGF0YWxvYWRlcikqbnVtX2Vwb2NocyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbCUyQyUyMG9wdGltaXplciUyQyUyMHRyYWluX2RhdGFsb2FkZXIlMkMlMjBldmFsX2RhdGFsb2FkZXIlMkMlMjBscl9zY2hlZHVsZXIlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwdHJhaW5fZGF0YWxvYWRlciUyQyUyMGV2YWxfZGF0YWxvYWRlciUyQyUyMGxyX3NjaGVkdWxlciUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0cmFpbihtb2RlbCUyQyUyMG9wdGltaXplciUyQyUyMHRyYWluX2RhdGFsb2FkZXIlMkMlMjBscl9zY2hlZHVsZXIpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdmFsaWRhdGUobW9kZWwlMkMlMjBldmFsX2RhdGFsb2FkZXIpJTBBJTJCJTIwJTIwJTIwaW5uZXJfdHJhaW5pbmdfbG9vcCgp",highlighted:`def training_function(args):
    accelerator = Accelerator()

<span class="hljs-addition">+   @find_executable_batch_size(starting_batch_size=args.batch_size)</span>
<span class="hljs-addition">+   def inner_training_loop(batch_size):</span>
<span class="hljs-addition">+       nonlocal accelerator # Ensure they can be used in our context</span>
<span class="hljs-addition">+       accelerator.free_memory() # Free all lingering references</span>
        model = get_model()
        model.to(accelerator.device)
        optimizer = get_optimizer()
        train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)
        lr_scheduler = get_scheduler(
            optimizer, 
            num_training_steps=len(train_dataloader)*num_epochs
        )
        model, optimizer, train_dataloader, eval_dataloader, lr_scheduler = accelerator.prepare(
            model, optimizer, train_dataloader, eval_dataloader, lr_scheduler
        )
        train(model, optimizer, train_dataloader, lr_scheduler)
        validate(model, eval_dataloader)
<span class="hljs-addition">+   inner_training_loop()</span>`,wrap:!1}}),ie=new J({props:{title:"Non-reproducible results between device setups",local:"non-reproducible-results-between-device-setups",headingTag:"h2"}}),de=new J({props:{title:"Performance issues on different GPUs",local:"performance-issues-on-different-gpus",headingTag:"h2"}}),ye=new J({props:{title:"Ask for help",local:"ask-for-help",headingTag:"h2"}}),be=new Kt({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/basic_tutorials/troubleshooting.md"}}),{c(){o=p("meta"),T=n(),r=p("p"),m=n(),f(h.$$.fragment),c=n(),i=p("p"),i.innerHTML=$,C=n(),f(j.$$.fragment),G=n(),k=p("p"),k.innerHTML=ft,Te=n(),Z=p("p"),Z.innerHTML=yt,$e=n(),x=p("ol"),x.innerHTML=Mt,Ue=n(),A=p("p"),A.innerHTML=gt,Je=n(),f(W.$$.fragment),je=n(),V=p("p"),V.innerHTML=bt,ve=n(),f(B.$$.fragment),_e=n(),f(F.$$.fragment),Ie=n(),H=p("p"),H.textContent=wt,Ce=n(),f(X.$$.fragment),Ge=n(),Y=p("p"),Y.textContent=Tt,ke=n(),z=p("p"),z.innerHTML=$t,Ze=n(),E=p("p"),E.innerHTML=Ut,xe=n(),f(_.$$.fragment),Ae=n(),N=p("p"),N.textContent=Jt,We=n(),f(R.$$.fragment),Ve=n(),f(L.$$.fragment),Be=n(),P=p("p"),P.textContent=jt,Fe=n(),Q=p("p"),Q.innerHTML=vt,He=n(),f(S.$$.fragment),Xe=n(),f(q.$$.fragment),Ye=n(),O=p("p"),O.textContent=_t,ze=n(),f(D.$$.fragment),Ee=n(),K=p("p"),K.innerHTML=It,Ne=n(),ee=p("p"),ee.innerHTML=Ct,Re=n(),f(te.$$.fragment),Le=n(),f(le.$$.fragment),Pe=n(),se=p("p"),se.textContent=Gt,Qe=n(),ne=p("p"),ne.innerHTML=kt,Se=n(),ae=p("p"),ae.innerHTML=Zt,qe=n(),f(I.$$.fragment),Oe=n(),f(oe.$$.fragment),De=n(),f(ie.$$.fragment),Ke=n(),re=p("p"),re.textContent=xt,et=n(),ce=p("p"),ce.innerHTML=At,tt=n(),pe=p("p"),pe.textContent=Wt,lt=n(),ue=p("p"),ue.innerHTML=Vt,st=n(),f(de.$$.fragment),nt=n(),me=p("p"),me.textContent=Bt,at=n(),he=p("ul"),he.innerHTML=Ft,ot=n(),fe=p("p"),fe.textContent=Ht,it=n(),f(ye.$$.fragment),rt=n(),Me=p("p"),Me.textContent=Xt,ct=n(),ge=p("ul"),ge.innerHTML=Yt,pt=n(),f(be.$$.fragment),ut=n(),we=p("p"),this.h()},l(e){const t=qt("svelte-u9bgzb",document.head);o=u(t,"META",{name:!0,content:!0}),t.forEach(l),T=a(e),r=u(e,"P",{}),Nt(r).forEach(l),m=a(e),y(h.$$.fragment,e),c=a(e),i=u(e,"P",{"data-svelte-h":!0}),d(i)!=="svelte-13d7d11"&&(i.innerHTML=$),C=a(e),y(j.$$.fragment,e),G=a(e),k=u(e,"P",{"data-svelte-h":!0}),d(k)!=="svelte-frnlaf"&&(k.innerHTML=ft),Te=a(e),Z=u(e,"P",{"data-svelte-h":!0}),d(Z)!=="svelte-1dztp11"&&(Z.innerHTML=yt),$e=a(e),x=u(e,"OL",{"data-svelte-h":!0}),d(x)!=="svelte-rs010c"&&(x.innerHTML=Mt),Ue=a(e),A=u(e,"P",{"data-svelte-h":!0}),d(A)!=="svelte-11t72px"&&(A.innerHTML=gt),Je=a(e),y(W.$$.fragment,e),je=a(e),V=u(e,"P",{"data-svelte-h":!0}),d(V)!=="svelte-1ir4wom"&&(V.innerHTML=bt),ve=a(e),y(B.$$.fragment,e),_e=a(e),y(F.$$.fragment,e),Ie=a(e),H=u(e,"P",{"data-svelte-h":!0}),d(H)!=="svelte-1xeblm7"&&(H.textContent=wt),Ce=a(e),y(X.$$.fragment,e),Ge=a(e),Y=u(e,"P",{"data-svelte-h":!0}),d(Y)!=="svelte-hwr0n3"&&(Y.textContent=Tt),ke=a(e),z=u(e,"P",{"data-svelte-h":!0}),d(z)!=="svelte-byepzk"&&(z.innerHTML=$t),Ze=a(e),E=u(e,"P",{"data-svelte-h":!0}),d(E)!=="svelte-3ggiaa"&&(E.innerHTML=Ut),xe=a(e),y(_.$$.fragment,e),Ae=a(e),N=u(e,"P",{"data-svelte-h":!0}),d(N)!=="svelte-73ievd"&&(N.textContent=Jt),We=a(e),y(R.$$.fragment,e),Ve=a(e),y(L.$$.fragment,e),Be=a(e),P=u(e,"P",{"data-svelte-h":!0}),d(P)!=="svelte-zyub5i"&&(P.textContent=jt),Fe=a(e),Q=u(e,"P",{"data-svelte-h":!0}),d(Q)!=="svelte-6xakrz"&&(Q.innerHTML=vt),He=a(e),y(S.$$.fragment,e),Xe=a(e),y(q.$$.fragment,e),Ye=a(e),O=u(e,"P",{"data-svelte-h":!0}),d(O)!=="svelte-1fprc5q"&&(O.textContent=_t),ze=a(e),y(D.$$.fragment,e),Ee=a(e),K=u(e,"P",{"data-svelte-h":!0}),d(K)!=="svelte-1pzvmvb"&&(K.innerHTML=It),Ne=a(e),ee=u(e,"P",{"data-svelte-h":!0}),d(ee)!=="svelte-gpp5fn"&&(ee.innerHTML=Ct),Re=a(e),y(te.$$.fragment,e),Le=a(e),y(le.$$.fragment,e),Pe=a(e),se=u(e,"P",{"data-svelte-h":!0}),d(se)!=="svelte-1nv6p1"&&(se.textContent=Gt),Qe=a(e),ne=u(e,"P",{"data-svelte-h":!0}),d(ne)!=="svelte-1n72pe5"&&(ne.innerHTML=kt),Se=a(e),ae=u(e,"P",{"data-svelte-h":!0}),d(ae)!=="svelte-11k5gop"&&(ae.innerHTML=Zt),qe=a(e),y(I.$$.fragment,e),Oe=a(e),y(oe.$$.fragment,e),De=a(e),y(ie.$$.fragment,e),Ke=a(e),re=u(e,"P",{"data-svelte-h":!0}),d(re)!=="svelte-1voh2ow"&&(re.textContent=xt),et=a(e),ce=u(e,"P",{"data-svelte-h":!0}),d(ce)!=="svelte-1ud0alx"&&(ce.innerHTML=At),tt=a(e),pe=u(e,"P",{"data-svelte-h":!0}),d(pe)!=="svelte-o0c4mt"&&(pe.textContent=Wt),lt=a(e),ue=u(e,"P",{"data-svelte-h":!0}),d(ue)!=="svelte-uhwue4"&&(ue.innerHTML=Vt),st=a(e),y(de.$$.fragment,e),nt=a(e),me=u(e,"P",{"data-svelte-h":!0}),d(me)!=="svelte-yw9ss7"&&(me.textContent=Bt),at=a(e),he=u(e,"UL",{"data-svelte-h":!0}),d(he)!=="svelte-1ci82g2"&&(he.innerHTML=Ft),ot=a(e),fe=u(e,"P",{"data-svelte-h":!0}),d(fe)!=="svelte-bc1071"&&(fe.textContent=Ht),it=a(e),y(ye.$$.fragment,e),rt=a(e),Me=u(e,"P",{"data-svelte-h":!0}),d(Me)!=="svelte-14i3nyp"&&(Me.textContent=Xt),ct=a(e),ge=u(e,"UL",{"data-svelte-h":!0}),d(ge)!=="svelte-1fb2pfu"&&(ge.innerHTML=Yt),pt=a(e),y(be.$$.fragment,e),ut=a(e),we=u(e,"P",{}),Nt(we).forEach(l),this.h()},h(){Rt(o,"name","hf:doc:metadata"),Rt(o,"content",il)},m(e,t){Ot(document.head,o),s(e,T,t),s(e,r,t),s(e,m,t),M(h,e,t),s(e,c,t),s(e,i,t),s(e,C,t),M(j,e,t),s(e,G,t),s(e,k,t),s(e,Te,t),s(e,Z,t),s(e,$e,t),s(e,x,t),s(e,Ue,t),s(e,A,t),s(e,Je,t),M(W,e,t),s(e,je,t),s(e,V,t),s(e,ve,t),M(B,e,t),s(e,_e,t),M(F,e,t),s(e,Ie,t),s(e,H,t),s(e,Ce,t),M(X,e,t),s(e,Ge,t),s(e,Y,t),s(e,ke,t),s(e,z,t),s(e,Ze,t),s(e,E,t),s(e,xe,t),M(_,e,t),s(e,Ae,t),s(e,N,t),s(e,We,t),M(R,e,t),s(e,Ve,t),M(L,e,t),s(e,Be,t),s(e,P,t),s(e,Fe,t),s(e,Q,t),s(e,He,t),M(S,e,t),s(e,Xe,t),M(q,e,t),s(e,Ye,t),s(e,O,t),s(e,ze,t),M(D,e,t),s(e,Ee,t),s(e,K,t),s(e,Ne,t),s(e,ee,t),s(e,Re,t),M(te,e,t),s(e,Le,t),M(le,e,t),s(e,Pe,t),s(e,se,t),s(e,Qe,t),s(e,ne,t),s(e,Se,t),s(e,ae,t),s(e,qe,t),M(I,e,t),s(e,Oe,t),M(oe,e,t),s(e,De,t),M(ie,e,t),s(e,Ke,t),s(e,re,t),s(e,et,t),s(e,ce,t),s(e,tt,t),s(e,pe,t),s(e,lt,t),s(e,ue,t),s(e,st,t),M(de,e,t),s(e,nt,t),s(e,me,t),s(e,at,t),s(e,he,t),s(e,ot,t),s(e,fe,t),s(e,it,t),M(ye,e,t),s(e,rt,t),s(e,Me,t),s(e,ct,t),s(e,ge,t),s(e,pt,t),M(be,e,t),s(e,ut,t),s(e,we,t),dt=!0},p(e,[t]){const zt={};t&2&&(zt.$$scope={dirty:t,ctx:e}),_.$set(zt);const Et={};t&2&&(Et.$$scope={dirty:t,ctx:e}),I.$set(Et)},i(e){dt||(g(h.$$.fragment,e),g(j.$$.fragment,e),g(W.$$.fragment,e),g(B.$$.fragment,e),g(F.$$.fragment,e),g(X.$$.fragment,e),g(_.$$.fragment,e),g(R.$$.fragment,e),g(L.$$.fragment,e),g(S.$$.fragment,e),g(q.$$.fragment,e),g(D.$$.fragment,e),g(te.$$.fragment,e),g(le.$$.fragment,e),g(I.$$.fragment,e),g(oe.$$.fragment,e),g(ie.$$.fragment,e),g(de.$$.fragment,e),g(ye.$$.fragment,e),g(be.$$.fragment,e),dt=!0)},o(e){b(h.$$.fragment,e),b(j.$$.fragment,e),b(W.$$.fragment,e),b(B.$$.fragment,e),b(F.$$.fragment,e),b(X.$$.fragment,e),b(_.$$.fragment,e),b(R.$$.fragment,e),b(L.$$.fragment,e),b(S.$$.fragment,e),b(q.$$.fragment,e),b(D.$$.fragment,e),b(te.$$.fragment,e),b(le.$$.fragment,e),b(I.$$.fragment,e),b(oe.$$.fragment,e),b(ie.$$.fragment,e),b(de.$$.fragment,e),b(ye.$$.fragment,e),b(be.$$.fragment,e),dt=!1},d(e){e&&(l(T),l(r),l(m),l(c),l(i),l(C),l(G),l(k),l(Te),l(Z),l($e),l(x),l(Ue),l(A),l(Je),l(je),l(V),l(ve),l(_e),l(Ie),l(H),l(Ce),l(Ge),l(Y),l(ke),l(z),l(Ze),l(E),l(xe),l(Ae),l(N),l(We),l(Ve),l(Be),l(P),l(Fe),l(Q),l(He),l(Xe),l(Ye),l(O),l(ze),l(Ee),l(K),l(Ne),l(ee),l(Re),l(Le),l(Pe),l(se),l(Qe),l(ne),l(Se),l(ae),l(qe),l(Oe),l(De),l(Ke),l(re),l(et),l(ce),l(tt),l(pe),l(lt),l(ue),l(st),l(nt),l(me),l(at),l(he),l(ot),l(fe),l(it),l(rt),l(Me),l(ct),l(ge),l(pt),l(ut),l(we)),l(o),w(h,e),w(j,e),w(W,e),w(B,e),w(F,e),w(X,e),w(_,e),w(R,e),w(L,e),w(S,e),w(q,e),w(D,e),w(te,e),w(le,e),w(I,e),w(oe,e),w(ie,e),w(de,e),w(ye,e),w(be,e)}}}const il='{"title":"Troubleshoot","local":"troubleshoot","sections":[{"title":"Logging","local":"logging","sections":[],"depth":2},{"title":"Hanging code and timeout errors","local":"hanging-code-and-timeout-errors","sections":[{"title":"Mismatched tensor shapes","local":"mismatched-tensor-shapes","sections":[],"depth":3},{"title":"Early stopping","local":"early-stopping","sections":[],"depth":3},{"title":"Low kernel versions on Linux","local":"low-kernel-versions-on-linux","sections":[],"depth":3},{"title":"MPI","local":"mpi","sections":[],"depth":3}],"depth":2},{"title":"Out-of-Memory","local":"out-of-memory","sections":[],"depth":2},{"title":"Non-reproducible results between device setups","local":"non-reproducible-results-between-device-setups","sections":[],"depth":2},{"title":"Performance issues on different GPUs","local":"performance-issues-on-different-gpus","sections":[],"depth":2},{"title":"Ask for help","local":"ask-for-help","sections":[],"depth":2}],"depth":1}';function rl(U){return Pt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class fl extends Qt{constructor(o){super(),St(this,o,rl,ol,Lt,{})}}export{fl as component};
