import{S as Pr,i as Kr,s as qr,e as r,k as i,w as d,t as a,M as so,c as o,d as e,m as c,a as u,x as m,h as n,b as T,G as l,g as p,y as f,q as M,o as b,B as j,v as eo}from"../chunks/vendor-hf-doc-builder.js";import{T as Vt}from"../chunks/Tip-hf-doc-builder.js";import{I as ua}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as A}from"../chunks/CodeBlock-hf-doc-builder.js";function lo(S){let h,C;return{c(){h=r("p"),C=a("This feature is currently available for PyTorch-only.")},l(w){h=o(w,"P",{});var y=u(h);C=n(y,"This feature is currently available for PyTorch-only."),y.forEach(e)},m(w,y){p(w,h,y),l(h,C)},d(w){w&&e(h)}}}function ao(S){let h,C,w,y,E;return{c(){h=r("p"),C=a("For multi-GPU training it requires DDP ("),w=r("code"),y=a("torch.distributed.launch"),E=a(").")},l(I){h=o(I,"P",{});var U=u(h);C=n(U,"For multi-GPU training it requires DDP ("),w=o(U,"CODE",{});var $=u(w);y=n($,"torch.distributed.launch"),$.forEach(e),E=n(U,")."),U.forEach(e)},m(I,U){p(I,h,U),l(h,C),l(h,w),l(w,y),l(h,E)},d(I){I&&e(h)}}}function no(S){let h,C,w,y,E;return{c(){h=r("p"),C=a("This feature can be used with any "),w=r("code"),y=a("nn.Module"),E=a("-based model.")},l(I){h=o(I,"P",{});var U=u(h);C=n(U,"This feature can be used with any "),w=o(U,"CODE",{});var $=u(w);y=n($,"nn.Module"),$.forEach(e),E=n(U,"-based model."),U.forEach(e)},m(I,U){p(I,h,U),l(h,C),l(h,w),l(w,y),l(h,E)},d(I){I&&e(h)}}}function to(S){let h,C,w,y,E,I,U,$,ia,Fe,W,Y,Ks,cs,ca,qs,ha,ze,x,wa,se,da,ma,He,hs,Oe,Gs,fa,Pe,ws,Ke,_s,Ma,qe,ks,ba,sl,Vs,ja,el,F,ya,ee,Ta,Ja,ll,ds,al,Bs,Ia,nl,L,z,le,ms,Ua,ae,Ca,tl,H,rl,O,ol,P,pl,_,va,ne,Za,Aa,te,Ea,Ga,re,_a,ka,ul,K,Va,gs,Ba,ga,il,fs,cl,D,Ra,oe,$a,Da,Rs,Na,Wa,hl,$s,La,wl,Ms,dl,G,Ds,Xa,Qa,pe,Sa,Ya,ue,xa,Fa,ie,za,Ha,ml,bs,fl,Ns,Oa,Ml,J,Pa,ce,Ka,qa,he,sn,en,we,ln,an,de,nn,tn,me,rn,on,fe,pn,un,Me,cn,hn,bl,q,wn,be,dn,mn,jl,Ws,fn,yl,js,Tl,k,Mn,je,bn,jn,ye,yn,Tn,Te,Jn,In,Jl,Ls,Un,Il,ys,Ul,v,Cn,Je,vn,Zn,Ie,An,En,Ue,Gn,_n,Ce,kn,Vn,ve,Bn,gn,Cl,V,Rn,Ze,$n,Dn,Ae,Nn,Wn,Ee,Ln,Xn,vl,Xs,Qn,Zl,ss,Sn,Ge,Yn,xn,Al,Ts,El,es,Fn,_e,zn,Hn,Gl,ls,On,ke,Pn,Kn,_l,B,qn,Ve,st,et,Be,lt,at,ge,nt,tt,kl,Js,Vl,N,rt,Re,ot,pt,$e,ut,it,Bl,Is,gl,g,ct,De,ht,wt,Ne,dt,mt,We,ft,Mt,Rl,as,bt,Le,jt,yt,$l,Qs,Tt,Dl,Us,Nl,X,ns,Xe,Cs,Jt,Qe,It,Wl,Ss,Ut,Ll,ts,Ct,Se,vt,Zt,Xl,vs,Ql,Ys,At,Sl,xs,Et,Yl,Fs,Gt,xl,Zs,Fl,zs,_t,zl,Hs,kt,Hl,As,Ol;return I=new ua({}),cs=new ua({}),hs=new A({props:{code:"d2dldCUyMGh0dHBzJTNBJTJGJTJGcmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSUyRmh1Z2dpbmdmYWNlJTJGdHJhbnNmb3JtZXJzJTJGbWFpbiUyRnNjcmlwdHMlMkZkaXN0cmlidXRlZCUyRnRvcmNoLWRpc3RyaWJ1dGVkLWdwdS10ZXN0LnB5",highlighted:"wget https://raw.githubusercontent.com/huggingface/transformers/main/scripts/distributed/torch-distributed-gpu-test.py"}}),ws=new A({props:{code:"cHl0aG9uJTIwLW0lMjB0b3JjaC5kaXN0cmlidXRlZC5ydW4lMjAtLW5wcm9jX3Blcl9ub2RlJTIwMiUyMC0tbm5vZGVzJTIwMSUyMHRvcmNoLWRpc3RyaWJ1dGVkLWdwdS10ZXN0LnB5",highlighted:"python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py"}}),ds=new A({props:{code:"TkNDTF9ERUJVRyUzRElORk8lMjBweXRob24lMjAtbSUyMHRvcmNoLmRpc3RyaWJ1dGVkLnJ1biUyMC0tbnByb2NfcGVyX25vZGUlMjAyJTIwLS1ubm9kZXMlMjAxJTIwdG9yY2gtZGlzdHJpYnV0ZWQtZ3B1LXRlc3QucHk=",highlighted:"NCCL_DEBUG=INFO python -m torch.distributed.run --nproc_per_node 2 --nnodes 1 torch-distributed-gpu-test.py"}}),ms=new ua({}),H=new Vt({props:{$$slots:{default:[lo]},$$scope:{ctx:S}}}),O=new Vt({props:{$$slots:{default:[ao]},$$scope:{ctx:S}}}),P=new Vt({props:{$$slots:{default:[no]},$$scope:{ctx:S}}}),fs=new A({props:{code:"LS1kZWJ1ZyUyMHVuZGVyZmxvd19vdmVyZmxvdw==",highlighted:"--debug underflow_overflow"}}),Ms=new A({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5kZWJ1Z191dGlscyUyMGltcG9ydCUyMERlYnVnVW5kZXJmbG93T3ZlcmZsb3clMEElMEFkZWJ1Z19vdmVyZmxvdyUyMCUzRCUyMERlYnVnVW5kZXJmbG93T3ZlcmZsb3cobW9kZWwp",highlighted:`<span class="hljs-keyword">from</span> transformers.debug_utils <span class="hljs-keyword">import</span> DebugUnderflowOverflow

debug_overflow = DebugUnderflowOverflow(model)`}}),bs=new A({props:{code:"RGV0ZWN0ZWQlMjBpbmYlMkZuYW4lMjBkdXJpbmclMjBiYXRjaF9udW1iZXIlM0QwJTBBTGFzdCUyMDIxJTIwZm9yd2FyZCUyMGZyYW1lcyUzQSUwQWFicyUyMG1pbiUyMCUyMGFicyUyMG1heCUyMCUyMG1ldGFkYXRhJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4xLmxheWVyLjEuRGVuc2VSZWx1RGVuc2UuZHJvcG91dCUyMERyb3BvdXQlMEEwLjAwZSUyQjAwJTIwMi41N2UlMkIwMiUyMGlucHV0JTVCMCU1RCUwQTAuMDBlJTJCMDAlMjAyLjg1ZSUyQjAyJTIwb3V0cHV0JTBBJTVCLi4uJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjAlMjBUNUxheWVyU2VsZkF0dGVudGlvbiUwQTYuNzhlLTA0JTIwMy4xNWUlMkIwMyUyMGlucHV0JTVCMCU1RCUwQTIuNjVlLTA0JTIwMy40MmUlMkIwMyUyMG91dHB1dCU1QjAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBOb25lJTIwb3V0cHV0JTVCMSU1RCUwQTIuMjVlLTAxJTIwMS4wMGUlMkIwNCUyMG91dHB1dCU1QjIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5sYXllcl9ub3JtJTIwVDVMYXllck5vcm0lMEE4LjY5ZS0wMiUyMDQuMThlLTAxJTIwd2VpZ2h0JTBBMi42NWUtMDQlMjAzLjQyZSUyQjAzJTIwaW5wdXQlNUIwJTVEJTBBMS43OWUtMDYlMjA0LjY1ZSUyQjAwJTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEuRGVuc2VSZWx1RGVuc2Uud2lfMCUyMExpbmVhciUwQTIuMTdlLTA3JTIwNC41MGUlMkIwMCUyMHdlaWdodCUwQTEuNzllLTA2JTIwNC42NWUlMkIwMCUyMGlucHV0JTVCMCU1RCUwQTIuNjhlLTA2JTIwMy43MGUlMkIwMSUyMG91dHB1dCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVuY29kZXIuYmxvY2suMi5sYXllci4xLkRlbnNlUmVsdURlbnNlLndpXzElMjBMaW5lYXIlMEE4LjA4ZS0wNyUyMDIuNjZlJTJCMDElMjB3ZWlnaHQlMEExLjc5ZS0wNiUyMDQuNjVlJTJCMDAlMjBpbnB1dCU1QjAlNUQlMEExLjI3ZS0wNCUyMDIuMzdlJTJCMDIlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5EZW5zZVJlbHVEZW5zZS5kcm9wb3V0JTIwRHJvcG91dCUwQTAuMDBlJTJCMDAlMjA4Ljc2ZSUyQjAzJTIwaW5wdXQlNUIwJTVEJTBBMC4wMGUlMkIwMCUyMDkuNzRlJTJCMDMlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5EZW5zZVJlbHVEZW5zZS53byUyMExpbmVhciUwQTEuMDFlLTA2JTIwNi40NGUlMkIwMCUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjA5Ljc0ZSUyQjAzJTIwaW5wdXQlNUIwJTVEJTBBMy4xOGUtMDQlMjA2LjI3ZSUyQjA0JTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEuRGVuc2VSZWx1RGVuc2UlMjBUNURlbnNlR2F0ZWRHZWx1RGVuc2UlMEExLjc5ZS0wNiUyMDQuNjVlJTJCMDAlMjBpbnB1dCU1QjAlNUQlMEEzLjE4ZS0wNCUyMDYuMjdlJTJCMDQlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5kcm9wb3V0JTIwRHJvcG91dCUwQTMuMThlLTA0JTIwNi4yN2UlMkIwNCUyMGlucHV0JTVCMCU1RCUwQTAuMDBlJTJCMDAlMjAlMjAlMjAlMjAlMjAlMjBpbmYlMjBvdXRwdXQ=",highlighted:`<span class="hljs-attribute">Detected</span> inf/nan during batch_number=<span class="hljs-number">0</span>
<span class="hljs-attribute">Last</span> <span class="hljs-number">21</span> forward frames:
<span class="hljs-attribute">abs</span> min  abs max  metadata
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">1</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.dropout Dropout
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">2</span>.<span class="hljs-number">57</span>e+<span class="hljs-number">02</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">2</span>.<span class="hljs-number">85</span>e+<span class="hljs-number">02</span> output<span class="hljs-meta">
[...]</span>
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">0</span> T5LayerSelfAttention
<span class="hljs-attribute">6</span>.<span class="hljs-number">78</span>e-<span class="hljs-number">04</span> <span class="hljs-number">3</span>.<span class="hljs-number">15</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">2</span>.<span class="hljs-number">65</span>e-<span class="hljs-number">04</span> <span class="hljs-number">3</span>.<span class="hljs-number">42</span>e+<span class="hljs-number">03</span> output[<span class="hljs-number">0</span>]
             <span class="hljs-attribute">None</span> output[<span class="hljs-number">1</span>]
<span class="hljs-attribute">2</span>.<span class="hljs-number">25</span>e-<span class="hljs-number">01</span> <span class="hljs-number">1</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">04</span> output[<span class="hljs-number">2</span>]
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.layer_norm T5LayerNorm
<span class="hljs-attribute">8</span>.<span class="hljs-number">69</span>e-<span class="hljs-number">02</span> <span class="hljs-number">4</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">01</span> weight
<span class="hljs-attribute">2</span>.<span class="hljs-number">65</span>e-<span class="hljs-number">04</span> <span class="hljs-number">3</span>.<span class="hljs-number">42</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_0 Linear
<span class="hljs-attribute">2</span>.<span class="hljs-number">17</span>e-<span class="hljs-number">07</span> <span class="hljs-number">4</span>.<span class="hljs-number">50</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">2</span>.<span class="hljs-number">68</span>e-<span class="hljs-number">06</span> <span class="hljs-number">3</span>.<span class="hljs-number">70</span>e+<span class="hljs-number">01</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_1 Linear
<span class="hljs-attribute">8</span>.<span class="hljs-number">08</span>e-<span class="hljs-number">07</span> <span class="hljs-number">2</span>.<span class="hljs-number">66</span>e+<span class="hljs-number">01</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">1</span>.<span class="hljs-number">27</span>e-<span class="hljs-number">04</span> <span class="hljs-number">2</span>.<span class="hljs-number">37</span>e+<span class="hljs-number">02</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.dropout Dropout
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">8</span>.<span class="hljs-number">76</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">9</span>.<span class="hljs-number">74</span>e+<span class="hljs-number">03</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wo Linear
<span class="hljs-attribute">1</span>.<span class="hljs-number">01</span>e-<span class="hljs-number">06</span> <span class="hljs-number">6</span>.<span class="hljs-number">44</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">9</span>.<span class="hljs-number">74</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense T5DenseGatedGeluDense
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.dropout Dropout
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span>      inf output`}}),js=new A({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEubGF5ZXJfbm9ybSUyMFQ1TGF5ZXJOb3JtJTBBOC42OWUtMDIlMjA0LjE4ZS0wMSUyMHdlaWdodCUwQTIuNjVlLTA0JTIwMy40MmUlMkIwMyUyMGlucHV0JTVCMCU1RCUwQTEuNzllLTA2JTIwNC42NWUlMkIwMCUyMG91dHB1dA==",highlighted:`                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.layer_norm T5LayerNorm
<span class="hljs-attribute">8</span>.<span class="hljs-number">69</span>e-<span class="hljs-number">02</span> <span class="hljs-number">4</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">01</span> weight
<span class="hljs-attribute">2</span>.<span class="hljs-number">65</span>e-<span class="hljs-number">04</span> <span class="hljs-number">3</span>.<span class="hljs-number">42</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> output`}}),ys=new A({props:{code:"RGV0ZWN0ZWQlMjBpbmYlMkZuYW4lMjBkdXJpbmclMjBiYXRjaF9udW1iZXIlM0QwJTBBTGFzdCUyMDIxJTIwZm9yd2FyZCUyMGZyYW1lcyUzQSUwQWFicyUyMG1pbiUyMCUyMGFicyUyMG1heCUyMCUyMG1ldGFkYXRhJTBBJTVCLi4uJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEuRGVuc2VSZWx1RGVuc2Uud2lfMCUyMExpbmVhciUwQTIuMTdlLTA3JTIwNC41MGUlMkIwMCUyMHdlaWdodCUwQTEuNzllLTA2JTIwNC42NWUlMkIwMCUyMGlucHV0JTVCMCU1RCUwQTIuNjhlLTA2JTIwMy43MGUlMkIwMSUyMG91dHB1dCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGVuY29kZXIuYmxvY2suMi5sYXllci4xLkRlbnNlUmVsdURlbnNlLndpXzElMjBMaW5lYXIlMEE4LjA4ZS0wNyUyMDIuNjZlJTJCMDElMjB3ZWlnaHQlMEExLjc5ZS0wNiUyMDQuNjVlJTJCMDAlMjBpbnB1dCU1QjAlNUQlMEExLjI3ZS0wNCUyMDIuMzdlJTJCMDIlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5EZW5zZVJlbHVEZW5zZS53byUyMExpbmVhciUwQTEuMDFlLTA2JTIwNi40NGUlMkIwMCUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjA5Ljc0ZSUyQjAzJTIwaW5wdXQlNUIwJTVEJTBBMy4xOGUtMDQlMjA2LjI3ZSUyQjA0JTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW5jb2Rlci5ibG9jay4yLmxheWVyLjEuRGVuc2VSZWx1RGVuc2UlMjBUNURlbnNlR2F0ZWRHZWx1RGVuc2UlMEExLjc5ZS0wNiUyMDQuNjVlJTJCMDAlMjBpbnB1dCU1QjAlNUQlMEEzLjE4ZS0wNCUyMDYuMjdlJTJCMDQlMjBvdXRwdXQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBlbmNvZGVyLmJsb2NrLjIubGF5ZXIuMS5kcm9wb3V0JTIwRHJvcG91dCUwQTMuMThlLTA0JTIwNi4yN2UlMkIwNCUyMGlucHV0JTVCMCU1RCUwQTAuMDBlJTJCMDAlMjAlMjAlMjAlMjAlMjAlMjBpbmYlMjBvdXRwdXQ=",highlighted:`<span class="hljs-attribute">Detected</span> inf/nan during batch_number=<span class="hljs-number">0</span>
<span class="hljs-attribute">Last</span> <span class="hljs-number">21</span> forward frames:
<span class="hljs-attribute">abs</span> min  abs max  metadata<span class="hljs-meta">
[...]</span>
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_0 Linear
<span class="hljs-attribute">2</span>.<span class="hljs-number">17</span>e-<span class="hljs-number">07</span> <span class="hljs-number">4</span>.<span class="hljs-number">50</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">2</span>.<span class="hljs-number">68</span>e-<span class="hljs-number">06</span> <span class="hljs-number">3</span>.<span class="hljs-number">70</span>e+<span class="hljs-number">01</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wi_1 Linear
<span class="hljs-attribute">8</span>.<span class="hljs-number">08</span>e-<span class="hljs-number">07</span> <span class="hljs-number">2</span>.<span class="hljs-number">66</span>e+<span class="hljs-number">01</span> weight
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">1</span>.<span class="hljs-number">27</span>e-<span class="hljs-number">04</span> <span class="hljs-number">2</span>.<span class="hljs-number">37</span>e+<span class="hljs-number">02</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense.wo Linear
<span class="hljs-attribute">1</span>.<span class="hljs-number">01</span>e-<span class="hljs-number">06</span> <span class="hljs-number">6</span>.<span class="hljs-number">44</span>e+<span class="hljs-number">00</span> weight
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span> <span class="hljs-number">9</span>.<span class="hljs-number">74</span>e+<span class="hljs-number">03</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.DenseReluDense T5DenseGatedGeluDense
<span class="hljs-attribute">1</span>.<span class="hljs-number">79</span>e-<span class="hljs-number">06</span> <span class="hljs-number">4</span>.<span class="hljs-number">65</span>e+<span class="hljs-number">00</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> output
                  <span class="hljs-attribute">encoder</span>.block.<span class="hljs-number">2</span>.layer.<span class="hljs-number">1</span>.dropout Dropout
<span class="hljs-attribute">3</span>.<span class="hljs-number">18</span>e-<span class="hljs-number">04</span> <span class="hljs-number">6</span>.<span class="hljs-number">27</span>e+<span class="hljs-number">04</span> input[<span class="hljs-number">0</span>]
<span class="hljs-attribute">0</span>.<span class="hljs-number">00</span>e+<span class="hljs-number">00</span>      inf output`}}),Ts=new A({props:{code:"Y2xhc3MlMjBUNURlbnNlR2F0ZWRHZWx1RGVuc2Uobm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMGRlZiUyMF9faW5pdF9fKHNlbGYlMkMlMjBjb25maWcpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3VwZXIoKS5fX2luaXRfXygpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi53aV8wJTIwJTNEJTIwbm4uTGluZWFyKGNvbmZpZy5kX21vZGVsJTJDJTIwY29uZmlnLmRfZmYlMkMlMjBiaWFzJTNERmFsc2UpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi53aV8xJTIwJTNEJTIwbm4uTGluZWFyKGNvbmZpZy5kX21vZGVsJTJDJTIwY29uZmlnLmRfZmYlMkMlMjBiaWFzJTNERmFsc2UpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2VsZi53byUyMCUzRCUyMG5uLkxpbmVhcihjb25maWcuZF9mZiUyQyUyMGNvbmZpZy5kX21vZGVsJTJDJTIwYmlhcyUzREZhbHNlKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuZHJvcG91dCUyMCUzRCUyMG5uLkRyb3BvdXQoY29uZmlnLmRyb3BvdXRfcmF0ZSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmdlbHVfYWN0JTIwJTNEJTIwQUNUMkZOJTVCJTIyZ2VsdV9uZXclMjIlNUQlMEElMEElMjAlMjAlMjAlMjBkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBoaWRkZW5fc3RhdGVzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGhpZGRlbl9nZWx1JTIwJTNEJTIwc2VsZi5nZWx1X2FjdChzZWxmLndpXzAoaGlkZGVuX3N0YXRlcykpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaGlkZGVuX2xpbmVhciUyMCUzRCUyMHNlbGYud2lfMShoaWRkZW5fc3RhdGVzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGhpZGRlbl9zdGF0ZXMlMjAlM0QlMjBoaWRkZW5fZ2VsdSUyMColMjBoaWRkZW5fbGluZWFyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaGlkZGVuX3N0YXRlcyUyMCUzRCUyMHNlbGYuZHJvcG91dChoaWRkZW5fc3RhdGVzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGhpZGRlbl9zdGF0ZXMlMjAlM0QlMjBzZWxmLndvKGhpZGRlbl9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwaGlkZGVuX3N0YXRlcw==",highlighted:`<span class="hljs-keyword">class</span> <span class="hljs-title class_">T5DenseGatedGeluDense</span>(nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self, config</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.wi_0 = nn.Linear(config.d_model, config.d_ff, bias=<span class="hljs-literal">False</span>)
        self.wi_1 = nn.Linear(config.d_model, config.d_ff, bias=<span class="hljs-literal">False</span>)
        self.wo = nn.Linear(config.d_ff, config.d_model, bias=<span class="hljs-literal">False</span>)
        self.dropout = nn.Dropout(config.dropout_rate)
        self.gelu_act = ACT2FN[<span class="hljs-string">&quot;gelu_new&quot;</span>]

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states</span>):
        hidden_gelu = self.gelu_act(self.wi_0(hidden_states))
        hidden_linear = self.wi_1(hidden_states)
        hidden_states = hidden_gelu * hidden_linear
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.wo(hidden_states)
        <span class="hljs-keyword">return</span> hidden_states`}}),Js=new A({props:{code:"ZGVmJTIwX2ZvcndhcmQoc2VsZiUyQyUyMGhpZGRlbl9zdGF0ZXMpJTNBJTBBJTIwJTIwJTIwJTIwaGlkZGVuX2dlbHUlMjAlM0QlMjBzZWxmLmdlbHVfYWN0KHNlbGYud2lfMChoaWRkZW5fc3RhdGVzKSklMEElMjAlMjAlMjAlMjBoaWRkZW5fbGluZWFyJTIwJTNEJTIwc2VsZi53aV8xKGhpZGRlbl9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwaGlkZGVuX3N0YXRlcyUyMCUzRCUyMGhpZGRlbl9nZWx1JTIwKiUyMGhpZGRlbl9saW5lYXIlMEElMjAlMjAlMjAlMjBoaWRkZW5fc3RhdGVzJTIwJTNEJTIwc2VsZi5kcm9wb3V0KGhpZGRlbl9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwaGlkZGVuX3N0YXRlcyUyMCUzRCUyMHNlbGYud28oaGlkZGVuX3N0YXRlcyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBoaWRkZW5fc3RhdGVzJTBBJTBBJTBBaW1wb3J0JTIwdG9yY2glMEElMEElMEFkZWYlMjBmb3J3YXJkKHNlbGYlMkMlMjBoaWRkZW5fc3RhdGVzKSUzQSUwQSUyMCUyMCUyMCUyMGlmJTIwdG9yY2guaXNfYXV0b2Nhc3RfZW5hYmxlZCgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwd2l0aCUyMHRvcmNoLmN1ZGEuYW1wLmF1dG9jYXN0KGVuYWJsZWQlM0RGYWxzZSklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByZXR1cm4lMjBzZWxmLl9mb3J3YXJkKGhpZGRlbl9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMHNlbGYuX2ZvcndhcmQoaGlkZGVuX3N0YXRlcyk=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">_forward</span>(<span class="hljs-params">self, hidden_states</span>):
    hidden_gelu = self.gelu_act(self.wi_0(hidden_states))
    hidden_linear = self.wi_1(hidden_states)
    hidden_states = hidden_gelu * hidden_linear
    hidden_states = self.dropout(hidden_states)
    hidden_states = self.wo(hidden_states)
    <span class="hljs-keyword">return</span> hidden_states


<span class="hljs-keyword">import</span> torch


<span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states</span>):
    <span class="hljs-keyword">if</span> torch.is_autocast_enabled():
        <span class="hljs-keyword">with</span> torch.cuda.amp.autocast(enabled=<span class="hljs-literal">False</span>):
            <span class="hljs-keyword">return</span> self._forward(hidden_states)
    <span class="hljs-keyword">else</span>:
        <span class="hljs-keyword">return</span> self._forward(hidden_states)`}}),Is=new A({props:{code:"ZnJvbSUyMGRlYnVnX3V0aWxzJTIwaW1wb3J0JTIwZGV0ZWN0X292ZXJmbG93JTBBJTBBJTBBY2xhc3MlMjBUNUxheWVyRkYobm4uTW9kdWxlKSUzQSUwQSUyMCUyMCUyMCUyMCU1Qi4uLiU1RCUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMGhpZGRlbl9zdGF0ZXMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yd2FyZGVkX3N0YXRlcyUyMCUzRCUyMHNlbGYubGF5ZXJfbm9ybShoaWRkZW5fc3RhdGVzKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGRldGVjdF9vdmVyZmxvdyhmb3J3YXJkZWRfc3RhdGVzJTJDJTIwJTIyYWZ0ZXIlMjBsYXllcl9ub3JtJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvcndhcmRlZF9zdGF0ZXMlMjAlM0QlMjBzZWxmLkRlbnNlUmVsdURlbnNlKGZvcndhcmRlZF9zdGF0ZXMpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGV0ZWN0X292ZXJmbG93KGZvcndhcmRlZF9zdGF0ZXMlMkMlMjAlMjJhZnRlciUyMERlbnNlUmVsdURlbnNlJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMGhpZGRlbl9zdGF0ZXMlMjAlMkIlMjBzZWxmLmRyb3BvdXQoZm9yd2FyZGVkX3N0YXRlcyk=",highlighted:`<span class="hljs-keyword">from</span> debug_utils <span class="hljs-keyword">import</span> detect_overflow


<span class="hljs-keyword">class</span> <span class="hljs-title class_">T5LayerFF</span>(nn.Module):
    [...]

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, hidden_states</span>):
        forwarded_states = self.layer_norm(hidden_states)
        detect_overflow(forwarded_states, <span class="hljs-string">&quot;after layer_norm&quot;</span>)
        forwarded_states = self.DenseReluDense(forwarded_states)
        detect_overflow(forwarded_states, <span class="hljs-string">&quot;after DenseReluDense&quot;</span>)
        <span class="hljs-keyword">return</span> hidden_states + self.dropout(forwarded_states)`}}),Us=new A({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5kZWJ1Z191dGlscyUyMGltcG9ydCUyMERlYnVnVW5kZXJmbG93T3ZlcmZsb3clMEElMEFkZWJ1Z19vdmVyZmxvdyUyMCUzRCUyMERlYnVnVW5kZXJmbG93T3ZlcmZsb3cobW9kZWwlMkMlMjBtYXhfZnJhbWVzX3RvX3NhdmUlM0QxMDAp",highlighted:`<span class="hljs-keyword">from</span> transformers.debug_utils <span class="hljs-keyword">import</span> DebugUnderflowOverflow

debug_overflow = DebugUnderflowOverflow(model, max_frames_to_save=<span class="hljs-number">100</span>)`}}),Cs=new ua({}),vs=new A({props:{code:"ZGVidWdfb3ZlcmZsb3clMjAlM0QlMjBEZWJ1Z1VuZGVyZmxvd092ZXJmbG93KG1vZGVsJTJDJTIwdHJhY2VfYmF0Y2hfbnVtcyUzRCU1QjElMkMlMjAzJTVEKQ==",highlighted:'debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[<span class="hljs-number">1</span>, <span class="hljs-number">3</span>])'}}),Zs=new A({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKioqJTIwU3RhcnRpbmclMjBiYXRjaCUyMG51bWJlciUzRDElMjAqKiolMEFhYnMlMjBtaW4lMjAlMjBhYnMlMjBtYXglMjAlMjBtZXRhZGF0YSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNoYXJlZCUyMEVtYmVkZGluZyUwQTEuMDFlLTA2JTIwNy45MmUlMkIwMiUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjAyLjQ3ZSUyQjA0JTIwaW5wdXQlNUIwJTVEJTBBNS4zNmUtMDUlMjA3LjkyZSUyQjAyJTIwb3V0cHV0JTBBJTVCLi4uJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGVjb2Rlci5kcm9wb3V0JTIwRHJvcG91dCUwQTEuNjBlLTA3JTIwMi4yN2UlMkIwMSUyMGlucHV0JTVCMCU1RCUwQTAuMDBlJTJCMDAlMjAyLjUyZSUyQjAxJTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZGVjb2RlciUyMFQ1U3RhY2slMEElMjAlMjAlMjAlMjAlMjBub3QlMjBhJTIwdGVuc29yJTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbG1faGVhZCUyMExpbmVhciUwQTEuMDFlLTA2JTIwNy45MmUlMkIwMiUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjAxLjExZSUyQjAwJTIwaW5wdXQlNUIwJTVEJTBBNi4wNmUtMDIlMjA4LjM5ZSUyQjAxJTIwb3V0cHV0JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwVDVGb3JDb25kaXRpb25hbEdlbmVyYXRpb24lMEElMjAlMjAlMjAlMjAlMjBub3QlMjBhJTIwdGVuc29yJTIwb3V0cHV0JTBBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKioqJTIwU3RhcnRpbmclMjBiYXRjaCUyMG51bWJlciUzRDMlMjAqKiolMEFhYnMlMjBtaW4lMjAlMjBhYnMlMjBtYXglMjAlMjBtZXRhZGF0YSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNoYXJlZCUyMEVtYmVkZGluZyUwQTEuMDFlLTA2JTIwNy45MmUlMkIwMiUyMHdlaWdodCUwQTAuMDBlJTJCMDAlMjAyLjc4ZSUyQjA0JTIwaW5wdXQlNUIwJTVEJTBBNS4zNmUtMDUlMjA3LjkyZSUyQjAyJTIwb3V0cHV0JTBBJTVCLi4uJTVE",highlighted:`                  *** Starting batch number=1 ***
abs min  abs max  metadata
                  shared Embedding
1.01e<span class="hljs-string">-06</span> 7.92e<span class="hljs-string">+02</span> weight
0.00e<span class="hljs-string">+00</span> 2.47e<span class="hljs-string">+04</span> input[0]
5.36e<span class="hljs-string">-05</span> 7.92e<span class="hljs-string">+02</span> output
[...]
                  decoder.dropout Dropout
1.60e<span class="hljs-string">-07</span> 2.27e<span class="hljs-string">+01</span> input[0]
0.00e<span class="hljs-string">+00</span> 2.52e<span class="hljs-string">+01</span> output
                  decoder T5Stack
     not a tensor output
                  lm_head Linear
1.01e<span class="hljs-string">-06</span> 7.92e<span class="hljs-string">+02</span> weight
0.00e<span class="hljs-string">+00</span> 1.11e<span class="hljs-string">+00</span> input[0]
6.06e<span class="hljs-string">-02</span> 8.39e<span class="hljs-string">+01</span> output
                   T5ForConditionalGeneration
     not a tensor output

                  *** Starting batch number=3 ***
abs min  abs max  metadata
                  shared Embedding
1.01e<span class="hljs-string">-06</span> 7.92e<span class="hljs-string">+02</span> weight
0.00e<span class="hljs-string">+00</span> 2.78e<span class="hljs-string">+04</span> input[0]
5.36e<span class="hljs-string">-05</span> 7.92e<span class="hljs-string">+02</span> output
[...]`}}),As=new A({props:{code:"ZGVidWdfb3ZlcmZsb3clMjAlM0QlMjBEZWJ1Z1VuZGVyZmxvd092ZXJmbG93KG1vZGVsJTJDJTIwdHJhY2VfYmF0Y2hfbnVtcyUzRCU1QjElMkMlMjAzJTVEJTJDJTIwYWJvcnRfYWZ0ZXJfYmF0Y2hfbnVtJTNEMyk=",highlighted:'debug_overflow = DebugUnderflowOverflow(model, trace_batch_nums=[<span class="hljs-number">1</span>, <span class="hljs-number">3</span>], abort_after_batch_num=<span class="hljs-number">3</span>)'}}),{c(){h=r("meta"),C=i(),w=r("h1"),y=r("a"),E=r("span"),d(I.$$.fragment),U=i(),$=r("span"),ia=a("Debugging"),Fe=i(),W=r("h2"),Y=r("a"),Ks=r("span"),d(cs.$$.fragment),ca=i(),qs=r("span"),ha=a("Multi-GPU Network Issues Debug"),ze=i(),x=r("p"),wa=a("When training or inferencing with "),se=r("code"),da=a("DistributedDataParallel"),ma=a(" and multiple GPU, if you run into issue of inter-communication between processes and/or nodes, you can use the following script to diagnose network issues."),He=i(),d(hs.$$.fragment),Oe=i(),Gs=r("p"),fa=a("For example to test how 2 GPUs interact do:"),Pe=i(),d(ws.$$.fragment),Ke=i(),_s=r("p"),Ma=a("If both processes can talk to each and allocate GPU memory each will print an OK status."),qe=i(),ks=r("p"),ba=a("For more GPUs or nodes adjust the arguments in the script."),sl=i(),Vs=r("p"),ja=a("You will find a lot more details inside the diagnostics script and even a recipe to how you could run it in a SLURM environment."),el=i(),F=r("p"),ya=a("An additional level of debug is to add "),ee=r("code"),Ta=a("NCCL_DEBUG=INFO"),Ja=a(" environment variable as follows:"),ll=i(),d(ds.$$.fragment),al=i(),Bs=r("p"),Ia=a("This will dump a lot of NCCL-related debug information, which you can then search online if you find that some problems are reported. Or if you\u2019re not sure how to interpret the output you can share the log file in an Issue."),nl=i(),L=r("h2"),z=r("a"),le=r("span"),d(ms.$$.fragment),Ua=i(),ae=r("span"),Ca=a("Underflow and Overflow Detection"),tl=i(),d(H.$$.fragment),rl=i(),d(O.$$.fragment),ol=i(),d(P.$$.fragment),pl=i(),_=r("p"),va=a("If you start getting "),ne=r("code"),Za=a("loss=NaN"),Aa=a(" or the model inhibits some other abnormal behavior due to "),te=r("code"),Ea=a("inf"),Ga=a(" or "),re=r("code"),_a=a("nan"),ka=a(` in
activations or weights one needs to discover where the first underflow or overflow happens and what led to it. Luckily
you can accomplish that easily by activating a special module that will do the detection automatically.`),ul=i(),K=r("p"),Va=a("If you\u2019re using "),gs=r("a"),Ba=a("Trainer"),ga=a(", you just need to add:"),il=i(),d(fs.$$.fragment),cl=i(),D=r("p"),Ra=a("to the normal command line arguments, or pass "),oe=r("code"),$a=a('debug="underflow_overflow"'),Da=a(` when creating the
`),Rs=r("a"),Na=a("TrainingArguments"),Wa=a(" object."),hl=i(),$s=r("p"),La=a("If you\u2019re using your own training loop or another Trainer you can accomplish the same with:"),wl=i(),d(Ms.$$.fragment),dl=i(),G=r("p"),Ds=r("a"),Xa=a("DebugUnderflowOverflow"),Qa=a(` inserts hooks into the model that immediately after each
forward call will test input and output variables and also the corresponding module\u2019s weights. As soon as `),pe=r("code"),Sa=a("inf"),Ya=a(` or
`),ue=r("code"),xa=a("nan"),Fa=a(` is detected in at least one element of the activations or weights, the program will assert and print a report
like this (this was caught with `),ie=r("code"),za=a("google/mt5-small"),Ha=a(" under fp16 mixed precision):"),ml=i(),d(bs.$$.fragment),fl=i(),Ns=r("p"),Oa=a("The example output has been trimmed in the middle for brevity."),Ml=i(),J=r("p"),Pa=a(`The second column shows the value of the absolute largest element, so if you have a closer look at the last few frames,
the inputs and outputs were in the range of `),ce=r("code"),Ka=a("1e4"),qa=a(`. So when this training was done under fp16 mixed precision the very
last step overflowed (since under `),he=r("code"),sn=a("fp16"),en=a(" the largest number before "),we=r("code"),ln=a("inf"),an=a(" is "),de=r("code"),nn=a("64e3"),tn=a(`). To avoid overflows under
`),me=r("code"),rn=a("fp16"),on=a(" the activations must remain way below "),fe=r("code"),pn=a("1e4"),un=a(", because "),Me=r("code"),cn=a("1e4 * 1e4 = 1e8"),hn=a(` so any matrix multiplication with
large activations is going to lead to a numerical overflow condition.`),bl=i(),q=r("p"),wn=a("At the very start of the trace you can discover at which batch number the problem occurred (here "),be=r("code"),dn=a("Detected inf/nan during batch_number=0"),mn=a(" means the problem occurred on the first batch)."),jl=i(),Ws=r("p"),fn=a(`Each reported frame starts by declaring the fully qualified entry for the corresponding module this frame is reporting
for. If we look just at this frame:`),yl=i(),d(js.$$.fragment),Tl=i(),k=r("p"),Mn=a("Here, "),je=r("code"),bn=a("encoder.block.2.layer.1.layer_norm"),jn=a(` indicates that it was a layer norm for the first layer, of the second
block of the encoder. And the specific calls of the `),ye=r("code"),yn=a("forward"),Tn=a(" is "),Te=r("code"),Jn=a("T5LayerNorm"),In=a("."),Jl=i(),Ls=r("p"),Un=a("Let\u2019s look at the last few frames of that report:"),Il=i(),d(ys.$$.fragment),Ul=i(),v=r("p"),Cn=a("The last frame reports for "),Je=r("code"),vn=a("Dropout.forward"),Zn=a(` function with the first entry for the only input and the second for the
only output. You can see that it was called from an attribute `),Ie=r("code"),An=a("dropout"),En=a(" inside "),Ue=r("code"),Gn=a("DenseReluDense"),_n=a(` class. We can see
that it happened during the first layer, of the 2nd block, during the very first batch. Finally, the absolute largest
input elements was `),Ce=r("code"),kn=a("6.27e+04"),Vn=a(" and same for the output was "),ve=r("code"),Bn=a("inf"),gn=a("."),Cl=i(),V=r("p"),Rn=a("You can see here, that "),Ze=r("code"),$n=a("T5DenseGatedGeluDense.forward"),Dn=a(` resulted in output activations, whose absolute max value was
around 62.7K, which is very close to fp16\u2019s top limit of 64K. In the next frame we have `),Ae=r("code"),Nn=a("Dropout"),Wn=a(` which renormalizes
the weights, after it zeroed some of the elements, which pushes the absolute max value to more than 64K, and we get an
overflow (`),Ee=r("code"),Ln=a("inf"),Xn=a(")."),vl=i(),Xs=r("p"),Qn=a(`As you can see it\u2019s the previous frames that we need to look into when the numbers start going into very large for fp16
numbers.`),Zl=i(),ss=r("p"),Sn=a("Let\u2019s match the report to the code from "),Ge=r("code"),Yn=a("models/t5/modeling_t5.py"),xn=a(":"),Al=i(),d(Ts.$$.fragment),El=i(),es=r("p"),Fn=a("Now it\u2019s easy to see the "),_e=r("code"),zn=a("dropout"),Hn=a(" call, and all the previous calls as well."),Gl=i(),ls=r("p"),On=a("Since the detection is happening in a forward hook, these reports are printed immediately after each "),ke=r("code"),Pn=a("forward"),Kn=a(`
returns.`),_l=i(),B=r("p"),qn=a(`Going back to the full report, to act on it and to fix the problem, we need to go a few frames up where the numbers
started to go up and most likely switch to the `),Ve=r("code"),st=a("fp32"),et=a(` mode here, so that the numbers don\u2019t overflow when multiplied
or summed up. Of course, there might be other solutions. For example, we could turn off `),Be=r("code"),lt=a("amp"),at=a(` temporarily if it\u2019s
enabled, after moving the original `),ge=r("code"),nt=a("forward"),tt=a(" into a helper wrapper, like so:"),kl=i(),d(Js.$$.fragment),Vl=i(),N=r("p"),rt=a(`Since the automatic detector only reports on inputs and outputs of full frames, once you know where to look, you may
want to analyse the intermediary stages of any specific `),Re=r("code"),ot=a("forward"),pt=a(` function as well. In such a case you can use the
`),$e=r("code"),ut=a("detect_overflow"),it=a(" helper function to inject the detector where you want it, for example:"),Bl=i(),d(Is.$$.fragment),gl=i(),g=r("p"),ct=a("You can see that we added 2 of these and now we track if "),De=r("code"),ht=a("inf"),wt=a(" or "),Ne=r("code"),dt=a("nan"),mt=a(" for "),We=r("code"),ft=a("forwarded_states"),Mt=a(` was detected
somewhere in between.`),Rl=i(),as=r("p"),bt=a("Actually, the detector already reports these because each of the calls in the example above is a "),Le=r("code"),jt=a("nn.Module"),yt=a(`, but
let\u2019s say if you had some local direct calculations this is how you\u2019d do that.`),$l=i(),Qs=r("p"),Tt=a(`Additionally, if you\u2019re instantiating the debugger in your own code, you can adjust the number of frames printed from
its default, e.g.:`),Dl=i(),d(Us.$$.fragment),Nl=i(),X=r("h3"),ns=r("a"),Xe=r("span"),d(Cs.$$.fragment),Jt=i(),Qe=r("span"),It=a("Specific batch absolute min and max value tracing"),Wl=i(),Ss=r("p"),Ut=a("The same debugging class can be used for per-batch tracing with the underflow/overflow detection feature turned off."),Ll=i(),ts=r("p"),Ct=a("Let\u2019s say you want to watch the absolute min and max values for all the ingredients of each "),Se=r("code"),vt=a("forward"),Zt=a(` call of a given
batch, and only do that for batches 1 and 3. Then you instantiate this class as:`),Xl=i(),d(vs.$$.fragment),Ql=i(),Ys=r("p"),At=a("And now full batches 1 and 3 will be traced using the same format as the underflow/overflow detector does."),Sl=i(),xs=r("p"),Et=a("Batches are 0-indexed."),Yl=i(),Fs=r("p"),Gt=a(`This is helpful if you know that the program starts misbehaving after a certain batch number, so you can fast-forward
right to that area. Here is a sample truncated output for such configuration:`),xl=i(),d(Zs.$$.fragment),Fl=i(),zs=r("p"),_t=a(`Here you will get a huge number of frames dumped - as many as there were forward calls in your model, so it may or may
not what you want, but sometimes it can be easier to use for debugging purposes than a normal debugger. For example, if
a problem starts happening at batch number 150. So you can dump traces for batches 149 and 150 and compare where
numbers started to diverge.`),zl=i(),Hs=r("p"),kt=a("You can also specify the batch number after which to stop the training, with:"),Hl=i(),d(As.$$.fragment),this.h()},l(s){const t=so('[data-svelte="svelte-1phssyn"]',document.head);h=o(t,"META",{name:!0,content:!0}),t.forEach(e),C=c(s),w=o(s,"H1",{class:!0});var Es=u(w);y=o(Es,"A",{id:!0,class:!0,href:!0});var Ye=u(y);E=o(Ye,"SPAN",{});var xe=u(E);m(I.$$.fragment,xe),xe.forEach(e),Ye.forEach(e),U=c(Es),$=o(Es,"SPAN",{});var Bt=u($);ia=n(Bt,"Debugging"),Bt.forEach(e),Es.forEach(e),Fe=c(s),W=o(s,"H2",{class:!0});var Pl=u(W);Y=o(Pl,"A",{id:!0,class:!0,href:!0});var gt=u(Y);Ks=o(gt,"SPAN",{});var Rt=u(Ks);m(cs.$$.fragment,Rt),Rt.forEach(e),gt.forEach(e),ca=c(Pl),qs=o(Pl,"SPAN",{});var $t=u(qs);ha=n($t,"Multi-GPU Network Issues Debug"),$t.forEach(e),Pl.forEach(e),ze=c(s),x=o(s,"P",{});var Kl=u(x);wa=n(Kl,"When training or inferencing with "),se=o(Kl,"CODE",{});var Dt=u(se);da=n(Dt,"DistributedDataParallel"),Dt.forEach(e),ma=n(Kl," and multiple GPU, if you run into issue of inter-communication between processes and/or nodes, you can use the following script to diagnose network issues."),Kl.forEach(e),He=c(s),m(hs.$$.fragment,s),Oe=c(s),Gs=o(s,"P",{});var Nt=u(Gs);fa=n(Nt,"For example to test how 2 GPUs interact do:"),Nt.forEach(e),Pe=c(s),m(ws.$$.fragment,s),Ke=c(s),_s=o(s,"P",{});var Wt=u(_s);Ma=n(Wt,"If both processes can talk to each and allocate GPU memory each will print an OK status."),Wt.forEach(e),qe=c(s),ks=o(s,"P",{});var Lt=u(ks);ba=n(Lt,"For more GPUs or nodes adjust the arguments in the script."),Lt.forEach(e),sl=c(s),Vs=o(s,"P",{});var Xt=u(Vs);ja=n(Xt,"You will find a lot more details inside the diagnostics script and even a recipe to how you could run it in a SLURM environment."),Xt.forEach(e),el=c(s),F=o(s,"P",{});var ql=u(F);ya=n(ql,"An additional level of debug is to add "),ee=o(ql,"CODE",{});var Qt=u(ee);Ta=n(Qt,"NCCL_DEBUG=INFO"),Qt.forEach(e),Ja=n(ql," environment variable as follows:"),ql.forEach(e),ll=c(s),m(ds.$$.fragment,s),al=c(s),Bs=o(s,"P",{});var St=u(Bs);Ia=n(St,"This will dump a lot of NCCL-related debug information, which you can then search online if you find that some problems are reported. Or if you\u2019re not sure how to interpret the output you can share the log file in an Issue."),St.forEach(e),nl=c(s),L=o(s,"H2",{class:!0});var sa=u(L);z=o(sa,"A",{id:!0,class:!0,href:!0});var Yt=u(z);le=o(Yt,"SPAN",{});var xt=u(le);m(ms.$$.fragment,xt),xt.forEach(e),Yt.forEach(e),Ua=c(sa),ae=o(sa,"SPAN",{});var Ft=u(ae);Ca=n(Ft,"Underflow and Overflow Detection"),Ft.forEach(e),sa.forEach(e),tl=c(s),m(H.$$.fragment,s),rl=c(s),m(O.$$.fragment,s),ol=c(s),m(P.$$.fragment,s),pl=c(s),_=o(s,"P",{});var rs=u(_);va=n(rs,"If you start getting "),ne=o(rs,"CODE",{});var zt=u(ne);Za=n(zt,"loss=NaN"),zt.forEach(e),Aa=n(rs," or the model inhibits some other abnormal behavior due to "),te=o(rs,"CODE",{});var Ht=u(te);Ea=n(Ht,"inf"),Ht.forEach(e),Ga=n(rs," or "),re=o(rs,"CODE",{});var Ot=u(re);_a=n(Ot,"nan"),Ot.forEach(e),ka=n(rs,` in
activations or weights one needs to discover where the first underflow or overflow happens and what led to it. Luckily
you can accomplish that easily by activating a special module that will do the detection automatically.`),rs.forEach(e),ul=c(s),K=o(s,"P",{});var ea=u(K);Va=n(ea,"If you\u2019re using "),gs=o(ea,"A",{href:!0});var Pt=u(gs);Ba=n(Pt,"Trainer"),Pt.forEach(e),ga=n(ea,", you just need to add:"),ea.forEach(e),il=c(s),m(fs.$$.fragment,s),cl=c(s),D=o(s,"P",{});var Os=u(D);Ra=n(Os,"to the normal command line arguments, or pass "),oe=o(Os,"CODE",{});var Kt=u(oe);$a=n(Kt,'debug="underflow_overflow"'),Kt.forEach(e),Da=n(Os,` when creating the
`),Rs=o(Os,"A",{href:!0});var qt=u(Rs);Na=n(qt,"TrainingArguments"),qt.forEach(e),Wa=n(Os," object."),Os.forEach(e),hl=c(s),$s=o(s,"P",{});var sr=u($s);La=n(sr,"If you\u2019re using your own training loop or another Trainer you can accomplish the same with:"),sr.forEach(e),wl=c(s),m(Ms.$$.fragment,s),dl=c(s),G=o(s,"P",{});var Q=u(G);Ds=o(Q,"A",{href:!0});var er=u(Ds);Xa=n(er,"DebugUnderflowOverflow"),er.forEach(e),Qa=n(Q,` inserts hooks into the model that immediately after each
forward call will test input and output variables and also the corresponding module\u2019s weights. As soon as `),pe=o(Q,"CODE",{});var lr=u(pe);Sa=n(lr,"inf"),lr.forEach(e),Ya=n(Q,` or
`),ue=o(Q,"CODE",{});var ar=u(ue);xa=n(ar,"nan"),ar.forEach(e),Fa=n(Q,` is detected in at least one element of the activations or weights, the program will assert and print a report
like this (this was caught with `),ie=o(Q,"CODE",{});var nr=u(ie);za=n(nr,"google/mt5-small"),nr.forEach(e),Ha=n(Q," under fp16 mixed precision):"),Q.forEach(e),ml=c(s),m(bs.$$.fragment,s),fl=c(s),Ns=o(s,"P",{});var tr=u(Ns);Oa=n(tr,"The example output has been trimmed in the middle for brevity."),tr.forEach(e),Ml=c(s),J=o(s,"P",{});var Z=u(J);Pa=n(Z,`The second column shows the value of the absolute largest element, so if you have a closer look at the last few frames,
the inputs and outputs were in the range of `),ce=o(Z,"CODE",{});var rr=u(ce);Ka=n(rr,"1e4"),rr.forEach(e),qa=n(Z,`. So when this training was done under fp16 mixed precision the very
last step overflowed (since under `),he=o(Z,"CODE",{});var or=u(he);sn=n(or,"fp16"),or.forEach(e),en=n(Z," the largest number before "),we=o(Z,"CODE",{});var pr=u(we);ln=n(pr,"inf"),pr.forEach(e),an=n(Z," is "),de=o(Z,"CODE",{});var ur=u(de);nn=n(ur,"64e3"),ur.forEach(e),tn=n(Z,`). To avoid overflows under
`),me=o(Z,"CODE",{});var ir=u(me);rn=n(ir,"fp16"),ir.forEach(e),on=n(Z," the activations must remain way below "),fe=o(Z,"CODE",{});var cr=u(fe);pn=n(cr,"1e4"),cr.forEach(e),un=n(Z,", because "),Me=o(Z,"CODE",{});var hr=u(Me);cn=n(hr,"1e4 * 1e4 = 1e8"),hr.forEach(e),hn=n(Z,` so any matrix multiplication with
large activations is going to lead to a numerical overflow condition.`),Z.forEach(e),bl=c(s),q=o(s,"P",{});var la=u(q);wn=n(la,"At the very start of the trace you can discover at which batch number the problem occurred (here "),be=o(la,"CODE",{});var wr=u(be);dn=n(wr,"Detected inf/nan during batch_number=0"),wr.forEach(e),mn=n(la," means the problem occurred on the first batch)."),la.forEach(e),jl=c(s),Ws=o(s,"P",{});var dr=u(Ws);fn=n(dr,`Each reported frame starts by declaring the fully qualified entry for the corresponding module this frame is reporting
for. If we look just at this frame:`),dr.forEach(e),yl=c(s),m(js.$$.fragment,s),Tl=c(s),k=o(s,"P",{});var os=u(k);Mn=n(os,"Here, "),je=o(os,"CODE",{});var mr=u(je);bn=n(mr,"encoder.block.2.layer.1.layer_norm"),mr.forEach(e),jn=n(os,` indicates that it was a layer norm for the first layer, of the second
block of the encoder. And the specific calls of the `),ye=o(os,"CODE",{});var fr=u(ye);yn=n(fr,"forward"),fr.forEach(e),Tn=n(os," is "),Te=o(os,"CODE",{});var Mr=u(Te);Jn=n(Mr,"T5LayerNorm"),Mr.forEach(e),In=n(os,"."),os.forEach(e),Jl=c(s),Ls=o(s,"P",{});var br=u(Ls);Un=n(br,"Let\u2019s look at the last few frames of that report:"),br.forEach(e),Il=c(s),m(ys.$$.fragment,s),Ul=c(s),v=o(s,"P",{});var R=u(v);Cn=n(R,"The last frame reports for "),Je=o(R,"CODE",{});var jr=u(Je);vn=n(jr,"Dropout.forward"),jr.forEach(e),Zn=n(R,` function with the first entry for the only input and the second for the
only output. You can see that it was called from an attribute `),Ie=o(R,"CODE",{});var yr=u(Ie);An=n(yr,"dropout"),yr.forEach(e),En=n(R," inside "),Ue=o(R,"CODE",{});var Tr=u(Ue);Gn=n(Tr,"DenseReluDense"),Tr.forEach(e),_n=n(R,` class. We can see
that it happened during the first layer, of the 2nd block, during the very first batch. Finally, the absolute largest
input elements was `),Ce=o(R,"CODE",{});var Jr=u(Ce);kn=n(Jr,"6.27e+04"),Jr.forEach(e),Vn=n(R," and same for the output was "),ve=o(R,"CODE",{});var Ir=u(ve);Bn=n(Ir,"inf"),Ir.forEach(e),gn=n(R,"."),R.forEach(e),Cl=c(s),V=o(s,"P",{});var ps=u(V);Rn=n(ps,"You can see here, that "),Ze=o(ps,"CODE",{});var Ur=u(Ze);$n=n(Ur,"T5DenseGatedGeluDense.forward"),Ur.forEach(e),Dn=n(ps,` resulted in output activations, whose absolute max value was
around 62.7K, which is very close to fp16\u2019s top limit of 64K. In the next frame we have `),Ae=o(ps,"CODE",{});var Cr=u(Ae);Nn=n(Cr,"Dropout"),Cr.forEach(e),Wn=n(ps,` which renormalizes
the weights, after it zeroed some of the elements, which pushes the absolute max value to more than 64K, and we get an
overflow (`),Ee=o(ps,"CODE",{});var vr=u(Ee);Ln=n(vr,"inf"),vr.forEach(e),Xn=n(ps,")."),ps.forEach(e),vl=c(s),Xs=o(s,"P",{});var Zr=u(Xs);Qn=n(Zr,`As you can see it\u2019s the previous frames that we need to look into when the numbers start going into very large for fp16
numbers.`),Zr.forEach(e),Zl=c(s),ss=o(s,"P",{});var aa=u(ss);Sn=n(aa,"Let\u2019s match the report to the code from "),Ge=o(aa,"CODE",{});var Ar=u(Ge);Yn=n(Ar,"models/t5/modeling_t5.py"),Ar.forEach(e),xn=n(aa,":"),aa.forEach(e),Al=c(s),m(Ts.$$.fragment,s),El=c(s),es=o(s,"P",{});var na=u(es);Fn=n(na,"Now it\u2019s easy to see the "),_e=o(na,"CODE",{});var Er=u(_e);zn=n(Er,"dropout"),Er.forEach(e),Hn=n(na," call, and all the previous calls as well."),na.forEach(e),Gl=c(s),ls=o(s,"P",{});var ta=u(ls);On=n(ta,"Since the detection is happening in a forward hook, these reports are printed immediately after each "),ke=o(ta,"CODE",{});var Gr=u(ke);Pn=n(Gr,"forward"),Gr.forEach(e),Kn=n(ta,`
returns.`),ta.forEach(e),_l=c(s),B=o(s,"P",{});var us=u(B);qn=n(us,`Going back to the full report, to act on it and to fix the problem, we need to go a few frames up where the numbers
started to go up and most likely switch to the `),Ve=o(us,"CODE",{});var _r=u(Ve);st=n(_r,"fp32"),_r.forEach(e),et=n(us,` mode here, so that the numbers don\u2019t overflow when multiplied
or summed up. Of course, there might be other solutions. For example, we could turn off `),Be=o(us,"CODE",{});var kr=u(Be);lt=n(kr,"amp"),kr.forEach(e),at=n(us,` temporarily if it\u2019s
enabled, after moving the original `),ge=o(us,"CODE",{});var Vr=u(ge);nt=n(Vr,"forward"),Vr.forEach(e),tt=n(us," into a helper wrapper, like so:"),us.forEach(e),kl=c(s),m(Js.$$.fragment,s),Vl=c(s),N=o(s,"P",{});var Ps=u(N);rt=n(Ps,`Since the automatic detector only reports on inputs and outputs of full frames, once you know where to look, you may
want to analyse the intermediary stages of any specific `),Re=o(Ps,"CODE",{});var Br=u(Re);ot=n(Br,"forward"),Br.forEach(e),pt=n(Ps,` function as well. In such a case you can use the
`),$e=o(Ps,"CODE",{});var gr=u($e);ut=n(gr,"detect_overflow"),gr.forEach(e),it=n(Ps," helper function to inject the detector where you want it, for example:"),Ps.forEach(e),Bl=c(s),m(Is.$$.fragment,s),gl=c(s),g=o(s,"P",{});var is=u(g);ct=n(is,"You can see that we added 2 of these and now we track if "),De=o(is,"CODE",{});var Rr=u(De);ht=n(Rr,"inf"),Rr.forEach(e),wt=n(is," or "),Ne=o(is,"CODE",{});var $r=u(Ne);dt=n($r,"nan"),$r.forEach(e),mt=n(is," for "),We=o(is,"CODE",{});var Dr=u(We);ft=n(Dr,"forwarded_states"),Dr.forEach(e),Mt=n(is,` was detected
somewhere in between.`),is.forEach(e),Rl=c(s),as=o(s,"P",{});var ra=u(as);bt=n(ra,"Actually, the detector already reports these because each of the calls in the example above is a "),Le=o(ra,"CODE",{});var Nr=u(Le);jt=n(Nr,"nn.Module"),Nr.forEach(e),yt=n(ra,`, but
let\u2019s say if you had some local direct calculations this is how you\u2019d do that.`),ra.forEach(e),$l=c(s),Qs=o(s,"P",{});var Wr=u(Qs);Tt=n(Wr,`Additionally, if you\u2019re instantiating the debugger in your own code, you can adjust the number of frames printed from
its default, e.g.:`),Wr.forEach(e),Dl=c(s),m(Us.$$.fragment,s),Nl=c(s),X=o(s,"H3",{class:!0});var oa=u(X);ns=o(oa,"A",{id:!0,class:!0,href:!0});var Lr=u(ns);Xe=o(Lr,"SPAN",{});var Xr=u(Xe);m(Cs.$$.fragment,Xr),Xr.forEach(e),Lr.forEach(e),Jt=c(oa),Qe=o(oa,"SPAN",{});var Qr=u(Qe);It=n(Qr,"Specific batch absolute min and max value tracing"),Qr.forEach(e),oa.forEach(e),Wl=c(s),Ss=o(s,"P",{});var Sr=u(Ss);Ut=n(Sr,"The same debugging class can be used for per-batch tracing with the underflow/overflow detection feature turned off."),Sr.forEach(e),Ll=c(s),ts=o(s,"P",{});var pa=u(ts);Ct=n(pa,"Let\u2019s say you want to watch the absolute min and max values for all the ingredients of each "),Se=o(pa,"CODE",{});var Yr=u(Se);vt=n(Yr,"forward"),Yr.forEach(e),Zt=n(pa,` call of a given
batch, and only do that for batches 1 and 3. Then you instantiate this class as:`),pa.forEach(e),Xl=c(s),m(vs.$$.fragment,s),Ql=c(s),Ys=o(s,"P",{});var xr=u(Ys);At=n(xr,"And now full batches 1 and 3 will be traced using the same format as the underflow/overflow detector does."),xr.forEach(e),Sl=c(s),xs=o(s,"P",{});var Fr=u(xs);Et=n(Fr,"Batches are 0-indexed."),Fr.forEach(e),Yl=c(s),Fs=o(s,"P",{});var zr=u(Fs);Gt=n(zr,`This is helpful if you know that the program starts misbehaving after a certain batch number, so you can fast-forward
right to that area. Here is a sample truncated output for such configuration:`),zr.forEach(e),xl=c(s),m(Zs.$$.fragment,s),Fl=c(s),zs=o(s,"P",{});var Hr=u(zs);_t=n(Hr,`Here you will get a huge number of frames dumped - as many as there were forward calls in your model, so it may or may
not what you want, but sometimes it can be easier to use for debugging purposes than a normal debugger. For example, if
a problem starts happening at batch number 150. So you can dump traces for batches 149 and 150 and compare where
numbers started to diverge.`),Hr.forEach(e),zl=c(s),Hs=o(s,"P",{});var Or=u(Hs);kt=n(Or,"You can also specify the batch number after which to stop the training, with:"),Or.forEach(e),Hl=c(s),m(As.$$.fragment,s),this.h()},h(){T(h,"name","hf:doc:metadata"),T(h,"content",JSON.stringify(ro)),T(y,"id","debugging"),T(y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(y,"href","#debugging"),T(w,"class","relative group"),T(Y,"id","multigpu-network-issues-debug"),T(Y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(Y,"href","#multigpu-network-issues-debug"),T(W,"class","relative group"),T(z,"id","underflow-and-overflow-detection"),T(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(z,"href","#underflow-and-overflow-detection"),T(L,"class","relative group"),T(gs,"href","/docs/transformers/v4.30.0/en/main_classes/trainer#transformers.Trainer"),T(Rs,"href","/docs/transformers/v4.30.0/en/main_classes/trainer#transformers.TrainingArguments"),T(Ds,"href","/docs/transformers/v4.30.0/en/internal/trainer_utils#transformers.debug_utils.DebugUnderflowOverflow"),T(ns,"id","specific-batch-absolute-min-and-max-value-tracing"),T(ns,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),T(ns,"href","#specific-batch-absolute-min-and-max-value-tracing"),T(X,"class","relative group")},m(s,t){l(document.head,h),p(s,C,t),p(s,w,t),l(w,y),l(y,E),f(I,E,null),l(w,U),l(w,$),l($,ia),p(s,Fe,t),p(s,W,t),l(W,Y),l(Y,Ks),f(cs,Ks,null),l(W,ca),l(W,qs),l(qs,ha),p(s,ze,t),p(s,x,t),l(x,wa),l(x,se),l(se,da),l(x,ma),p(s,He,t),f(hs,s,t),p(s,Oe,t),p(s,Gs,t),l(Gs,fa),p(s,Pe,t),f(ws,s,t),p(s,Ke,t),p(s,_s,t),l(_s,Ma),p(s,qe,t),p(s,ks,t),l(ks,ba),p(s,sl,t),p(s,Vs,t),l(Vs,ja),p(s,el,t),p(s,F,t),l(F,ya),l(F,ee),l(ee,Ta),l(F,Ja),p(s,ll,t),f(ds,s,t),p(s,al,t),p(s,Bs,t),l(Bs,Ia),p(s,nl,t),p(s,L,t),l(L,z),l(z,le),f(ms,le,null),l(L,Ua),l(L,ae),l(ae,Ca),p(s,tl,t),f(H,s,t),p(s,rl,t),f(O,s,t),p(s,ol,t),f(P,s,t),p(s,pl,t),p(s,_,t),l(_,va),l(_,ne),l(ne,Za),l(_,Aa),l(_,te),l(te,Ea),l(_,Ga),l(_,re),l(re,_a),l(_,ka),p(s,ul,t),p(s,K,t),l(K,Va),l(K,gs),l(gs,Ba),l(K,ga),p(s,il,t),f(fs,s,t),p(s,cl,t),p(s,D,t),l(D,Ra),l(D,oe),l(oe,$a),l(D,Da),l(D,Rs),l(Rs,Na),l(D,Wa),p(s,hl,t),p(s,$s,t),l($s,La),p(s,wl,t),f(Ms,s,t),p(s,dl,t),p(s,G,t),l(G,Ds),l(Ds,Xa),l(G,Qa),l(G,pe),l(pe,Sa),l(G,Ya),l(G,ue),l(ue,xa),l(G,Fa),l(G,ie),l(ie,za),l(G,Ha),p(s,ml,t),f(bs,s,t),p(s,fl,t),p(s,Ns,t),l(Ns,Oa),p(s,Ml,t),p(s,J,t),l(J,Pa),l(J,ce),l(ce,Ka),l(J,qa),l(J,he),l(he,sn),l(J,en),l(J,we),l(we,ln),l(J,an),l(J,de),l(de,nn),l(J,tn),l(J,me),l(me,rn),l(J,on),l(J,fe),l(fe,pn),l(J,un),l(J,Me),l(Me,cn),l(J,hn),p(s,bl,t),p(s,q,t),l(q,wn),l(q,be),l(be,dn),l(q,mn),p(s,jl,t),p(s,Ws,t),l(Ws,fn),p(s,yl,t),f(js,s,t),p(s,Tl,t),p(s,k,t),l(k,Mn),l(k,je),l(je,bn),l(k,jn),l(k,ye),l(ye,yn),l(k,Tn),l(k,Te),l(Te,Jn),l(k,In),p(s,Jl,t),p(s,Ls,t),l(Ls,Un),p(s,Il,t),f(ys,s,t),p(s,Ul,t),p(s,v,t),l(v,Cn),l(v,Je),l(Je,vn),l(v,Zn),l(v,Ie),l(Ie,An),l(v,En),l(v,Ue),l(Ue,Gn),l(v,_n),l(v,Ce),l(Ce,kn),l(v,Vn),l(v,ve),l(ve,Bn),l(v,gn),p(s,Cl,t),p(s,V,t),l(V,Rn),l(V,Ze),l(Ze,$n),l(V,Dn),l(V,Ae),l(Ae,Nn),l(V,Wn),l(V,Ee),l(Ee,Ln),l(V,Xn),p(s,vl,t),p(s,Xs,t),l(Xs,Qn),p(s,Zl,t),p(s,ss,t),l(ss,Sn),l(ss,Ge),l(Ge,Yn),l(ss,xn),p(s,Al,t),f(Ts,s,t),p(s,El,t),p(s,es,t),l(es,Fn),l(es,_e),l(_e,zn),l(es,Hn),p(s,Gl,t),p(s,ls,t),l(ls,On),l(ls,ke),l(ke,Pn),l(ls,Kn),p(s,_l,t),p(s,B,t),l(B,qn),l(B,Ve),l(Ve,st),l(B,et),l(B,Be),l(Be,lt),l(B,at),l(B,ge),l(ge,nt),l(B,tt),p(s,kl,t),f(Js,s,t),p(s,Vl,t),p(s,N,t),l(N,rt),l(N,Re),l(Re,ot),l(N,pt),l(N,$e),l($e,ut),l(N,it),p(s,Bl,t),f(Is,s,t),p(s,gl,t),p(s,g,t),l(g,ct),l(g,De),l(De,ht),l(g,wt),l(g,Ne),l(Ne,dt),l(g,mt),l(g,We),l(We,ft),l(g,Mt),p(s,Rl,t),p(s,as,t),l(as,bt),l(as,Le),l(Le,jt),l(as,yt),p(s,$l,t),p(s,Qs,t),l(Qs,Tt),p(s,Dl,t),f(Us,s,t),p(s,Nl,t),p(s,X,t),l(X,ns),l(ns,Xe),f(Cs,Xe,null),l(X,Jt),l(X,Qe),l(Qe,It),p(s,Wl,t),p(s,Ss,t),l(Ss,Ut),p(s,Ll,t),p(s,ts,t),l(ts,Ct),l(ts,Se),l(Se,vt),l(ts,Zt),p(s,Xl,t),f(vs,s,t),p(s,Ql,t),p(s,Ys,t),l(Ys,At),p(s,Sl,t),p(s,xs,t),l(xs,Et),p(s,Yl,t),p(s,Fs,t),l(Fs,Gt),p(s,xl,t),f(Zs,s,t),p(s,Fl,t),p(s,zs,t),l(zs,_t),p(s,zl,t),p(s,Hs,t),l(Hs,kt),p(s,Hl,t),f(As,s,t),Ol=!0},p(s,[t]){const Es={};t&2&&(Es.$$scope={dirty:t,ctx:s}),H.$set(Es);const Ye={};t&2&&(Ye.$$scope={dirty:t,ctx:s}),O.$set(Ye);const xe={};t&2&&(xe.$$scope={dirty:t,ctx:s}),P.$set(xe)},i(s){Ol||(M(I.$$.fragment,s),M(cs.$$.fragment,s),M(hs.$$.fragment,s),M(ws.$$.fragment,s),M(ds.$$.fragment,s),M(ms.$$.fragment,s),M(H.$$.fragment,s),M(O.$$.fragment,s),M(P.$$.fragment,s),M(fs.$$.fragment,s),M(Ms.$$.fragment,s),M(bs.$$.fragment,s),M(js.$$.fragment,s),M(ys.$$.fragment,s),M(Ts.$$.fragment,s),M(Js.$$.fragment,s),M(Is.$$.fragment,s),M(Us.$$.fragment,s),M(Cs.$$.fragment,s),M(vs.$$.fragment,s),M(Zs.$$.fragment,s),M(As.$$.fragment,s),Ol=!0)},o(s){b(I.$$.fragment,s),b(cs.$$.fragment,s),b(hs.$$.fragment,s),b(ws.$$.fragment,s),b(ds.$$.fragment,s),b(ms.$$.fragment,s),b(H.$$.fragment,s),b(O.$$.fragment,s),b(P.$$.fragment,s),b(fs.$$.fragment,s),b(Ms.$$.fragment,s),b(bs.$$.fragment,s),b(js.$$.fragment,s),b(ys.$$.fragment,s),b(Ts.$$.fragment,s),b(Js.$$.fragment,s),b(Is.$$.fragment,s),b(Us.$$.fragment,s),b(Cs.$$.fragment,s),b(vs.$$.fragment,s),b(Zs.$$.fragment,s),b(As.$$.fragment,s),Ol=!1},d(s){e(h),s&&e(C),s&&e(w),j(I),s&&e(Fe),s&&e(W),j(cs),s&&e(ze),s&&e(x),s&&e(He),j(hs,s),s&&e(Oe),s&&e(Gs),s&&e(Pe),j(ws,s),s&&e(Ke),s&&e(_s),s&&e(qe),s&&e(ks),s&&e(sl),s&&e(Vs),s&&e(el),s&&e(F),s&&e(ll),j(ds,s),s&&e(al),s&&e(Bs),s&&e(nl),s&&e(L),j(ms),s&&e(tl),j(H,s),s&&e(rl),j(O,s),s&&e(ol),j(P,s),s&&e(pl),s&&e(_),s&&e(ul),s&&e(K),s&&e(il),j(fs,s),s&&e(cl),s&&e(D),s&&e(hl),s&&e($s),s&&e(wl),j(Ms,s),s&&e(dl),s&&e(G),s&&e(ml),j(bs,s),s&&e(fl),s&&e(Ns),s&&e(Ml),s&&e(J),s&&e(bl),s&&e(q),s&&e(jl),s&&e(Ws),s&&e(yl),j(js,s),s&&e(Tl),s&&e(k),s&&e(Jl),s&&e(Ls),s&&e(Il),j(ys,s),s&&e(Ul),s&&e(v),s&&e(Cl),s&&e(V),s&&e(vl),s&&e(Xs),s&&e(Zl),s&&e(ss),s&&e(Al),j(Ts,s),s&&e(El),s&&e(es),s&&e(Gl),s&&e(ls),s&&e(_l),s&&e(B),s&&e(kl),j(Js,s),s&&e(Vl),s&&e(N),s&&e(Bl),j(Is,s),s&&e(gl),s&&e(g),s&&e(Rl),s&&e(as),s&&e($l),s&&e(Qs),s&&e(Dl),j(Us,s),s&&e(Nl),s&&e(X),j(Cs),s&&e(Wl),s&&e(Ss),s&&e(Ll),s&&e(ts),s&&e(Xl),j(vs,s),s&&e(Ql),s&&e(Ys),s&&e(Sl),s&&e(xs),s&&e(Yl),s&&e(Fs),s&&e(xl),j(Zs,s),s&&e(Fl),s&&e(zs),s&&e(zl),s&&e(Hs),s&&e(Hl),j(As,s)}}}const ro={local:"debugging",sections:[{local:"multigpu-network-issues-debug",title:"Multi-GPU Network Issues Debug"},{local:"underflow-and-overflow-detection",sections:[{local:"specific-batch-absolute-min-and-max-value-tracing",title:"Specific batch absolute min and max value tracing"}],title:"Underflow and Overflow Detection"}],title:"Debugging"};function oo(S){return eo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ho extends Pr{constructor(h){super();Kr(this,h,oo,to,qr,{})}}export{ho as default,ro as metadata};
