import{s as O,n as ee,o as te}from"../chunks/scheduler.defa9a21.js";import{S as ie,i as ae,g as o,s as n,r as v,A as ne,h as r,f as i,c as s,j as F,u as P,x as T,k as K,y as se,a,v as U,d as x,t as C,w as M}from"../chunks/index.fe795e71.js";import{C as le}from"../chunks/CodeBlock.204b6c34.js";import{H as X,E as oe}from"../chunks/getInferenceSnippets.2234a8dd.js";function re(Z){let l,k,_,L,p,H,m,q='A <a href="https://cloud.google.com/tpu/docs/intro-to-tpu" rel="nofollow">TPU (Tensor Processing Unit)</a> is a type of hardware specifically designed for training models efficiently. Accelerate supports TPU training, but there are a few things you should be aware of, namely graph compilation. This tutorial briefly discusses compilation, and for more details, take a look at the <a href="../concept_guides/training_tpu">Training on TPUs with Accelerate</a> guide.',A,c,E,h,Y="A TPU creates a graph of all the operations in the training step such as the forward pass, backward pass and optimizer step. This is why the first training step always takes a while because building and compiling this graph takes time. But once compilation is complete, it is cached and all subsequent steps are much faster.",R,f,j="The key is to avoid compiling your code again or else training is super slow. This means all your operations must be exactly the same:",S,u,Q="<li>all tensors in your batches must have the same length (for example, no dynamic padding for NLP tasks)</li> <li>your code must be static (for example, no layers with for loops that have different lengths depending on the input such as a LSTM)</li>",z,d,B,g,D='A common language model design is to tie the weights of the embedding and softmax layers. However, moving the model to a TPU (either yourself or passing it to the <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.prepare">prepare()</a> method) breaks the weight tying and you’ll need to retie the weights.',G,$,I='To add special behavior (like weight tying) in your script for TPUs, set <code>distributed_type</code> to <code>DistributedType.TPU</code> first. Then you can use the <a href="https://huggingface.co/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.tie_weights" rel="nofollow">tie_weights</a> method to tie the weights.',J,y,N,w,V,b,W;return p=new X({props:{title:"TPU training",local:"tpu-training",headingTag:"h1"}}),c=new X({props:{title:"Compilation",local:"compilation",headingTag:"h2"}}),d=new X({props:{title:"Weight tying",local:"weight-tying",headingTag:"h2"}}),y=new le({props:{code:"aWYlMjBhY2NlbGVyYXRvci5kaXN0cmlidXRlZF90eXBlJTIwJTNEJTNEJTIwRGlzdHJpYnV0ZWRUeXBlLlRQVSUzQSUwQSUyMCUyMCUyMCUyMG1vZGVsLnRpZV93ZWlnaHRzKCk=",highlighted:`<span class="hljs-keyword">if</span> accelerator.distributed_type == DistributedType.TPU:
    model.tie_weights()`,wrap:!1}}),w=new oe({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/basic_tutorials/tpu.md"}}),{c(){l=o("meta"),k=n(),_=o("p"),L=n(),v(p.$$.fragment),H=n(),m=o("p"),m.innerHTML=q,A=n(),v(c.$$.fragment),E=n(),h=o("p"),h.textContent=Y,R=n(),f=o("p"),f.textContent=j,S=n(),u=o("ul"),u.innerHTML=Q,z=n(),v(d.$$.fragment),B=n(),g=o("p"),g.innerHTML=D,G=n(),$=o("p"),$.innerHTML=I,J=n(),v(y.$$.fragment),N=n(),v(w.$$.fragment),V=n(),b=o("p"),this.h()},l(e){const t=ne("svelte-u9bgzb",document.head);l=r(t,"META",{name:!0,content:!0}),t.forEach(i),k=s(e),_=r(e,"P",{}),F(_).forEach(i),L=s(e),P(p.$$.fragment,e),H=s(e),m=r(e,"P",{"data-svelte-h":!0}),T(m)!=="svelte-1efoa6g"&&(m.innerHTML=q),A=s(e),P(c.$$.fragment,e),E=s(e),h=r(e,"P",{"data-svelte-h":!0}),T(h)!=="svelte-76he8q"&&(h.textContent=Y),R=s(e),f=r(e,"P",{"data-svelte-h":!0}),T(f)!=="svelte-1ga0n4t"&&(f.textContent=j),S=s(e),u=r(e,"UL",{"data-svelte-h":!0}),T(u)!=="svelte-fxztf"&&(u.innerHTML=Q),z=s(e),P(d.$$.fragment,e),B=s(e),g=r(e,"P",{"data-svelte-h":!0}),T(g)!=="svelte-1u75nqb"&&(g.innerHTML=D),G=s(e),$=r(e,"P",{"data-svelte-h":!0}),T($)!=="svelte-w62ir7"&&($.innerHTML=I),J=s(e),P(y.$$.fragment,e),N=s(e),P(w.$$.fragment,e),V=s(e),b=r(e,"P",{}),F(b).forEach(i),this.h()},h(){K(l,"name","hf:doc:metadata"),K(l,"content",pe)},m(e,t){se(document.head,l),a(e,k,t),a(e,_,t),a(e,L,t),U(p,e,t),a(e,H,t),a(e,m,t),a(e,A,t),U(c,e,t),a(e,E,t),a(e,h,t),a(e,R,t),a(e,f,t),a(e,S,t),a(e,u,t),a(e,z,t),U(d,e,t),a(e,B,t),a(e,g,t),a(e,G,t),a(e,$,t),a(e,J,t),U(y,e,t),a(e,N,t),U(w,e,t),a(e,V,t),a(e,b,t),W=!0},p:ee,i(e){W||(x(p.$$.fragment,e),x(c.$$.fragment,e),x(d.$$.fragment,e),x(y.$$.fragment,e),x(w.$$.fragment,e),W=!0)},o(e){C(p.$$.fragment,e),C(c.$$.fragment,e),C(d.$$.fragment,e),C(y.$$.fragment,e),C(w.$$.fragment,e),W=!1},d(e){e&&(i(k),i(_),i(L),i(H),i(m),i(A),i(E),i(h),i(R),i(f),i(S),i(u),i(z),i(B),i(g),i(G),i($),i(J),i(N),i(V),i(b)),i(l),M(p,e),M(c,e),M(d,e),M(y,e),M(w,e)}}}const pe='{"title":"TPU training","local":"tpu-training","sections":[{"title":"Compilation","local":"compilation","sections":[],"depth":2},{"title":"Weight tying","local":"weight-tying","sections":[],"depth":2}],"depth":1}';function me(Z){return te(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class de extends ie{constructor(l){super(),ae(this,l,me,re,O,{})}}export{de as component};
