import{S as Rr,i as Ir,s as Nr,e as a,k as p,w as v,t as i,M as Vr,c as l,d as t,m,a as s,x as w,h as f,b as c,G as r,g as n,y as T,q as M,o as J,B as _,v as Fr}from"../chunks/vendor-hf-doc-builder.js";import{T as Ar}from"../chunks/Tip-hf-doc-builder.js";import{I as We}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as je}from"../chunks/CodeBlock-hf-doc-builder.js";function Pr(ie){let d,y,h,u,Z;return{c(){d=a("p"),y=i("Note: A multi GPU setup can use the majority of the strategies described in the "),h=a("a"),u=i("single GPU section"),Z=i(". You must be aware of simple techniques, though, that can be used for a better usage."),this.h()},l(b){d=l(b,"P",{});var B=s(d);y=f(B,"Note: A multi GPU setup can use the majority of the strategies described in the "),h=l(B,"A",{href:!0});var j=s(h);u=f(j,"single GPU section"),j.forEach(t),Z=f(B,". You must be aware of simple techniques, though, that can be used for a better usage."),B.forEach(t),this.h()},h(){c(h,"href","./perf_infer_gpu_one")},m(b,B){n(b,d,B),r(d,y),r(d,h),r(h,u),r(d,Z)},d(b){b&&t(d)}}}function Xr(ie){let d,y;return{c(){d=a("p"),y=i("Flash Attention can only be used for models using fp16 or bf16 dtype. Make sure to cast your model to the appropriate dtype before using BetterTransformer.")},l(h){d=l(h,"P",{});var u=s(d);y=f(u,"Flash Attention can only be used for models using fp16 or bf16 dtype. Make sure to cast your model to the appropriate dtype before using BetterTransformer."),u.forEach(t)},m(h,u){n(h,d,u),r(d,y)},d(h){h&&t(d)}}}function Cr(ie){let d,y,h,u,Z,b,B,j,mt,Ae,fe,dt,Re,A,Ie,U,R,ve,z,ht,we,ut,Ne,H,Q,bt,yt,Ve,ce,vt,Fe,I,Pe,g,N,Te,S,wt,Me,Tt,Xe,V,Mt,F,Je,Jt,_t,Bt,Ce,pe,$t,xe,Y,ze,$,Et,q,kt,Zt,L,_e,Ut,gt,He,D,Qe,me,Gt,Se,K,Ye,de,Wt,qe,O,Le,P,jt,ee,At,Rt,De,G,X,Be,te,It,$e,Nt,Ke,C,Vt,re,Ee,Ft,Pt,Oe,E,Xt,ke,Ct,xt,Ze,zt,Ht,et,k,Qt,oe,St,Yt,ae,qt,Lt,tt,W,x,Ue,le,Dt,ge,Kt,rt,he,Ot,ot,se,at;return b=new We({}),A=new Ar({props:{$$slots:{default:[Pr]},$$scope:{ctx:ie}}}),z=new We({}),I=new Ar({props:{$$slots:{default:[Xr]},$$scope:{ctx:ie}}}),S=new We({}),Y=new je({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMzUwbSUyMiklMEElMjMlMjBjb252ZXJ0JTIwdGhlJTIwbW9kZWwlMjB0byUyMEJldHRlclRyYW5zZm9ybWVyJTBBbW9kZWwudG9fYmV0dGVydHJhbnNmb3JtZXIoKSUwQSUwQSUyMyUyMFVzZSUyMGl0JTIwZm9yJTIwdHJhaW5pbmclMjBvciUyMGluZmVyZW5jZQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>)
<span class="hljs-comment"># convert the model to BetterTransformer</span>
model.to_bettertransformer()

<span class="hljs-comment"># Use it for training or inference</span>`}}),D=new je({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMzUwbSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyKS50byglMjJjdWRhJTIyKSUwQSUyMyUyMGNvbnZlcnQlMjB0aGUlMjBtb2RlbCUyMHRvJTIwQmV0dGVyVHJhbnNmb3JtZXIlMEFtb2RlbC50b19iZXR0ZXJ0cmFuc2Zvcm1lcigpJTBBJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMkhlbGxvJTIwbXklMjBkb2clMjBpcyUyMGN1dGUlMjBhbmQlMjIlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTJCJTIwd2l0aCUyMHRvcmNoLmJhY2tlbmRzLmN1ZGEuc2RwX2tlcm5lbChlbmFibGVfZmxhc2glM0RUcnVlJTJDJTIwZW5hYmxlX21hdGglM0RGYWxzZSUyQyUyMGVuYWJsZV9tZW1fZWZmaWNpZW50JTNERmFsc2UpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(&quot;facebook/opt-350m&quot;)
model = AutoModelForCausalLM.from_pretrained(&quot;facebook/opt-350m&quot;).to(&quot;cuda&quot;)
# convert the model to BetterTransformer
model.to_bettertransformer()

input_text = &quot;Hello my dog is cute and&quot;
inputs = tokenizer(input_text, return_tensors=&quot;pt&quot;).to(&quot;cuda&quot;)

<span class="hljs-addition">+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):</span>
    outputs = model.generate(**inputs)

print(tokenizer.decode(outputs[0], skip_special_tokens=True))`}}),K=new je({props:{code:"UnVudGltZUVycm9yJTNBJTIwTm8lMjBhdmFpbGFibGUlMjBrZXJuZWwuJTIwJTIwQWJvcnRpbmclMjBleGVjdXRpb24u",highlighted:"RuntimeError: No available kernel.  Aborting execution."}}),O=new je({props:{code:"cGlwMyUyMGluc3RhbGwlMjAtVSUyMC0tcHJlJTIwdG9yY2glMjB0b3JjaHZpc2lvbiUyMHRvcmNoYXVkaW8lMjAtLWluZGV4LXVybCUyMGh0dHBzJTNBJTJGJTJGZG93bmxvYWQucHl0b3JjaC5vcmclMkZ3aGwlMkZuaWdodGx5JTJGY3UxMTg=",highlighted:"pip3 install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118"}}),te=new We({}),le=new We({}),se=new je({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMEEpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMzUwbSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcpJTBBJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMkhlbGxvJTIwbXklMjBkb2clMjBpcyUyMGN1dGUlMjBhbmQlMjIlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBd2l0aCUyMHRvcmNoLmJhY2tlbmRzLmN1ZGEuc2RwX2tlcm5lbChlbmFibGVfZmxhc2glM0RUcnVlJTJDJTIwZW5hYmxlX21hdGglM0RGYWxzZSUyQyUyMGVuYWJsZV9tZW1fZWZmaWNpZW50JTNERmFsc2UpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_compute_dtype=torch.float16
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, quantization_config=quantization_config)

input_text = <span class="hljs-string">&quot;Hello my dog is cute and&quot;</span>
inputs = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-keyword">with</span> torch.backends.cuda.sdp_kernel(enable_flash=<span class="hljs-literal">True</span>, enable_math=<span class="hljs-literal">False</span>, enable_mem_efficient=<span class="hljs-literal">False</span>):
    outputs = model.generate(**inputs)

<span class="hljs-built_in">print</span>(tokenizer.decode(outputs[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`}}),{c(){d=a("meta"),y=p(),h=a("h1"),u=a("a"),Z=a("span"),v(b.$$.fragment),B=p(),j=a("span"),mt=i("Efficient Inference on a Multiple GPUs"),Ae=p(),fe=a("p"),dt=i("This document contains information on how to efficiently infer on a multiple GPUs."),Re=p(),v(A.$$.fragment),Ie=p(),U=a("h2"),R=a("a"),ve=a("span"),v(z.$$.fragment),ht=p(),we=a("span"),ut=i("BetterTransformer"),Ne=p(),H=a("p"),Q=a("a"),bt=i("BetterTransformer"),yt=i(" converts \u{1F917} Transformers models to use the PyTorch-native fastpath execution, which calls optimized kernels like Flash Attention under the hood."),Ve=p(),ce=a("p"),vt=i("BetterTransformer is also supported for faster inference on single and multi-GPU for text, image, and audio models."),Fe=p(),v(I.$$.fragment),Pe=p(),g=a("h3"),N=a("a"),Te=a("span"),v(S.$$.fragment),wt=p(),Me=a("span"),Tt=i("Decoder models"),Xe=p(),V=a("p"),Mt=i("For text models, especially decoder-based models (GPT, T5, Llama, etc.), the BetterTransformer API converts all attention operations to use the "),F=a("a"),Je=a("code"),Jt=i("torch.nn.functional.scaled_dot_product_attention"),_t=i(" operator"),Bt=i(" (SDPA) that is only available in PyTorch 2.0 and onwards."),Ce=p(),pe=a("p"),$t=i("To convert a model to BetterTransformer:"),xe=p(),v(Y.$$.fragment),ze=p(),$=a("p"),Et=i("SDPA can also call "),q=a("a"),kt=i("Flash Attention"),Zt=i(" kernels under the hood. To enable Flash Attention or to check that it is available in a given setting (hardware, problem size), use "),L=a("a"),_e=a("code"),Ut=i("torch.backends.cuda.sdp_kernel"),gt=i(" as a context manager:"),He=p(),v(D.$$.fragment),Qe=p(),me=a("p"),Gt=i("If you see a bug with a traceback saying"),Se=p(),v(K.$$.fragment),Ye=p(),de=a("p"),Wt=i("try using the PyTorch nightly version, which may have a broader coverage for Flash Attention:"),qe=p(),v(O.$$.fragment),Le=p(),P=a("p"),jt=i("Have a look at this "),ee=a("a"),At=i("blog post"),Rt=i(" to learn more about what is possible with the BetterTransformer + SDPA API."),De=p(),G=a("h3"),X=a("a"),Be=a("span"),v(te.$$.fragment),It=p(),$e=a("span"),Nt=i("Encoder models"),Ke=p(),C=a("p"),Vt=i("For encoder models during inference, BetterTransformer dispatches the forward call of encoder layers to an equivalent of "),re=a("a"),Ee=a("code"),Ft=i("torch.nn.TransformerEncoderLayer"),Pt=i(" that will execute the fastpath implementation of the encoder layers."),Oe=p(),E=a("p"),Xt=i("Because "),ke=a("code"),Ct=i("torch.nn.TransformerEncoderLayer"),xt=i(" fastpath does not support training, it is dispatched to "),Ze=a("code"),zt=i("torch.nn.functional.scaled_dot_product_attention"),Ht=i(" instead, which does not leverage nested tensors but can use Flash Attention or Memory-Efficient Attention fused kernels."),et=p(),k=a("p"),Qt=i("More details about BetterTransformer performance can be found in this "),oe=a("a"),St=i("blog post"),Yt=i(", and you can learn more about BetterTransformer for encoder models in this "),ae=a("a"),qt=i("blog"),Lt=i("."),tt=p(),W=a("h2"),x=a("a"),Ue=a("span"),v(le.$$.fragment),Dt=p(),ge=a("span"),Kt=i("Advanced usage: mixing FP4 (or Int8) and BetterTransformer"),rt=p(),he=a("p"),Ot=i("You can combine the different methods described above to get the best performance for your model. For example, you can use BetterTransformer with FP4 mixed-precision inference + flash attention:"),ot=p(),v(se.$$.fragment),this.h()},l(e){const o=Vr('[data-svelte="svelte-1phssyn"]',document.head);d=l(o,"META",{name:!0,content:!0}),o.forEach(t),y=m(e),h=l(e,"H1",{class:!0});var ne=s(h);u=l(ne,"A",{id:!0,class:!0,href:!0});var Ge=s(u);Z=l(Ge,"SPAN",{});var rr=s(Z);w(b.$$.fragment,rr),rr.forEach(t),Ge.forEach(t),B=m(ne),j=l(ne,"SPAN",{});var or=s(j);mt=f(or,"Efficient Inference on a Multiple GPUs"),or.forEach(t),ne.forEach(t),Ae=m(e),fe=l(e,"P",{});var ar=s(fe);dt=f(ar,"This document contains information on how to efficiently infer on a multiple GPUs."),ar.forEach(t),Re=m(e),w(A.$$.fragment,e),Ie=m(e),U=l(e,"H2",{class:!0});var lt=s(U);R=l(lt,"A",{id:!0,class:!0,href:!0});var lr=s(R);ve=l(lr,"SPAN",{});var sr=s(ve);w(z.$$.fragment,sr),sr.forEach(t),lr.forEach(t),ht=m(lt),we=l(lt,"SPAN",{});var nr=s(we);ut=f(nr,"BetterTransformer"),nr.forEach(t),lt.forEach(t),Ne=m(e),H=l(e,"P",{});var er=s(H);Q=l(er,"A",{href:!0,rel:!0});var ir=s(Q);bt=f(ir,"BetterTransformer"),ir.forEach(t),yt=f(er," converts \u{1F917} Transformers models to use the PyTorch-native fastpath execution, which calls optimized kernels like Flash Attention under the hood."),er.forEach(t),Ve=m(e),ce=l(e,"P",{});var fr=s(ce);vt=f(fr,"BetterTransformer is also supported for faster inference on single and multi-GPU for text, image, and audio models."),fr.forEach(t),Fe=m(e),w(I.$$.fragment,e),Pe=m(e),g=l(e,"H3",{class:!0});var st=s(g);N=l(st,"A",{id:!0,class:!0,href:!0});var cr=s(N);Te=l(cr,"SPAN",{});var pr=s(Te);w(S.$$.fragment,pr),pr.forEach(t),cr.forEach(t),wt=m(st),Me=l(st,"SPAN",{});var mr=s(Me);Tt=f(mr,"Decoder models"),mr.forEach(t),st.forEach(t),Xe=m(e),V=l(e,"P",{});var nt=s(V);Mt=f(nt,"For text models, especially decoder-based models (GPT, T5, Llama, etc.), the BetterTransformer API converts all attention operations to use the "),F=l(nt,"A",{href:!0,rel:!0});var tr=s(F);Je=l(tr,"CODE",{});var dr=s(Je);Jt=f(dr,"torch.nn.functional.scaled_dot_product_attention"),dr.forEach(t),_t=f(tr," operator"),tr.forEach(t),Bt=f(nt," (SDPA) that is only available in PyTorch 2.0 and onwards."),nt.forEach(t),Ce=m(e),pe=l(e,"P",{});var hr=s(pe);$t=f(hr,"To convert a model to BetterTransformer:"),hr.forEach(t),xe=m(e),w(Y.$$.fragment,e),ze=m(e),$=l(e,"P",{});var ue=s($);Et=f(ue,"SDPA can also call "),q=l(ue,"A",{href:!0,rel:!0});var ur=s(q);kt=f(ur,"Flash Attention"),ur.forEach(t),Zt=f(ue," kernels under the hood. To enable Flash Attention or to check that it is available in a given setting (hardware, problem size), use "),L=l(ue,"A",{href:!0,rel:!0});var br=s(L);_e=l(br,"CODE",{});var yr=s(_e);Ut=f(yr,"torch.backends.cuda.sdp_kernel"),yr.forEach(t),br.forEach(t),gt=f(ue," as a context manager:"),ue.forEach(t),He=m(e),w(D.$$.fragment,e),Qe=m(e),me=l(e,"P",{});var vr=s(me);Gt=f(vr,"If you see a bug with a traceback saying"),vr.forEach(t),Se=m(e),w(K.$$.fragment,e),Ye=m(e),de=l(e,"P",{});var wr=s(de);Wt=f(wr,"try using the PyTorch nightly version, which may have a broader coverage for Flash Attention:"),wr.forEach(t),qe=m(e),w(O.$$.fragment,e),Le=m(e),P=l(e,"P",{});var it=s(P);jt=f(it,"Have a look at this "),ee=l(it,"A",{href:!0,rel:!0});var Tr=s(ee);At=f(Tr,"blog post"),Tr.forEach(t),Rt=f(it," to learn more about what is possible with the BetterTransformer + SDPA API."),it.forEach(t),De=m(e),G=l(e,"H3",{class:!0});var ft=s(G);X=l(ft,"A",{id:!0,class:!0,href:!0});var Mr=s(X);Be=l(Mr,"SPAN",{});var Jr=s(Be);w(te.$$.fragment,Jr),Jr.forEach(t),Mr.forEach(t),It=m(ft),$e=l(ft,"SPAN",{});var _r=s($e);Nt=f(_r,"Encoder models"),_r.forEach(t),ft.forEach(t),Ke=m(e),C=l(e,"P",{});var ct=s(C);Vt=f(ct,"For encoder models during inference, BetterTransformer dispatches the forward call of encoder layers to an equivalent of "),re=l(ct,"A",{href:!0,rel:!0});var Br=s(re);Ee=l(Br,"CODE",{});var $r=s(Ee);Ft=f($r,"torch.nn.TransformerEncoderLayer"),$r.forEach(t),Br.forEach(t),Pt=f(ct," that will execute the fastpath implementation of the encoder layers."),ct.forEach(t),Oe=m(e),E=l(e,"P",{});var be=s(E);Xt=f(be,"Because "),ke=l(be,"CODE",{});var Er=s(ke);Ct=f(Er,"torch.nn.TransformerEncoderLayer"),Er.forEach(t),xt=f(be," fastpath does not support training, it is dispatched to "),Ze=l(be,"CODE",{});var kr=s(Ze);zt=f(kr,"torch.nn.functional.scaled_dot_product_attention"),kr.forEach(t),Ht=f(be," instead, which does not leverage nested tensors but can use Flash Attention or Memory-Efficient Attention fused kernels."),be.forEach(t),et=m(e),k=l(e,"P",{});var ye=s(k);Qt=f(ye,"More details about BetterTransformer performance can be found in this "),oe=l(ye,"A",{href:!0,rel:!0});var Zr=s(oe);St=f(Zr,"blog post"),Zr.forEach(t),Yt=f(ye,", and you can learn more about BetterTransformer for encoder models in this "),ae=l(ye,"A",{href:!0,rel:!0});var Ur=s(ae);qt=f(Ur,"blog"),Ur.forEach(t),Lt=f(ye,"."),ye.forEach(t),tt=m(e),W=l(e,"H2",{class:!0});var pt=s(W);x=l(pt,"A",{id:!0,class:!0,href:!0});var gr=s(x);Ue=l(gr,"SPAN",{});var Gr=s(Ue);w(le.$$.fragment,Gr),Gr.forEach(t),gr.forEach(t),Dt=m(pt),ge=l(pt,"SPAN",{});var Wr=s(ge);Kt=f(Wr,"Advanced usage: mixing FP4 (or Int8) and BetterTransformer"),Wr.forEach(t),pt.forEach(t),rt=m(e),he=l(e,"P",{});var jr=s(he);Ot=f(jr,"You can combine the different methods described above to get the best performance for your model. For example, you can use BetterTransformer with FP4 mixed-precision inference + flash attention:"),jr.forEach(t),ot=m(e),w(se.$$.fragment,e),this.h()},h(){c(d,"name","hf:doc:metadata"),c(d,"content",JSON.stringify(xr)),c(u,"id","efficient-inference-on-a-multiple-gpus"),c(u,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(u,"href","#efficient-inference-on-a-multiple-gpus"),c(h,"class","relative group"),c(R,"id","bettertransformer"),c(R,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(R,"href","#bettertransformer"),c(U,"class","relative group"),c(Q,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),c(Q,"rel","nofollow"),c(N,"id","decoder-models"),c(N,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(N,"href","#decoder-models"),c(g,"class","relative group"),c(F,"href","https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention"),c(F,"rel","nofollow"),c(q,"href","https://arxiv.org/abs/2205.14135"),c(q,"rel","nofollow"),c(L,"href","https://pytorch.org/docs/master/backends.html#torch.backends.cuda.sdp_kernel"),c(L,"rel","nofollow"),c(ee,"href","https://pytorch.org/blog/out-of-the-box-acceleration/"),c(ee,"rel","nofollow"),c(X,"id","encoder-models"),c(X,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(X,"href","#encoder-models"),c(G,"class","relative group"),c(re,"href","https://pytorch.org/docs/stable/generated/torch.nn.TransformerEncoderLayer.html"),c(re,"rel","nofollow"),c(oe,"href","https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2"),c(oe,"rel","nofollow"),c(ae,"href","https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/"),c(ae,"rel","nofollow"),c(x,"id","advanced-usage-mixing-fp4-or-int8-and-bettertransformer"),c(x,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(x,"href","#advanced-usage-mixing-fp4-or-int8-and-bettertransformer"),c(W,"class","relative group")},m(e,o){r(document.head,d),n(e,y,o),n(e,h,o),r(h,u),r(u,Z),T(b,Z,null),r(h,B),r(h,j),r(j,mt),n(e,Ae,o),n(e,fe,o),r(fe,dt),n(e,Re,o),T(A,e,o),n(e,Ie,o),n(e,U,o),r(U,R),r(R,ve),T(z,ve,null),r(U,ht),r(U,we),r(we,ut),n(e,Ne,o),n(e,H,o),r(H,Q),r(Q,bt),r(H,yt),n(e,Ve,o),n(e,ce,o),r(ce,vt),n(e,Fe,o),T(I,e,o),n(e,Pe,o),n(e,g,o),r(g,N),r(N,Te),T(S,Te,null),r(g,wt),r(g,Me),r(Me,Tt),n(e,Xe,o),n(e,V,o),r(V,Mt),r(V,F),r(F,Je),r(Je,Jt),r(F,_t),r(V,Bt),n(e,Ce,o),n(e,pe,o),r(pe,$t),n(e,xe,o),T(Y,e,o),n(e,ze,o),n(e,$,o),r($,Et),r($,q),r(q,kt),r($,Zt),r($,L),r(L,_e),r(_e,Ut),r($,gt),n(e,He,o),T(D,e,o),n(e,Qe,o),n(e,me,o),r(me,Gt),n(e,Se,o),T(K,e,o),n(e,Ye,o),n(e,de,o),r(de,Wt),n(e,qe,o),T(O,e,o),n(e,Le,o),n(e,P,o),r(P,jt),r(P,ee),r(ee,At),r(P,Rt),n(e,De,o),n(e,G,o),r(G,X),r(X,Be),T(te,Be,null),r(G,It),r(G,$e),r($e,Nt),n(e,Ke,o),n(e,C,o),r(C,Vt),r(C,re),r(re,Ee),r(Ee,Ft),r(C,Pt),n(e,Oe,o),n(e,E,o),r(E,Xt),r(E,ke),r(ke,Ct),r(E,xt),r(E,Ze),r(Ze,zt),r(E,Ht),n(e,et,o),n(e,k,o),r(k,Qt),r(k,oe),r(oe,St),r(k,Yt),r(k,ae),r(ae,qt),r(k,Lt),n(e,tt,o),n(e,W,o),r(W,x),r(x,Ue),T(le,Ue,null),r(W,Dt),r(W,ge),r(ge,Kt),n(e,rt,o),n(e,he,o),r(he,Ot),n(e,ot,o),T(se,e,o),at=!0},p(e,[o]){const ne={};o&2&&(ne.$$scope={dirty:o,ctx:e}),A.$set(ne);const Ge={};o&2&&(Ge.$$scope={dirty:o,ctx:e}),I.$set(Ge)},i(e){at||(M(b.$$.fragment,e),M(A.$$.fragment,e),M(z.$$.fragment,e),M(I.$$.fragment,e),M(S.$$.fragment,e),M(Y.$$.fragment,e),M(D.$$.fragment,e),M(K.$$.fragment,e),M(O.$$.fragment,e),M(te.$$.fragment,e),M(le.$$.fragment,e),M(se.$$.fragment,e),at=!0)},o(e){J(b.$$.fragment,e),J(A.$$.fragment,e),J(z.$$.fragment,e),J(I.$$.fragment,e),J(S.$$.fragment,e),J(Y.$$.fragment,e),J(D.$$.fragment,e),J(K.$$.fragment,e),J(O.$$.fragment,e),J(te.$$.fragment,e),J(le.$$.fragment,e),J(se.$$.fragment,e),at=!1},d(e){t(d),e&&t(y),e&&t(h),_(b),e&&t(Ae),e&&t(fe),e&&t(Re),_(A,e),e&&t(Ie),e&&t(U),_(z),e&&t(Ne),e&&t(H),e&&t(Ve),e&&t(ce),e&&t(Fe),_(I,e),e&&t(Pe),e&&t(g),_(S),e&&t(Xe),e&&t(V),e&&t(Ce),e&&t(pe),e&&t(xe),_(Y,e),e&&t(ze),e&&t($),e&&t(He),_(D,e),e&&t(Qe),e&&t(me),e&&t(Se),_(K,e),e&&t(Ye),e&&t(de),e&&t(qe),_(O,e),e&&t(Le),e&&t(P),e&&t(De),e&&t(G),_(te),e&&t(Ke),e&&t(C),e&&t(Oe),e&&t(E),e&&t(et),e&&t(k),e&&t(tt),e&&t(W),_(le),e&&t(rt),e&&t(he),e&&t(ot),_(se,e)}}}const xr={local:"efficient-inference-on-a-multiple-gpus",sections:[{local:"bettertransformer",sections:[{local:"decoder-models",title:"Decoder models"},{local:"encoder-models",title:"Encoder models"}],title:"BetterTransformer"},{local:"advanced-usage-mixing-fp4-or-int8-and-bettertransformer",title:"Advanced usage: mixing FP4 (or Int8) and BetterTransformer"}],title:"Efficient Inference on a Multiple GPUs"};function zr(ie){return Fr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class qr extends Rr{constructor(d){super();Ir(this,d,zr,Cr,Nr,{})}}export{qr as default,xr as metadata};
