import{S as fm,i as mm,s as cm,e as l,k as m,w as u,t as i,M as um,c as r,d as t,m as c,a,x as d,h as n,b as f,N as $a,G as o,g as p,y as h,q as b,o as v,B as y,v as dm}from"../chunks/vendor-hf-doc-builder.js";import{T as Xn}from"../chunks/Tip-hf-doc-builder.js";import{I as $}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as J}from"../chunks/CodeBlock-hf-doc-builder.js";function hm(le){let w,M;return{c(){w=l("p"),M=i("Flash Attention can only be used for models using fp16 or bf16 dtype. Make sure to cast your model to the appropriate dtype before using BetterTransformer.")},l(_){w=r(_,"P",{});var g=a(w);M=n(g,"Flash Attention can only be used for models using fp16 or bf16 dtype. Make sure to cast your model to the appropriate dtype before using BetterTransformer."),g.forEach(t)},m(_,g){p(_,w,g),o(w,M)},d(_){_&&t(w)}}}function bm(le){let w,M;return{c(){w=l("p"),M=i("Note that this feature can also be used in a multi GPU setup.")},l(_){w=r(_,"P",{});var g=a(w);M=n(g,"Note that this feature can also be used in a multi GPU setup."),g.forEach(t)},m(_,g){p(_,w,g),o(w,M)},d(_){_&&t(w)}}}function vm(le){let w,M;return{c(){w=l("p"),M=i("Note that this feature can also be used in a multi GPU setup.")},l(_){w=r(_,"P",{});var g=a(w);M=n(g,"Note that this feature can also be used in a multi GPU setup."),g.forEach(t)},m(_,g){p(_,w,g),o(w,M)},d(_){_&&t(w)}}}function ym(le){let w,M,_,g,yo,Pe,Ja,wo,Ua,Al,G,ka,$t,Ga,Za,Jt,Wa,Ba,Nl,V,re,_o,Ie,Pa,go,Ia,Vl,xe,je,xa,ja,Xl,Ut,Ra,ql,ae,zl,X,se,Mo,Re,Ca,Eo,Fa,Yl,Z,Aa,Ce,To,Na,Va,Fe,Xa,qa,Sl,W,za,Ae,Ya,Sa,Ne,Ha,Qa,Hl,B,Da,Ve,$o,La,Oa,kt,Ka,es,Ql,Xe,Dl,ie,ts,Gt,os,ls,Ll,qe,Ol,P,rs,ze,as,ss,Jo,is,ns,Kl,q,ne,Uo,Ye,ps,ko,fs,er,pe,ms,fe,Go,cs,us,ds,tr,Zt,hs,or,Se,lr,I,bs,He,vs,ys,Qe,Zo,ws,_s,rr,De,ar,Wt,gs,sr,Le,ir,Bt,Ms,nr,Oe,pr,Pt,Es,fr,x,Ts,Ke,$s,Js,Wo,Us,ks,mr,z,me,Bo,et,Gs,It,Po,Zs,Ws,cr,ce,Bs,Io,Ps,Is,ur,ue,dr,Y,de,xo,tt,xs,jo,js,hr,j,Ro,he,Rs,Co,Cs,Fs,Fo,As,Ns,Ao,be,Vs,No,Xs,qs,Vo,zs,Ys,Xo,ve,Ss,qo,Hs,Qs,zo,Ds,br,S,ye,Yo,ot,Ls,So,Os,vr,xt,Ks,yr,lt,wr,R,ei,Ho,ti,oi,Qo,li,ri,_r,H,we,Do,rt,ai,Lo,si,gr,jt,ii,Mr,at,Er,C,ni,Oo,pi,fi,Ko,mi,ci,Tr,st,$r,Rt,ui,Jr,Q,_e,el,it,di,tl,hi,Ur,ge,bi,Ct,vi,yi,kr,D,Me,ol,nt,wi,Ft,ll,_i,gi,Gr,Ee,Zr,E,Mi,pt,rl,Ei,Ti,al,$i,Ji,sl,Ui,ki,il,Gi,Zi,nl,Wi,Bi,Wr,At,Nt,qn,Br,F,Pi,ft,Ii,xi,mt,ji,Ri,Pr,Vt,Xt,zn,Ir,Te,Ci,qt,Fi,Ai,xr,L,$e,pl,ct,Ni,fl,Vi,jr,A,O,Xi,ml,qi,zi,cl,Yi,Si,Hi,Je,Qi,ul,Di,Li,dl,Oi,Ki,Ue,en,hl,tn,on,bl,ln,Rr,K,ke,vl,ut,rn,yl,an,Cr,zt,sn,Fr,dt,Ar,Yt,nn,Nr,Ge,T,pn,wl,fn,mn,_l,cn,un,gl,dn,hn,Ml,bn,vn,El,yn,wn,_n,Tl,gn,Vr,St,Mn,Xr,ht,qr,ee,Ze,$l,bt,En,Jl,Tn,zr,Ht,$n,Yr,vt,Sr,N,Jn,Ul,Un,kn,kl,Gn,Zn,Hr,yt,Qr,Qt,Wn,Dr,te,We,Gl,wt,Bn,Zl,Pn,Lr,Dt,In,Or,Lt,_t,Ot,Yn,Kr,Kt,xn,ea,eo,gt,to,Sn,ta,oe,Be,Wl,Mt,jn,Bl,Rn,oa,oo,Cn,la,Et,ra;return Pe=new $({}),Ie=new $({}),ae=new Xn({props:{$$slots:{default:[hm]},$$scope:{ctx:le}}}),Re=new $({}),Xe=new J({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC50b19iZXR0ZXJ0cmFuc2Zvcm1lcigp",highlighted:"model = model.to_bettertransformer()"}}),qe=new J({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC5yZXZlcnNlX2JldHRlcnRyYW5zZm9ybWVyKCklMEFtb2RlbC5zYXZlX3ByZXRyYWluZWQoJTIyc2F2ZWRfbW9kZWwlMjIp",highlighted:`model = model.reverse_bettertransformer()
model.save_pretrained(<span class="hljs-string">&quot;saved_model&quot;</span>)`}}),Ye=new $({}),Se=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMzUwbSUyMiklMEElMjMlMjBjb252ZXJ0JTIwdGhlJTIwbW9kZWwlMjB0byUyMEJldHRlclRyYW5zZm9ybWVyJTBBbW9kZWwudG9fYmV0dGVydHJhbnNmb3JtZXIoKSUwQSUwQSUyMyUyMFVzZSUyMGl0JTIwZm9yJTIwdHJhaW5pbmclMjBvciUyMGluZmVyZW5jZQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>)
<span class="hljs-comment"># convert the model to BetterTransformer</span>
model.to_bettertransformer()

<span class="hljs-comment"># Use it for training or inference</span>`}}),De=new J({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMzUwbSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KS50byglMjJjdWRhJTIyKSUwQSUyMyUyMGNvbnZlcnQlMjB0aGUlMjBtb2RlbCUyMHRvJTIwQmV0dGVyVHJhbnNmb3JtZXIlMEFtb2RlbC50b19iZXR0ZXJ0cmFuc2Zvcm1lcigpJTBBJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMkhlbGxvJTIwbXklMjBkb2clMjBpcyUyMGN1dGUlMjBhbmQlMjIlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTJCJTIwd2l0aCUyMHRvcmNoLmJhY2tlbmRzLmN1ZGEuc2RwX2tlcm5lbChlbmFibGVfZmxhc2glM0RUcnVlJTJDJTIwZW5hYmxlX21hdGglM0RGYWxzZSUyQyUyMGVuYWJsZV9tZW1fZWZmaWNpZW50JTNERmFsc2UpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(&quot;facebook/opt-350m&quot;)
model = AutoModelForCausalLM.from_pretrained(&quot;facebook/opt-350m&quot;, torch_dtype=torch.float16).to(&quot;cuda&quot;)
# convert the model to BetterTransformer
model.to_bettertransformer()

input_text = &quot;Hello my dog is cute and&quot;
inputs = tokenizer(input_text, return_tensors=&quot;pt&quot;).to(&quot;cuda&quot;)

<span class="hljs-addition">+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):</span>
    outputs = model.generate(**inputs)

print(tokenizer.decode(outputs[0], skip_special_tokens=True))`}}),Le=new J({props:{code:"UnVudGltZUVycm9yJTNBJTIwTm8lMjBhdmFpbGFibGUlMjBrZXJuZWwuJTIwJTIwQWJvcnRpbmclMjBleGVjdXRpb24u",highlighted:"RuntimeError: No available kernel.  Aborting execution."}}),Oe=new J({props:{code:"cGlwMyUyMGluc3RhbGwlMjAtVSUyMC0tcHJlJTIwdG9yY2glMjB0b3JjaHZpc2lvbiUyMHRvcmNoYXVkaW8lMjAtLWluZGV4LXVybCUyMGh0dHBzJTNBJTJGJTJGZG93bmxvYWQucHl0b3JjaC5vcmclMkZ3aGwlMkZuaWdodGx5JTJGY3UxMTg=",highlighted:"pip3 install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118"}}),et=new $({}),ue=new Xn({props:{$$slots:{default:[bm]},$$scope:{ctx:le}}}),tt=new $({}),ot=new $({}),lt=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`}}),rt=new $({}),at=new J({props:{code:"bW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`}}),st=new J({props:{code:"bWF4X21lbW9yeV9tYXBwaW5nJTIwJTNEJTIwJTdCMCUzQSUyMCUyMjYwME1CJTIyJTJDJTIwMSUzQSUyMCUyMjFHQiUyMiU3RCUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tM2IlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMjBtYXhfbWVtb3J5JTNEbWF4X21lbW9yeV9tYXBwaW5nJTBBKQ==",highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;600MB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;1GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`}}),it=new $({}),nt=new $({}),Ee=new Xn({props:{$$slots:{default:[vm]},$$scope:{ctx:le}}}),ct=new $({}),ut=new $({}),dt=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),ht=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMmI1JTIyJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSklMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIySGVsbG8lMkMlMjBteSUyMGxsYW1hJTIwaXMlMjBjdXRlJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQW91dHB1dHMlMjAlM0QlMjB0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKGdlbmVyYXRlZF9pZHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_name)
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)

prompt = <span class="hljs-string">&quot;Hello, my llama is cute&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
generated_ids = model.generate(**inputs)
outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)`}}),bt=new $({}),vt=new J({props:{code:"bW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),yt=new J({props:{code:"bWF4X21lbW9yeV9tYXBwaW5nJTIwJTNEJTIwJTdCMCUzQSUyMCUyMjFHQiUyMiUyQyUyMDElM0ElMjAlMjIyR0IlMjIlN0QlMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTNiJTIyJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlJTJDJTIwbWF4X21lbW9yeSUzRG1heF9tZW1vcnlfbWFwcGluZyUwQSk=",highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;1GB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;2GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`}}),wt=new $({}),Mt=new $({}),Et=new J({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMEEpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMzUwbSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcpJTBBJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMkhlbGxvJTIwbXklMjBkb2clMjBpcyUyMGN1dGUlMjBhbmQlMjIlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBd2l0aCUyMHRvcmNoLmJhY2tlbmRzLmN1ZGEuc2RwX2tlcm5lbChlbmFibGVfZmxhc2glM0RUcnVlJTJDJTIwZW5hYmxlX21hdGglM0RGYWxzZSUyQyUyMGVuYWJsZV9tZW1fZWZmaWNpZW50JTNERmFsc2UpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_compute_dtype=torch.float16
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, quantization_config=quantization_config)

input_text = <span class="hljs-string">&quot;Hello my dog is cute and&quot;</span>
inputs = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-keyword">with</span> torch.backends.cuda.sdp_kernel(enable_flash=<span class="hljs-literal">True</span>, enable_math=<span class="hljs-literal">False</span>, enable_mem_efficient=<span class="hljs-literal">False</span>):
    outputs = model.generate(**inputs)

<span class="hljs-built_in">print</span>(tokenizer.decode(outputs[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`}}),{c(){w=l("meta"),M=m(),_=l("h1"),g=l("a"),yo=l("span"),u(Pe.$$.fragment),Ja=m(),wo=l("span"),Ua=i("Efficient Inference on a Single GPU"),Al=m(),G=l("p"),ka=i("In addition to this guide, relevant information can be found as well in "),$t=l("a"),Ga=i("the guide for training on a single GPU"),Za=i(" and "),Jt=l("a"),Wa=i("the guide for inference on CPUs"),Ba=i("."),Nl=m(),V=l("h2"),re=l("a"),_o=l("span"),u(Ie.$$.fragment),Pa=m(),go=l("span"),Ia=i("BetterTransformer"),Vl=m(),xe=l("p"),je=l("a"),xa=i("BetterTransformer"),ja=i(" converts \u{1F917} Transformers models to use the PyTorch-native fastpath execution, which calls optimized kernels like Flash Attention under the hood."),Xl=m(),Ut=l("p"),Ra=i("BetterTransformer is also supported for faster inference on single and multi-GPU for text, image, and audio models."),ql=m(),u(ae.$$.fragment),zl=m(),X=l("h3"),se=l("a"),Mo=l("span"),u(Re.$$.fragment),Ca=m(),Eo=l("span"),Fa=i("Encoder models"),Yl=m(),Z=l("p"),Aa=i("PyTorch-native "),Ce=l("a"),To=l("code"),Na=i("nn.MultiHeadAttention"),Va=i(" attention fastpath, called BetterTransformer, can be used with Transformers through the integration in the "),Fe=l("a"),Xa=i("\u{1F917} Optimum library"),qa=i("."),Sl=m(),W=l("p"),za=i("PyTorch\u2019s attention fastpath allows to speed up inference through kernel fusions and the use of "),Ae=l("a"),Ya=i("nested tensors"),Sa=i(". Detailed benchmarks can be found in "),Ne=l("a"),Ha=i("this blog post"),Qa=i("."),Hl=m(),B=l("p"),Da=i("After installing the "),Ve=l("a"),$o=l("code"),La=i("optimum"),Oa=i(" package, to use Better Transformer during inference, the relevant internal modules are replaced by calling "),kt=l("a"),Ka=i("to_bettertransformer()"),es=i(":"),Ql=m(),u(Xe.$$.fragment),Dl=m(),ie=l("p"),ts=i("The method "),Gt=l("a"),os=i("reverse_bettertransformer()"),ls=i(" allows to go back to the original modeling, which should be used before saving the model in order to use the canonical transformers modeling:"),Ll=m(),u(qe.$$.fragment),Ol=m(),P=l("p"),rs=i("Have a look at this "),ze=l("a"),as=i("blog post"),ss=i(" to learn more about what is possible to do with "),Jo=l("code"),is=i("BetterTransformer"),ns=i(" API for encoder models."),Kl=m(),q=l("h3"),ne=l("a"),Uo=l("span"),u(Ye.$$.fragment),ps=m(),ko=l("span"),fs=i("Decoder models"),er=m(),pe=l("p"),ms=i("For text models, especially decoder-based models (GPT, T5, Llama, etc.), the BetterTransformer API converts all attention operations to use the "),fe=l("a"),Go=l("code"),cs=i("torch.nn.functional.scaled_dot_product_attention"),us=i(" operator"),ds=i(" (SDPA) that is only available in PyTorch 2.0 and onwards."),tr=m(),Zt=l("p"),hs=i("To convert a model to BetterTransformer:"),or=m(),u(Se.$$.fragment),lr=m(),I=l("p"),bs=i("SDPA can also call "),He=l("a"),vs=i("Flash Attention"),ys=i(" kernels under the hood. To enable Flash Attention or to check that it is available in a given setting (hardware, problem size), use "),Qe=l("a"),Zo=l("code"),ws=i("torch.backends.cuda.sdp_kernel"),_s=i(" as a context manager:"),rr=m(),u(De.$$.fragment),ar=m(),Wt=l("p"),gs=i("If you see a bug with a traceback saying"),sr=m(),u(Le.$$.fragment),ir=m(),Bt=l("p"),Ms=i("try using the PyTorch nightly version, which may have a broader coverage for Flash Attention:"),nr=m(),u(Oe.$$.fragment),pr=m(),Pt=l("p"),Es=i("Or make sure your model is correctly casted in float16 or bfloat16"),fr=m(),x=l("p"),Ts=i("Have a look at "),Ke=l("a"),$s=i("this detailed blogpost"),Js=i(" to read more about what is possible to do with "),Wo=l("code"),Us=i("BetterTransformer"),ks=i(" + SDPA API."),mr=m(),z=l("h2"),me=l("a"),Bo=l("span"),u(et.$$.fragment),Gs=m(),It=l("span"),Po=l("code"),Zs=i("bitsandbytes"),Ws=i(" integration for FP4 mixed-precision inference"),cr=m(),ce=l("p"),Bs=i("You can install "),Io=l("code"),Ps=i("bitsandbytes"),Is=i(" and benefit from easy model compression on GPUs. Using FP4 quantization you can expect to reduce up to 8x the model size compared to its native full precision version. Check out below how to get started."),ur=m(),u(ue.$$.fragment),dr=m(),Y=l("h3"),de=l("a"),xo=l("span"),u(tt.$$.fragment),xs=m(),jo=l("span"),js=i("Requirements"),hr=m(),j=l("ul"),Ro=l("li"),he=l("p"),Rs=i("Latest "),Co=l("code"),Cs=i("bitsandbytes"),Fs=i(` library
`),Fo=l("code"),As=i("pip install bitsandbytes>=0.39.0"),Ns=m(),Ao=l("li"),be=l("p"),Vs=i("Install latest "),No=l("code"),Xs=i("accelerate"),qs=i(` from source
`),Vo=l("code"),zs=i("pip install git+https://github.com/huggingface/accelerate.git"),Ys=m(),Xo=l("li"),ve=l("p"),Ss=i("Install latest "),qo=l("code"),Hs=i("transformers"),Qs=i(` from source
`),zo=l("code"),Ds=i("pip install git+https://github.com/huggingface/transformers.git"),br=m(),S=l("h3"),ye=l("a"),Yo=l("span"),u(ot.$$.fragment),Ls=m(),So=l("span"),Os=i("Running FP4 models - single GPU setup - Quickstart"),vr=m(),xt=l("p"),Ks=i("You can quickly run a FP4 model on a single GPU by running the following code:"),yr=m(),u(lt.$$.fragment),wr=m(),R=l("p"),ei=i("Note that "),Ho=l("code"),ti=i("device_map"),oi=i(" is optional but setting "),Qo=l("code"),li=i("device_map = 'auto'"),ri=i(" is prefered for inference as it will dispatch efficiently the model on the available ressources."),_r=m(),H=l("h3"),we=l("a"),Do=l("span"),u(rt.$$.fragment),ai=m(),Lo=l("span"),si=i("Running FP4 models - multi GPU setup"),gr=m(),jt=l("p"),ii=i("The way to load your mixed 4-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Mr=m(),u(at.$$.fragment),Er=m(),C=l("p"),ni=i("But you can control the GPU RAM you want to allocate on each GPU using "),Oo=l("code"),pi=i("accelerate"),fi=i(". Use the "),Ko=l("code"),mi=i("max_memory"),ci=i(" argument as follows:"),Tr=m(),u(st.$$.fragment),$r=m(),Rt=l("p"),ui=i("In this example, the first GPU will use 600MB of memory and the second 1GB."),Jr=m(),Q=l("h3"),_e=l("a"),el=l("span"),u(it.$$.fragment),di=m(),tl=l("span"),hi=i("Advanced usage"),Ur=m(),ge=l("p"),bi=i("For more advanced usage of this method, please have a look at the "),Ct=l("a"),vi=i("quantization"),yi=i(" documentation page."),kr=m(),D=l("h2"),Me=l("a"),ol=l("span"),u(nt.$$.fragment),wi=m(),Ft=l("span"),ll=l("code"),_i=i("bitsandbytes"),gi=i(" integration for Int8 mixed-precision matrix decomposition"),Gr=m(),u(Ee.$$.fragment),Zr=m(),E=l("p"),Mi=i("From the paper "),pt=l("a"),rl=l("code"),Ei=i("LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),Ti=i(`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),al=l("code"),$i=i("nn.Linear"),Ji=i(" size by 2 for "),sl=l("code"),Ui=i("float16"),ki=i(" and "),il=l("code"),Gi=i("bfloat16"),Zi=i(" weights and by 4 for "),nl=l("code"),Wi=i("float32"),Bi=i(" weights, with close to no impact to the quality by operating on the outliers in half-precision."),Wr=m(),At=l("p"),Nt=l("img"),Br=m(),F=l("p"),Pi=i(`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),ft=l("a"),Ii=i("paper"),xi=i(" or our "),mt=l("a"),ji=i("blogpost about the integration"),Ri=i("."),Pr=m(),Vt=l("p"),Xt=l("img"),Ir=m(),Te=l("p"),Ci=i(`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),qt=l("a"),Fi=i("Google colab"),Ai=i("."),xr=m(),L=l("h3"),$e=l("a"),pl=l("span"),u(ct.$$.fragment),Ni=m(),fl=l("span"),Vi=i("Requirements"),jr=m(),A=l("ul"),O=l("li"),Xi=i("If you have "),ml=l("code"),qi=i("bitsandbytes<0.37.0"),zi=i(", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),cl=l("code"),Yi=i("bitsandbytes>=0.37.0"),Si=i(", all GPUs should be supported."),Hi=m(),Je=l("li"),Qi=i("Install the correct version of "),ul=l("code"),Di=i("bitsandbytes"),Li=i(` by running:
`),dl=l("code"),Oi=i("pip install bitsandbytes>=0.31.5"),Ki=m(),Ue=l("li"),en=i("Install "),hl=l("code"),tn=i("accelerate"),on=m(),bl=l("code"),ln=i("pip install accelerate>=0.12.0"),Rr=m(),K=l("h3"),ke=l("a"),vl=l("span"),u(ut.$$.fragment),rn=m(),yl=l("span"),an=i("Running mixed-Int8 models - single GPU setup"),Cr=m(),zt=l("p"),sn=i("After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),Fr=m(),u(dt.$$.fragment),Ar=m(),Yt=l("p"),nn=i("For text generation, we recommend:"),Nr=m(),Ge=l("ul"),T=l("li"),pn=i("using the model\u2019s "),wl=l("code"),fn=i("generate()"),mn=i(" method instead of the "),_l=l("code"),cn=i("pipeline()"),un=i(" function. Although inference is possible with the "),gl=l("code"),dn=i("pipeline()"),hn=i(" function, it is not optimized for mixed-8bit models, and will be slower than using the "),Ml=l("code"),bn=i("generate()"),vn=i(" method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),El=l("code"),yn=i("pipeline()"),wn=i(" function for mixed-8bit models."),_n=m(),Tl=l("li"),gn=i("placing all inputs on the same device as the model."),Vr=m(),St=l("p"),Mn=i("Here is a simple example:"),Xr=m(),u(ht.$$.fragment),qr=m(),ee=l("h3"),Ze=l("a"),$l=l("span"),u(bt.$$.fragment),En=m(),Jl=l("span"),Tn=i("Running mixed-int8 models - multi GPU setup"),zr=m(),Ht=l("p"),$n=i("The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Yr=m(),u(vt.$$.fragment),Sr=m(),N=l("p"),Jn=i("But you can control the GPU RAM you want to allocate on each GPU using "),Ul=l("code"),Un=i("accelerate"),kn=i(". Use the "),kl=l("code"),Gn=i("max_memory"),Zn=i(" argument as follows:"),Hr=m(),u(yt.$$.fragment),Qr=m(),Qt=l("p"),Wn=i("In this example, the first GPU will use 1GB of memory and the second 2GB."),Dr=m(),te=l("h3"),We=l("a"),Gl=l("span"),u(wt.$$.fragment),Bn=m(),Zl=l("span"),Pn=i("Colab demos"),Lr=m(),Dt=l("p"),In=i(`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),Or=m(),Lt=l("p"),_t=l("a"),Ot=l("img"),Kr=m(),Kt=l("p"),xn=i("Or this demo for BLOOM-3B:"),ea=m(),eo=l("p"),gt=l("a"),to=l("img"),ta=m(),oe=l("h2"),Be=l("a"),Wl=l("span"),u(Mt.$$.fragment),jn=m(),Bl=l("span"),Rn=i("Advanced usage: mixing FP4 (or Int8) and BetterTransformer"),oa=m(),oo=l("p"),Cn=i("You can combine the different methods described above to get the best performance for your model. For example, you can use BetterTransformer with FP4 mixed-precision inference + flash attention:"),la=m(),u(Et.$$.fragment),this.h()},l(e){const s=um('[data-svelte="svelte-1phssyn"]',document.head);w=r(s,"META",{name:!0,content:!0}),s.forEach(t),M=c(e),_=r(e,"H1",{class:!0});var Tt=a(_);g=r(Tt,"A",{id:!0,class:!0,href:!0});var Pl=a(g);yo=r(Pl,"SPAN",{});var Il=a(yo);d(Pe.$$.fragment,Il),Il.forEach(t),Pl.forEach(t),Ja=c(Tt),wo=r(Tt,"SPAN",{});var Hn=a(wo);Ua=n(Hn,"Efficient Inference on a Single GPU"),Hn.forEach(t),Tt.forEach(t),Al=c(e),G=r(e,"P",{});var lo=a(G);ka=n(lo,"In addition to this guide, relevant information can be found as well in "),$t=r(lo,"A",{href:!0});var Qn=a($t);Ga=n(Qn,"the guide for training on a single GPU"),Qn.forEach(t),Za=n(lo," and "),Jt=r(lo,"A",{href:!0});var Dn=a(Jt);Wa=n(Dn,"the guide for inference on CPUs"),Dn.forEach(t),Ba=n(lo,"."),lo.forEach(t),Nl=c(e),V=r(e,"H2",{class:!0});var aa=a(V);re=r(aa,"A",{id:!0,class:!0,href:!0});var Ln=a(re);_o=r(Ln,"SPAN",{});var On=a(_o);d(Ie.$$.fragment,On),On.forEach(t),Ln.forEach(t),Pa=c(aa),go=r(aa,"SPAN",{});var Kn=a(go);Ia=n(Kn,"BetterTransformer"),Kn.forEach(t),aa.forEach(t),Vl=c(e),xe=r(e,"P",{});var Fn=a(xe);je=r(Fn,"A",{href:!0,rel:!0});var ep=a(je);xa=n(ep,"BetterTransformer"),ep.forEach(t),ja=n(Fn," converts \u{1F917} Transformers models to use the PyTorch-native fastpath execution, which calls optimized kernels like Flash Attention under the hood."),Fn.forEach(t),Xl=c(e),Ut=r(e,"P",{});var tp=a(Ut);Ra=n(tp,"BetterTransformer is also supported for faster inference on single and multi-GPU for text, image, and audio models."),tp.forEach(t),ql=c(e),d(ae.$$.fragment,e),zl=c(e),X=r(e,"H3",{class:!0});var sa=a(X);se=r(sa,"A",{id:!0,class:!0,href:!0});var op=a(se);Mo=r(op,"SPAN",{});var lp=a(Mo);d(Re.$$.fragment,lp),lp.forEach(t),op.forEach(t),Ca=c(sa),Eo=r(sa,"SPAN",{});var rp=a(Eo);Fa=n(rp,"Encoder models"),rp.forEach(t),sa.forEach(t),Yl=c(e),Z=r(e,"P",{});var ro=a(Z);Aa=n(ro,"PyTorch-native "),Ce=r(ro,"A",{href:!0,rel:!0});var ap=a(Ce);To=r(ap,"CODE",{});var sp=a(To);Na=n(sp,"nn.MultiHeadAttention"),sp.forEach(t),ap.forEach(t),Va=n(ro," attention fastpath, called BetterTransformer, can be used with Transformers through the integration in the "),Fe=r(ro,"A",{href:!0,rel:!0});var ip=a(Fe);Xa=n(ip,"\u{1F917} Optimum library"),ip.forEach(t),qa=n(ro,"."),ro.forEach(t),Sl=c(e),W=r(e,"P",{});var ao=a(W);za=n(ao,"PyTorch\u2019s attention fastpath allows to speed up inference through kernel fusions and the use of "),Ae=r(ao,"A",{href:!0,rel:!0});var np=a(Ae);Ya=n(np,"nested tensors"),np.forEach(t),Sa=n(ao,". Detailed benchmarks can be found in "),Ne=r(ao,"A",{href:!0,rel:!0});var pp=a(Ne);Ha=n(pp,"this blog post"),pp.forEach(t),Qa=n(ao,"."),ao.forEach(t),Hl=c(e),B=r(e,"P",{});var so=a(B);Da=n(so,"After installing the "),Ve=r(so,"A",{href:!0,rel:!0});var fp=a(Ve);$o=r(fp,"CODE",{});var mp=a($o);La=n(mp,"optimum"),mp.forEach(t),fp.forEach(t),Oa=n(so," package, to use Better Transformer during inference, the relevant internal modules are replaced by calling "),kt=r(so,"A",{href:!0});var cp=a(kt);Ka=n(cp,"to_bettertransformer()"),cp.forEach(t),es=n(so,":"),so.forEach(t),Ql=c(e),d(Xe.$$.fragment,e),Dl=c(e),ie=r(e,"P",{});var ia=a(ie);ts=n(ia,"The method "),Gt=r(ia,"A",{href:!0});var up=a(Gt);os=n(up,"reverse_bettertransformer()"),up.forEach(t),ls=n(ia," allows to go back to the original modeling, which should be used before saving the model in order to use the canonical transformers modeling:"),ia.forEach(t),Ll=c(e),d(qe.$$.fragment,e),Ol=c(e),P=r(e,"P",{});var io=a(P);rs=n(io,"Have a look at this "),ze=r(io,"A",{href:!0,rel:!0});var dp=a(ze);as=n(dp,"blog post"),dp.forEach(t),ss=n(io," to learn more about what is possible to do with "),Jo=r(io,"CODE",{});var hp=a(Jo);is=n(hp,"BetterTransformer"),hp.forEach(t),ns=n(io," API for encoder models."),io.forEach(t),Kl=c(e),q=r(e,"H3",{class:!0});var na=a(q);ne=r(na,"A",{id:!0,class:!0,href:!0});var bp=a(ne);Uo=r(bp,"SPAN",{});var vp=a(Uo);d(Ye.$$.fragment,vp),vp.forEach(t),bp.forEach(t),ps=c(na),ko=r(na,"SPAN",{});var yp=a(ko);fs=n(yp,"Decoder models"),yp.forEach(t),na.forEach(t),er=c(e),pe=r(e,"P",{});var pa=a(pe);ms=n(pa,"For text models, especially decoder-based models (GPT, T5, Llama, etc.), the BetterTransformer API converts all attention operations to use the "),fe=r(pa,"A",{href:!0,rel:!0});var An=a(fe);Go=r(An,"CODE",{});var wp=a(Go);cs=n(wp,"torch.nn.functional.scaled_dot_product_attention"),wp.forEach(t),us=n(An," operator"),An.forEach(t),ds=n(pa," (SDPA) that is only available in PyTorch 2.0 and onwards."),pa.forEach(t),tr=c(e),Zt=r(e,"P",{});var _p=a(Zt);hs=n(_p,"To convert a model to BetterTransformer:"),_p.forEach(t),or=c(e),d(Se.$$.fragment,e),lr=c(e),I=r(e,"P",{});var no=a(I);bs=n(no,"SDPA can also call "),He=r(no,"A",{href:!0,rel:!0});var gp=a(He);vs=n(gp,"Flash Attention"),gp.forEach(t),ys=n(no," kernels under the hood. To enable Flash Attention or to check that it is available in a given setting (hardware, problem size), use "),Qe=r(no,"A",{href:!0,rel:!0});var Mp=a(Qe);Zo=r(Mp,"CODE",{});var Ep=a(Zo);ws=n(Ep,"torch.backends.cuda.sdp_kernel"),Ep.forEach(t),Mp.forEach(t),_s=n(no," as a context manager:"),no.forEach(t),rr=c(e),d(De.$$.fragment,e),ar=c(e),Wt=r(e,"P",{});var Tp=a(Wt);gs=n(Tp,"If you see a bug with a traceback saying"),Tp.forEach(t),sr=c(e),d(Le.$$.fragment,e),ir=c(e),Bt=r(e,"P",{});var $p=a(Bt);Ms=n($p,"try using the PyTorch nightly version, which may have a broader coverage for Flash Attention:"),$p.forEach(t),nr=c(e),d(Oe.$$.fragment,e),pr=c(e),Pt=r(e,"P",{});var Jp=a(Pt);Es=n(Jp,"Or make sure your model is correctly casted in float16 or bfloat16"),Jp.forEach(t),fr=c(e),x=r(e,"P",{});var po=a(x);Ts=n(po,"Have a look at "),Ke=r(po,"A",{href:!0,rel:!0});var Up=a(Ke);$s=n(Up,"this detailed blogpost"),Up.forEach(t),Js=n(po," to read more about what is possible to do with "),Wo=r(po,"CODE",{});var kp=a(Wo);Us=n(kp,"BetterTransformer"),kp.forEach(t),ks=n(po," + SDPA API."),po.forEach(t),mr=c(e),z=r(e,"H2",{class:!0});var fa=a(z);me=r(fa,"A",{id:!0,class:!0,href:!0});var Gp=a(me);Bo=r(Gp,"SPAN",{});var Zp=a(Bo);d(et.$$.fragment,Zp),Zp.forEach(t),Gp.forEach(t),Gs=c(fa),It=r(fa,"SPAN",{});var Nn=a(It);Po=r(Nn,"CODE",{});var Wp=a(Po);Zs=n(Wp,"bitsandbytes"),Wp.forEach(t),Ws=n(Nn," integration for FP4 mixed-precision inference"),Nn.forEach(t),fa.forEach(t),cr=c(e),ce=r(e,"P",{});var ma=a(ce);Bs=n(ma,"You can install "),Io=r(ma,"CODE",{});var Bp=a(Io);Ps=n(Bp,"bitsandbytes"),Bp.forEach(t),Is=n(ma," and benefit from easy model compression on GPUs. Using FP4 quantization you can expect to reduce up to 8x the model size compared to its native full precision version. Check out below how to get started."),ma.forEach(t),ur=c(e),d(ue.$$.fragment,e),dr=c(e),Y=r(e,"H3",{class:!0});var ca=a(Y);de=r(ca,"A",{id:!0,class:!0,href:!0});var Pp=a(de);xo=r(Pp,"SPAN",{});var Ip=a(xo);d(tt.$$.fragment,Ip),Ip.forEach(t),Pp.forEach(t),xs=c(ca),jo=r(ca,"SPAN",{});var xp=a(jo);js=n(xp,"Requirements"),xp.forEach(t),ca.forEach(t),hr=c(e),j=r(e,"UL",{});var fo=a(j);Ro=r(fo,"LI",{});var jp=a(Ro);he=r(jp,"P",{});var xl=a(he);Rs=n(xl,"Latest "),Co=r(xl,"CODE",{});var Rp=a(Co);Cs=n(Rp,"bitsandbytes"),Rp.forEach(t),Fs=n(xl,` library
`),Fo=r(xl,"CODE",{});var Cp=a(Fo);As=n(Cp,"pip install bitsandbytes>=0.39.0"),Cp.forEach(t),xl.forEach(t),jp.forEach(t),Ns=c(fo),Ao=r(fo,"LI",{});var Fp=a(Ao);be=r(Fp,"P",{});var jl=a(be);Vs=n(jl,"Install latest "),No=r(jl,"CODE",{});var Ap=a(No);Xs=n(Ap,"accelerate"),Ap.forEach(t),qs=n(jl,` from source
`),Vo=r(jl,"CODE",{});var Np=a(Vo);zs=n(Np,"pip install git+https://github.com/huggingface/accelerate.git"),Np.forEach(t),jl.forEach(t),Fp.forEach(t),Ys=c(fo),Xo=r(fo,"LI",{});var Vp=a(Xo);ve=r(Vp,"P",{});var Rl=a(ve);Ss=n(Rl,"Install latest "),qo=r(Rl,"CODE",{});var Xp=a(qo);Hs=n(Xp,"transformers"),Xp.forEach(t),Qs=n(Rl,` from source
`),zo=r(Rl,"CODE",{});var qp=a(zo);Ds=n(qp,"pip install git+https://github.com/huggingface/transformers.git"),qp.forEach(t),Rl.forEach(t),Vp.forEach(t),fo.forEach(t),br=c(e),S=r(e,"H3",{class:!0});var ua=a(S);ye=r(ua,"A",{id:!0,class:!0,href:!0});var zp=a(ye);Yo=r(zp,"SPAN",{});var Yp=a(Yo);d(ot.$$.fragment,Yp),Yp.forEach(t),zp.forEach(t),Ls=c(ua),So=r(ua,"SPAN",{});var Sp=a(So);Os=n(Sp,"Running FP4 models - single GPU setup - Quickstart"),Sp.forEach(t),ua.forEach(t),vr=c(e),xt=r(e,"P",{});var Hp=a(xt);Ks=n(Hp,"You can quickly run a FP4 model on a single GPU by running the following code:"),Hp.forEach(t),yr=c(e),d(lt.$$.fragment,e),wr=c(e),R=r(e,"P",{});var mo=a(R);ei=n(mo,"Note that "),Ho=r(mo,"CODE",{});var Qp=a(Ho);ti=n(Qp,"device_map"),Qp.forEach(t),oi=n(mo," is optional but setting "),Qo=r(mo,"CODE",{});var Dp=a(Qo);li=n(Dp,"device_map = 'auto'"),Dp.forEach(t),ri=n(mo," is prefered for inference as it will dispatch efficiently the model on the available ressources."),mo.forEach(t),_r=c(e),H=r(e,"H3",{class:!0});var da=a(H);we=r(da,"A",{id:!0,class:!0,href:!0});var Lp=a(we);Do=r(Lp,"SPAN",{});var Op=a(Do);d(rt.$$.fragment,Op),Op.forEach(t),Lp.forEach(t),ai=c(da),Lo=r(da,"SPAN",{});var Kp=a(Lo);si=n(Kp,"Running FP4 models - multi GPU setup"),Kp.forEach(t),da.forEach(t),gr=c(e),jt=r(e,"P",{});var ef=a(jt);ii=n(ef,"The way to load your mixed 4-bit model in multiple GPUs is as follows (same command as single GPU setup):"),ef.forEach(t),Mr=c(e),d(at.$$.fragment,e),Er=c(e),C=r(e,"P",{});var co=a(C);ni=n(co,"But you can control the GPU RAM you want to allocate on each GPU using "),Oo=r(co,"CODE",{});var tf=a(Oo);pi=n(tf,"accelerate"),tf.forEach(t),fi=n(co,". Use the "),Ko=r(co,"CODE",{});var of=a(Ko);mi=n(of,"max_memory"),of.forEach(t),ci=n(co," argument as follows:"),co.forEach(t),Tr=c(e),d(st.$$.fragment,e),$r=c(e),Rt=r(e,"P",{});var lf=a(Rt);ui=n(lf,"In this example, the first GPU will use 600MB of memory and the second 1GB."),lf.forEach(t),Jr=c(e),Q=r(e,"H3",{class:!0});var ha=a(Q);_e=r(ha,"A",{id:!0,class:!0,href:!0});var rf=a(_e);el=r(rf,"SPAN",{});var af=a(el);d(it.$$.fragment,af),af.forEach(t),rf.forEach(t),di=c(ha),tl=r(ha,"SPAN",{});var sf=a(tl);hi=n(sf,"Advanced usage"),sf.forEach(t),ha.forEach(t),Ur=c(e),ge=r(e,"P",{});var ba=a(ge);bi=n(ba,"For more advanced usage of this method, please have a look at the "),Ct=r(ba,"A",{href:!0});var nf=a(Ct);vi=n(nf,"quantization"),nf.forEach(t),yi=n(ba," documentation page."),ba.forEach(t),kr=c(e),D=r(e,"H2",{class:!0});var va=a(D);Me=r(va,"A",{id:!0,class:!0,href:!0});var pf=a(Me);ol=r(pf,"SPAN",{});var ff=a(ol);d(nt.$$.fragment,ff),ff.forEach(t),pf.forEach(t),wi=c(va),Ft=r(va,"SPAN",{});var Vn=a(Ft);ll=r(Vn,"CODE",{});var mf=a(ll);_i=n(mf,"bitsandbytes"),mf.forEach(t),gi=n(Vn," integration for Int8 mixed-precision matrix decomposition"),Vn.forEach(t),va.forEach(t),Gr=c(e),d(Ee.$$.fragment,e),Zr=c(e),E=r(e,"P",{});var U=a(E);Mi=n(U,"From the paper "),pt=r(U,"A",{href:!0,rel:!0});var cf=a(pt);rl=r(cf,"CODE",{});var uf=a(rl);Ei=n(uf,"LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),uf.forEach(t),cf.forEach(t),Ti=n(U,`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),al=r(U,"CODE",{});var df=a(al);$i=n(df,"nn.Linear"),df.forEach(t),Ji=n(U," size by 2 for "),sl=r(U,"CODE",{});var hf=a(sl);Ui=n(hf,"float16"),hf.forEach(t),ki=n(U," and "),il=r(U,"CODE",{});var bf=a(il);Gi=n(bf,"bfloat16"),bf.forEach(t),Zi=n(U," weights and by 4 for "),nl=r(U,"CODE",{});var vf=a(nl);Wi=n(vf,"float32"),vf.forEach(t),Bi=n(U," weights, with close to no impact to the quality by operating on the outliers in half-precision."),U.forEach(t),Wr=c(e),At=r(e,"P",{});var yf=a(At);Nt=r(yf,"IMG",{src:!0,alt:!0}),yf.forEach(t),Br=c(e),F=r(e,"P",{});var uo=a(F);Pi=n(uo,`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),ft=r(uo,"A",{href:!0,rel:!0});var wf=a(ft);Ii=n(wf,"paper"),wf.forEach(t),xi=n(uo," or our "),mt=r(uo,"A",{href:!0,rel:!0});var _f=a(mt);ji=n(_f,"blogpost about the integration"),_f.forEach(t),Ri=n(uo,"."),uo.forEach(t),Pr=c(e),Vt=r(e,"P",{});var gf=a(Vt);Xt=r(gf,"IMG",{src:!0,alt:!0}),gf.forEach(t),Ir=c(e),Te=r(e,"P",{});var ya=a(Te);Ci=n(ya,`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),qt=r(ya,"A",{href:!0});var Mf=a(qt);Fi=n(Mf,"Google colab"),Mf.forEach(t),Ai=n(ya,"."),ya.forEach(t),xr=c(e),L=r(e,"H3",{class:!0});var wa=a(L);$e=r(wa,"A",{id:!0,class:!0,href:!0});var Ef=a($e);pl=r(Ef,"SPAN",{});var Tf=a(pl);d(ct.$$.fragment,Tf),Tf.forEach(t),Ef.forEach(t),Ni=c(wa),fl=r(wa,"SPAN",{});var $f=a(fl);Vi=n($f,"Requirements"),$f.forEach(t),wa.forEach(t),jr=c(e),A=r(e,"UL",{});var ho=a(A);O=r(ho,"LI",{});var bo=a(O);Xi=n(bo,"If you have "),ml=r(bo,"CODE",{});var Jf=a(ml);qi=n(Jf,"bitsandbytes<0.37.0"),Jf.forEach(t),zi=n(bo,", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),cl=r(bo,"CODE",{});var Uf=a(cl);Yi=n(Uf,"bitsandbytes>=0.37.0"),Uf.forEach(t),Si=n(bo,", all GPUs should be supported."),bo.forEach(t),Hi=c(ho),Je=r(ho,"LI",{});var Cl=a(Je);Qi=n(Cl,"Install the correct version of "),ul=r(Cl,"CODE",{});var kf=a(ul);Di=n(kf,"bitsandbytes"),kf.forEach(t),Li=n(Cl,` by running:
`),dl=r(Cl,"CODE",{});var Gf=a(dl);Oi=n(Gf,"pip install bitsandbytes>=0.31.5"),Gf.forEach(t),Cl.forEach(t),Ki=c(ho),Ue=r(ho,"LI",{});var Fl=a(Ue);en=n(Fl,"Install "),hl=r(Fl,"CODE",{});var Zf=a(hl);tn=n(Zf,"accelerate"),Zf.forEach(t),on=c(Fl),bl=r(Fl,"CODE",{});var Wf=a(bl);ln=n(Wf,"pip install accelerate>=0.12.0"),Wf.forEach(t),Fl.forEach(t),ho.forEach(t),Rr=c(e),K=r(e,"H3",{class:!0});var _a=a(K);ke=r(_a,"A",{id:!0,class:!0,href:!0});var Bf=a(ke);vl=r(Bf,"SPAN",{});var Pf=a(vl);d(ut.$$.fragment,Pf),Pf.forEach(t),Bf.forEach(t),rn=c(_a),yl=r(_a,"SPAN",{});var If=a(yl);an=n(If,"Running mixed-Int8 models - single GPU setup"),If.forEach(t),_a.forEach(t),Cr=c(e),zt=r(e,"P",{});var xf=a(zt);sn=n(xf,"After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),xf.forEach(t),Fr=c(e),d(dt.$$.fragment,e),Ar=c(e),Yt=r(e,"P",{});var jf=a(Yt);nn=n(jf,"For text generation, we recommend:"),jf.forEach(t),Nr=c(e),Ge=r(e,"UL",{});var ga=a(Ge);T=r(ga,"LI",{});var k=a(T);pn=n(k,"using the model\u2019s "),wl=r(k,"CODE",{});var Rf=a(wl);fn=n(Rf,"generate()"),Rf.forEach(t),mn=n(k," method instead of the "),_l=r(k,"CODE",{});var Cf=a(_l);cn=n(Cf,"pipeline()"),Cf.forEach(t),un=n(k," function. Although inference is possible with the "),gl=r(k,"CODE",{});var Ff=a(gl);dn=n(Ff,"pipeline()"),Ff.forEach(t),hn=n(k," function, it is not optimized for mixed-8bit models, and will be slower than using the "),Ml=r(k,"CODE",{});var Af=a(Ml);bn=n(Af,"generate()"),Af.forEach(t),vn=n(k," method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),El=r(k,"CODE",{});var Nf=a(El);yn=n(Nf,"pipeline()"),Nf.forEach(t),wn=n(k," function for mixed-8bit models."),k.forEach(t),_n=c(ga),Tl=r(ga,"LI",{});var Vf=a(Tl);gn=n(Vf,"placing all inputs on the same device as the model."),Vf.forEach(t),ga.forEach(t),Vr=c(e),St=r(e,"P",{});var Xf=a(St);Mn=n(Xf,"Here is a simple example:"),Xf.forEach(t),Xr=c(e),d(ht.$$.fragment,e),qr=c(e),ee=r(e,"H3",{class:!0});var Ma=a(ee);Ze=r(Ma,"A",{id:!0,class:!0,href:!0});var qf=a(Ze);$l=r(qf,"SPAN",{});var zf=a($l);d(bt.$$.fragment,zf),zf.forEach(t),qf.forEach(t),En=c(Ma),Jl=r(Ma,"SPAN",{});var Yf=a(Jl);Tn=n(Yf,"Running mixed-int8 models - multi GPU setup"),Yf.forEach(t),Ma.forEach(t),zr=c(e),Ht=r(e,"P",{});var Sf=a(Ht);$n=n(Sf,"The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Sf.forEach(t),Yr=c(e),d(vt.$$.fragment,e),Sr=c(e),N=r(e,"P",{});var vo=a(N);Jn=n(vo,"But you can control the GPU RAM you want to allocate on each GPU using "),Ul=r(vo,"CODE",{});var Hf=a(Ul);Un=n(Hf,"accelerate"),Hf.forEach(t),kn=n(vo,". Use the "),kl=r(vo,"CODE",{});var Qf=a(kl);Gn=n(Qf,"max_memory"),Qf.forEach(t),Zn=n(vo," argument as follows:"),vo.forEach(t),Hr=c(e),d(yt.$$.fragment,e),Qr=c(e),Qt=r(e,"P",{});var Df=a(Qt);Wn=n(Df,"In this example, the first GPU will use 1GB of memory and the second 2GB."),Df.forEach(t),Dr=c(e),te=r(e,"H3",{class:!0});var Ea=a(te);We=r(Ea,"A",{id:!0,class:!0,href:!0});var Lf=a(We);Gl=r(Lf,"SPAN",{});var Of=a(Gl);d(wt.$$.fragment,Of),Of.forEach(t),Lf.forEach(t),Bn=c(Ea),Zl=r(Ea,"SPAN",{});var Kf=a(Zl);Pn=n(Kf,"Colab demos"),Kf.forEach(t),Ea.forEach(t),Lr=c(e),Dt=r(e,"P",{});var em=a(Dt);In=n(em,`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),em.forEach(t),Or=c(e),Lt=r(e,"P",{});var tm=a(Lt);_t=r(tm,"A",{href:!0,rel:!0});var om=a(_t);Ot=r(om,"IMG",{src:!0,alt:!0}),om.forEach(t),tm.forEach(t),Kr=c(e),Kt=r(e,"P",{});var lm=a(Kt);xn=n(lm,"Or this demo for BLOOM-3B:"),lm.forEach(t),ea=c(e),eo=r(e,"P",{});var rm=a(eo);gt=r(rm,"A",{href:!0,rel:!0});var am=a(gt);to=r(am,"IMG",{src:!0,alt:!0}),am.forEach(t),rm.forEach(t),ta=c(e),oe=r(e,"H2",{class:!0});var Ta=a(oe);Be=r(Ta,"A",{id:!0,class:!0,href:!0});var sm=a(Be);Wl=r(sm,"SPAN",{});var im=a(Wl);d(Mt.$$.fragment,im),im.forEach(t),sm.forEach(t),jn=c(Ta),Bl=r(Ta,"SPAN",{});var nm=a(Bl);Rn=n(nm,"Advanced usage: mixing FP4 (or Int8) and BetterTransformer"),nm.forEach(t),Ta.forEach(t),oa=c(e),oo=r(e,"P",{});var pm=a(oo);Cn=n(pm,"You can combine the different methods described above to get the best performance for your model. For example, you can use BetterTransformer with FP4 mixed-precision inference + flash attention:"),pm.forEach(t),la=c(e),d(Et.$$.fragment,e),this.h()},h(){f(w,"name","hf:doc:metadata"),f(w,"content",JSON.stringify(wm)),f(g,"id","efficient-inference-on-a-single-gpu"),f(g,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(g,"href","#efficient-inference-on-a-single-gpu"),f(_,"class","relative group"),f($t,"href","perf_train_gpu_one"),f(Jt,"href","perf_infer_cpu"),f(re,"id","bettertransformer"),f(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(re,"href","#bettertransformer"),f(V,"class","relative group"),f(je,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),f(je,"rel","nofollow"),f(se,"id","encoder-models"),f(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(se,"href","#encoder-models"),f(X,"class","relative group"),f(Ce,"href","https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/"),f(Ce,"rel","nofollow"),f(Fe,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),f(Fe,"rel","nofollow"),f(Ae,"href","https://pytorch.org/docs/stable/nested.html"),f(Ae,"rel","nofollow"),f(Ne,"href","https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2"),f(Ne,"rel","nofollow"),f(Ve,"href","https://github.com/huggingface/optimum"),f(Ve,"rel","nofollow"),f(kt,"href","/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.to_bettertransformer"),f(Gt,"href","/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.reverse_bettertransformer"),f(ze,"href","https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2"),f(ze,"rel","nofollow"),f(ne,"id","decoder-models"),f(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ne,"href","#decoder-models"),f(q,"class","relative group"),f(fe,"href","https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention"),f(fe,"rel","nofollow"),f(He,"href","https://arxiv.org/abs/2205.14135"),f(He,"rel","nofollow"),f(Qe,"href","https://pytorch.org/docs/master/backends.html#torch.backends.cuda.sdp_kernel"),f(Qe,"rel","nofollow"),f(Ke,"href","https://pytorch.org/blog/out-of-the-box-acceleration/"),f(Ke,"rel","nofollow"),f(me,"id","bitsandbytes-integration-for-fp4-mixedprecision-inference"),f(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(me,"href","#bitsandbytes-integration-for-fp4-mixedprecision-inference"),f(z,"class","relative group"),f(de,"id","requirements-for-fp4-mixedprecision-inference"),f(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(de,"href","#requirements-for-fp4-mixedprecision-inference"),f(Y,"class","relative group"),f(ye,"id","running-fp4-models-single-gpu-setup-quickstart"),f(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ye,"href","#running-fp4-models-single-gpu-setup-quickstart"),f(S,"class","relative group"),f(we,"id","running-fp4-models-multi-gpu-setup"),f(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(we,"href","#running-fp4-models-multi-gpu-setup"),f(H,"class","relative group"),f(_e,"id","advanced-usage"),f(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(_e,"href","#advanced-usage"),f(Q,"class","relative group"),f(Ct,"href","main_classes/quantization"),f(Me,"id","bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),f(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Me,"href","#bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),f(D,"class","relative group"),f(pt,"href","https://arxiv.org/abs/2208.07339"),f(pt,"rel","nofollow"),$a(Nt.src,qn="https://cdn-uploads.huggingface.co/production/uploads/1659861207959-62441d1d9fdefb55a0b7d12c.png")||f(Nt,"src",qn),f(Nt,"alt","HFxbitsandbytes.png"),f(ft,"href","https://arxiv.org/abs/2208.07339"),f(ft,"rel","nofollow"),f(mt,"href","https://huggingface.co/blog/hf-bitsandbytes-integration"),f(mt,"rel","nofollow"),$a(Xt.src,zn="https://cdn-uploads.huggingface.co/production/uploads/1660567469965-62441d1d9fdefb55a0b7d12c.gif")||f(Xt,"src",zn),f(Xt,"alt","MixedInt8.gif"),f(qt,"href","#colab-demos"),f($e,"id","requirements-for-int8-mixedprecision-matrix-decomposition"),f($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f($e,"href","#requirements-for-int8-mixedprecision-matrix-decomposition"),f(L,"class","relative group"),f(ke,"id","running-mixedint8-models-single-gpu-setup"),f(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ke,"href","#running-mixedint8-models-single-gpu-setup"),f(K,"class","relative group"),f(Ze,"id","running-mixedint8-models-multi-gpu-setup"),f(Ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ze,"href","#running-mixedint8-models-multi-gpu-setup"),f(ee,"class","relative group"),f(We,"id","colab-demos"),f(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(We,"href","#colab-demos"),f(te,"class","relative group"),$a(Ot.src,Yn="https://colab.research.google.com/assets/colab-badge.svg")||f(Ot,"src",Yn),f(Ot,"alt","Open In Colab: T5-11b demo"),f(_t,"href","https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing"),f(_t,"rel","nofollow"),$a(to.src,Sn="https://colab.research.google.com/assets/colab-badge.svg")||f(to,"src",Sn),f(to,"alt","Open In Colab: BLOOM-3b demo"),f(gt,"href","https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing"),f(gt,"rel","nofollow"),f(Be,"id","advanced-usage-mixing-fp4-or-int8-and-bettertransformer"),f(Be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Be,"href","#advanced-usage-mixing-fp4-or-int8-and-bettertransformer"),f(oe,"class","relative group")},m(e,s){o(document.head,w),p(e,M,s),p(e,_,s),o(_,g),o(g,yo),h(Pe,yo,null),o(_,Ja),o(_,wo),o(wo,Ua),p(e,Al,s),p(e,G,s),o(G,ka),o(G,$t),o($t,Ga),o(G,Za),o(G,Jt),o(Jt,Wa),o(G,Ba),p(e,Nl,s),p(e,V,s),o(V,re),o(re,_o),h(Ie,_o,null),o(V,Pa),o(V,go),o(go,Ia),p(e,Vl,s),p(e,xe,s),o(xe,je),o(je,xa),o(xe,ja),p(e,Xl,s),p(e,Ut,s),o(Ut,Ra),p(e,ql,s),h(ae,e,s),p(e,zl,s),p(e,X,s),o(X,se),o(se,Mo),h(Re,Mo,null),o(X,Ca),o(X,Eo),o(Eo,Fa),p(e,Yl,s),p(e,Z,s),o(Z,Aa),o(Z,Ce),o(Ce,To),o(To,Na),o(Z,Va),o(Z,Fe),o(Fe,Xa),o(Z,qa),p(e,Sl,s),p(e,W,s),o(W,za),o(W,Ae),o(Ae,Ya),o(W,Sa),o(W,Ne),o(Ne,Ha),o(W,Qa),p(e,Hl,s),p(e,B,s),o(B,Da),o(B,Ve),o(Ve,$o),o($o,La),o(B,Oa),o(B,kt),o(kt,Ka),o(B,es),p(e,Ql,s),h(Xe,e,s),p(e,Dl,s),p(e,ie,s),o(ie,ts),o(ie,Gt),o(Gt,os),o(ie,ls),p(e,Ll,s),h(qe,e,s),p(e,Ol,s),p(e,P,s),o(P,rs),o(P,ze),o(ze,as),o(P,ss),o(P,Jo),o(Jo,is),o(P,ns),p(e,Kl,s),p(e,q,s),o(q,ne),o(ne,Uo),h(Ye,Uo,null),o(q,ps),o(q,ko),o(ko,fs),p(e,er,s),p(e,pe,s),o(pe,ms),o(pe,fe),o(fe,Go),o(Go,cs),o(fe,us),o(pe,ds),p(e,tr,s),p(e,Zt,s),o(Zt,hs),p(e,or,s),h(Se,e,s),p(e,lr,s),p(e,I,s),o(I,bs),o(I,He),o(He,vs),o(I,ys),o(I,Qe),o(Qe,Zo),o(Zo,ws),o(I,_s),p(e,rr,s),h(De,e,s),p(e,ar,s),p(e,Wt,s),o(Wt,gs),p(e,sr,s),h(Le,e,s),p(e,ir,s),p(e,Bt,s),o(Bt,Ms),p(e,nr,s),h(Oe,e,s),p(e,pr,s),p(e,Pt,s),o(Pt,Es),p(e,fr,s),p(e,x,s),o(x,Ts),o(x,Ke),o(Ke,$s),o(x,Js),o(x,Wo),o(Wo,Us),o(x,ks),p(e,mr,s),p(e,z,s),o(z,me),o(me,Bo),h(et,Bo,null),o(z,Gs),o(z,It),o(It,Po),o(Po,Zs),o(It,Ws),p(e,cr,s),p(e,ce,s),o(ce,Bs),o(ce,Io),o(Io,Ps),o(ce,Is),p(e,ur,s),h(ue,e,s),p(e,dr,s),p(e,Y,s),o(Y,de),o(de,xo),h(tt,xo,null),o(Y,xs),o(Y,jo),o(jo,js),p(e,hr,s),p(e,j,s),o(j,Ro),o(Ro,he),o(he,Rs),o(he,Co),o(Co,Cs),o(he,Fs),o(he,Fo),o(Fo,As),o(j,Ns),o(j,Ao),o(Ao,be),o(be,Vs),o(be,No),o(No,Xs),o(be,qs),o(be,Vo),o(Vo,zs),o(j,Ys),o(j,Xo),o(Xo,ve),o(ve,Ss),o(ve,qo),o(qo,Hs),o(ve,Qs),o(ve,zo),o(zo,Ds),p(e,br,s),p(e,S,s),o(S,ye),o(ye,Yo),h(ot,Yo,null),o(S,Ls),o(S,So),o(So,Os),p(e,vr,s),p(e,xt,s),o(xt,Ks),p(e,yr,s),h(lt,e,s),p(e,wr,s),p(e,R,s),o(R,ei),o(R,Ho),o(Ho,ti),o(R,oi),o(R,Qo),o(Qo,li),o(R,ri),p(e,_r,s),p(e,H,s),o(H,we),o(we,Do),h(rt,Do,null),o(H,ai),o(H,Lo),o(Lo,si),p(e,gr,s),p(e,jt,s),o(jt,ii),p(e,Mr,s),h(at,e,s),p(e,Er,s),p(e,C,s),o(C,ni),o(C,Oo),o(Oo,pi),o(C,fi),o(C,Ko),o(Ko,mi),o(C,ci),p(e,Tr,s),h(st,e,s),p(e,$r,s),p(e,Rt,s),o(Rt,ui),p(e,Jr,s),p(e,Q,s),o(Q,_e),o(_e,el),h(it,el,null),o(Q,di),o(Q,tl),o(tl,hi),p(e,Ur,s),p(e,ge,s),o(ge,bi),o(ge,Ct),o(Ct,vi),o(ge,yi),p(e,kr,s),p(e,D,s),o(D,Me),o(Me,ol),h(nt,ol,null),o(D,wi),o(D,Ft),o(Ft,ll),o(ll,_i),o(Ft,gi),p(e,Gr,s),h(Ee,e,s),p(e,Zr,s),p(e,E,s),o(E,Mi),o(E,pt),o(pt,rl),o(rl,Ei),o(E,Ti),o(E,al),o(al,$i),o(E,Ji),o(E,sl),o(sl,Ui),o(E,ki),o(E,il),o(il,Gi),o(E,Zi),o(E,nl),o(nl,Wi),o(E,Bi),p(e,Wr,s),p(e,At,s),o(At,Nt),p(e,Br,s),p(e,F,s),o(F,Pi),o(F,ft),o(ft,Ii),o(F,xi),o(F,mt),o(mt,ji),o(F,Ri),p(e,Pr,s),p(e,Vt,s),o(Vt,Xt),p(e,Ir,s),p(e,Te,s),o(Te,Ci),o(Te,qt),o(qt,Fi),o(Te,Ai),p(e,xr,s),p(e,L,s),o(L,$e),o($e,pl),h(ct,pl,null),o(L,Ni),o(L,fl),o(fl,Vi),p(e,jr,s),p(e,A,s),o(A,O),o(O,Xi),o(O,ml),o(ml,qi),o(O,zi),o(O,cl),o(cl,Yi),o(O,Si),o(A,Hi),o(A,Je),o(Je,Qi),o(Je,ul),o(ul,Di),o(Je,Li),o(Je,dl),o(dl,Oi),o(A,Ki),o(A,Ue),o(Ue,en),o(Ue,hl),o(hl,tn),o(Ue,on),o(Ue,bl),o(bl,ln),p(e,Rr,s),p(e,K,s),o(K,ke),o(ke,vl),h(ut,vl,null),o(K,rn),o(K,yl),o(yl,an),p(e,Cr,s),p(e,zt,s),o(zt,sn),p(e,Fr,s),h(dt,e,s),p(e,Ar,s),p(e,Yt,s),o(Yt,nn),p(e,Nr,s),p(e,Ge,s),o(Ge,T),o(T,pn),o(T,wl),o(wl,fn),o(T,mn),o(T,_l),o(_l,cn),o(T,un),o(T,gl),o(gl,dn),o(T,hn),o(T,Ml),o(Ml,bn),o(T,vn),o(T,El),o(El,yn),o(T,wn),o(Ge,_n),o(Ge,Tl),o(Tl,gn),p(e,Vr,s),p(e,St,s),o(St,Mn),p(e,Xr,s),h(ht,e,s),p(e,qr,s),p(e,ee,s),o(ee,Ze),o(Ze,$l),h(bt,$l,null),o(ee,En),o(ee,Jl),o(Jl,Tn),p(e,zr,s),p(e,Ht,s),o(Ht,$n),p(e,Yr,s),h(vt,e,s),p(e,Sr,s),p(e,N,s),o(N,Jn),o(N,Ul),o(Ul,Un),o(N,kn),o(N,kl),o(kl,Gn),o(N,Zn),p(e,Hr,s),h(yt,e,s),p(e,Qr,s),p(e,Qt,s),o(Qt,Wn),p(e,Dr,s),p(e,te,s),o(te,We),o(We,Gl),h(wt,Gl,null),o(te,Bn),o(te,Zl),o(Zl,Pn),p(e,Lr,s),p(e,Dt,s),o(Dt,In),p(e,Or,s),p(e,Lt,s),o(Lt,_t),o(_t,Ot),p(e,Kr,s),p(e,Kt,s),o(Kt,xn),p(e,ea,s),p(e,eo,s),o(eo,gt),o(gt,to),p(e,ta,s),p(e,oe,s),o(oe,Be),o(Be,Wl),h(Mt,Wl,null),o(oe,jn),o(oe,Bl),o(Bl,Rn),p(e,oa,s),p(e,oo,s),o(oo,Cn),p(e,la,s),h(Et,e,s),ra=!0},p(e,[s]){const Tt={};s&2&&(Tt.$$scope={dirty:s,ctx:e}),ae.$set(Tt);const Pl={};s&2&&(Pl.$$scope={dirty:s,ctx:e}),ue.$set(Pl);const Il={};s&2&&(Il.$$scope={dirty:s,ctx:e}),Ee.$set(Il)},i(e){ra||(b(Pe.$$.fragment,e),b(Ie.$$.fragment,e),b(ae.$$.fragment,e),b(Re.$$.fragment,e),b(Xe.$$.fragment,e),b(qe.$$.fragment,e),b(Ye.$$.fragment,e),b(Se.$$.fragment,e),b(De.$$.fragment,e),b(Le.$$.fragment,e),b(Oe.$$.fragment,e),b(et.$$.fragment,e),b(ue.$$.fragment,e),b(tt.$$.fragment,e),b(ot.$$.fragment,e),b(lt.$$.fragment,e),b(rt.$$.fragment,e),b(at.$$.fragment,e),b(st.$$.fragment,e),b(it.$$.fragment,e),b(nt.$$.fragment,e),b(Ee.$$.fragment,e),b(ct.$$.fragment,e),b(ut.$$.fragment,e),b(dt.$$.fragment,e),b(ht.$$.fragment,e),b(bt.$$.fragment,e),b(vt.$$.fragment,e),b(yt.$$.fragment,e),b(wt.$$.fragment,e),b(Mt.$$.fragment,e),b(Et.$$.fragment,e),ra=!0)},o(e){v(Pe.$$.fragment,e),v(Ie.$$.fragment,e),v(ae.$$.fragment,e),v(Re.$$.fragment,e),v(Xe.$$.fragment,e),v(qe.$$.fragment,e),v(Ye.$$.fragment,e),v(Se.$$.fragment,e),v(De.$$.fragment,e),v(Le.$$.fragment,e),v(Oe.$$.fragment,e),v(et.$$.fragment,e),v(ue.$$.fragment,e),v(tt.$$.fragment,e),v(ot.$$.fragment,e),v(lt.$$.fragment,e),v(rt.$$.fragment,e),v(at.$$.fragment,e),v(st.$$.fragment,e),v(it.$$.fragment,e),v(nt.$$.fragment,e),v(Ee.$$.fragment,e),v(ct.$$.fragment,e),v(ut.$$.fragment,e),v(dt.$$.fragment,e),v(ht.$$.fragment,e),v(bt.$$.fragment,e),v(vt.$$.fragment,e),v(yt.$$.fragment,e),v(wt.$$.fragment,e),v(Mt.$$.fragment,e),v(Et.$$.fragment,e),ra=!1},d(e){t(w),e&&t(M),e&&t(_),y(Pe),e&&t(Al),e&&t(G),e&&t(Nl),e&&t(V),y(Ie),e&&t(Vl),e&&t(xe),e&&t(Xl),e&&t(Ut),e&&t(ql),y(ae,e),e&&t(zl),e&&t(X),y(Re),e&&t(Yl),e&&t(Z),e&&t(Sl),e&&t(W),e&&t(Hl),e&&t(B),e&&t(Ql),y(Xe,e),e&&t(Dl),e&&t(ie),e&&t(Ll),y(qe,e),e&&t(Ol),e&&t(P),e&&t(Kl),e&&t(q),y(Ye),e&&t(er),e&&t(pe),e&&t(tr),e&&t(Zt),e&&t(or),y(Se,e),e&&t(lr),e&&t(I),e&&t(rr),y(De,e),e&&t(ar),e&&t(Wt),e&&t(sr),y(Le,e),e&&t(ir),e&&t(Bt),e&&t(nr),y(Oe,e),e&&t(pr),e&&t(Pt),e&&t(fr),e&&t(x),e&&t(mr),e&&t(z),y(et),e&&t(cr),e&&t(ce),e&&t(ur),y(ue,e),e&&t(dr),e&&t(Y),y(tt),e&&t(hr),e&&t(j),e&&t(br),e&&t(S),y(ot),e&&t(vr),e&&t(xt),e&&t(yr),y(lt,e),e&&t(wr),e&&t(R),e&&t(_r),e&&t(H),y(rt),e&&t(gr),e&&t(jt),e&&t(Mr),y(at,e),e&&t(Er),e&&t(C),e&&t(Tr),y(st,e),e&&t($r),e&&t(Rt),e&&t(Jr),e&&t(Q),y(it),e&&t(Ur),e&&t(ge),e&&t(kr),e&&t(D),y(nt),e&&t(Gr),y(Ee,e),e&&t(Zr),e&&t(E),e&&t(Wr),e&&t(At),e&&t(Br),e&&t(F),e&&t(Pr),e&&t(Vt),e&&t(Ir),e&&t(Te),e&&t(xr),e&&t(L),y(ct),e&&t(jr),e&&t(A),e&&t(Rr),e&&t(K),y(ut),e&&t(Cr),e&&t(zt),e&&t(Fr),y(dt,e),e&&t(Ar),e&&t(Yt),e&&t(Nr),e&&t(Ge),e&&t(Vr),e&&t(St),e&&t(Xr),y(ht,e),e&&t(qr),e&&t(ee),y(bt),e&&t(zr),e&&t(Ht),e&&t(Yr),y(vt,e),e&&t(Sr),e&&t(N),e&&t(Hr),y(yt,e),e&&t(Qr),e&&t(Qt),e&&t(Dr),e&&t(te),y(wt),e&&t(Lr),e&&t(Dt),e&&t(Or),e&&t(Lt),e&&t(Kr),e&&t(Kt),e&&t(ea),e&&t(eo),e&&t(ta),e&&t(oe),y(Mt),e&&t(oa),e&&t(oo),e&&t(la),y(Et,e)}}}const wm={local:"efficient-inference-on-a-single-gpu",sections:[{local:"bettertransformer",sections:[{local:"encoder-models",title:"Encoder models"},{local:"decoder-models",title:"Decoder models"}],title:"BetterTransformer"},{local:"bitsandbytes-integration-for-fp4-mixedprecision-inference",sections:[{local:"requirements-for-fp4-mixedprecision-inference",title:"Requirements "},{local:"running-fp4-models-single-gpu-setup-quickstart",title:"Running FP4 models - single GPU setup - Quickstart"},{local:"running-fp4-models-multi-gpu-setup",title:"Running FP4 models - multi GPU setup"},{local:"advanced-usage",title:"Advanced usage"}],title:"`bitsandbytes` integration for FP4 mixed-precision inference"},{local:"bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition",sections:[{local:"requirements-for-int8-mixedprecision-matrix-decomposition",title:"Requirements "},{local:"running-mixedint8-models-single-gpu-setup",title:"Running mixed-Int8 models - single GPU setup"},{local:"running-mixedint8-models-multi-gpu-setup",title:"Running mixed-int8 models - multi GPU setup"},{local:"colab-demos",title:"Colab demos"}],title:"`bitsandbytes` integration for Int8 mixed-precision matrix decomposition"},{local:"advanced-usage-mixing-fp4-or-int8-and-bettertransformer",title:"Advanced usage: mixing FP4 (or Int8) and BetterTransformer"}],title:"Efficient Inference on a Single GPU"};function _m(le){return dm(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $m extends fm{constructor(w){super();mm(this,w,_m,ym,cm,{})}}export{$m as default,wm as metadata};
