import{S as sm,i as nm,s as im,e as l,k as m,w as u,t as s,M as pm,c as r,d as t,m as c,a,x as d,h as n,b as f,N as Ea,G as o,g as p,y as h,q as b,o as v,B as y,v as fm}from"../chunks/vendor-hf-doc-builder.js";import{T as Ai}from"../chunks/Tip-hf-doc-builder.js";import{I as $}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as U}from"../chunks/CodeBlock-hf-doc-builder.js";function mm(le){let w,M;return{c(){w=l("p"),M=s("Flash Attention can only be used for models using fp16 or bf16 dtype. Make sure to cast your model to the appropriate dtype before using BetterTransformer.")},l(g){w=r(g,"P",{});var _=a(w);M=n(_,"Flash Attention can only be used for models using fp16 or bf16 dtype. Make sure to cast your model to the appropriate dtype before using BetterTransformer."),_.forEach(t)},m(g,_){p(g,w,_),o(w,M)},d(g){g&&t(w)}}}function cm(le){let w,M;return{c(){w=l("p"),M=s("Note that this feature can also be used in a multi GPU setup.")},l(g){w=r(g,"P",{});var _=a(w);M=n(_,"Note that this feature can also be used in a multi GPU setup."),_.forEach(t)},m(g,_){p(g,w,_),o(w,M)},d(g){g&&t(w)}}}function um(le){let w,M;return{c(){w=l("p"),M=s("Note that this feature can also be used in a multi GPU setup.")},l(g){w=r(g,"P",{});var _=a(w);M=n(_,"Note that this feature can also be used in a multi GPU setup."),_.forEach(t)},m(g,_){p(g,w,_),o(w,M)},d(g){g&&t(w)}}}function dm(le){let w,M,g,_,vo,Pe,Ta,yo,$a,Fl,G,Ua,$t,Ja,ka,Ut,Ga,Za,Al,V,re,wo,Ie,Wa,go,Ba,Nl,xe,je,Pa,Ia,Vl,Jt,xa,Xl,ae,ql,X,se,_o,Re,ja,Mo,Ra,zl,Z,Ca,Ce,Eo,Fa,Aa,Fe,Na,Va,Yl,W,Xa,Ae,qa,za,Ne,Ya,Sa,Sl,B,Ha,Ve,To,Qa,Da,kt,La,Oa,Hl,Xe,Ql,ne,Ka,Gt,es,ts,Dl,qe,Ll,P,os,ze,ls,rs,$o,as,ss,Ol,q,ie,Uo,Ye,ns,Jo,is,Kl,pe,ps,fe,ko,fs,ms,cs,er,Zt,us,tr,Se,or,I,ds,He,hs,bs,Qe,Go,vs,ys,lr,De,rr,Wt,ws,ar,Le,sr,Bt,gs,nr,Oe,ir,x,_s,Ke,Ms,Es,Zo,Ts,$s,pr,z,me,Wo,et,Us,Pt,Bo,Js,ks,fr,ce,Gs,Po,Zs,Ws,mr,ue,cr,Y,de,Io,tt,Bs,xo,Ps,ur,j,jo,he,Is,Ro,xs,js,Co,Rs,Cs,Fo,be,Fs,Ao,As,Ns,No,Vs,Xs,Vo,ve,qs,Xo,zs,Ys,qo,Ss,dr,S,ye,zo,ot,Hs,Yo,Qs,hr,It,Ds,br,lt,vr,R,Ls,So,Os,Ks,Ho,en,tn,yr,H,we,Qo,rt,on,Do,ln,wr,xt,rn,gr,at,_r,C,an,Lo,sn,nn,Oo,pn,fn,Mr,st,Er,jt,mn,Tr,Q,ge,Ko,nt,cn,el,un,$r,_e,dn,Rt,hn,bn,Ur,D,Me,tl,it,vn,Ct,ol,yn,wn,Jr,Ee,kr,E,gn,pt,ll,_n,Mn,rl,En,Tn,al,$n,Un,sl,Jn,kn,nl,Gn,Zn,Gr,Ft,At,Ni,Zr,F,Wn,ft,Bn,Pn,mt,In,xn,Wr,Nt,Vt,Vi,Br,Te,jn,Xt,Rn,Cn,Pr,L,$e,il,ct,Fn,pl,An,Ir,A,O,Nn,fl,Vn,Xn,ml,qn,zn,Yn,Ue,Sn,cl,Hn,Qn,ul,Dn,Ln,Je,On,dl,Kn,ei,hl,ti,xr,K,ke,bl,ut,oi,vl,li,jr,qt,ri,Rr,dt,Cr,zt,ai,Fr,Ge,T,si,yl,ni,ii,wl,pi,fi,gl,mi,ci,_l,ui,di,Ml,hi,bi,vi,El,yi,Ar,Yt,wi,Nr,ht,Vr,ee,Ze,Tl,bt,gi,$l,_i,Xr,St,Mi,qr,vt,zr,N,Ei,Ul,Ti,$i,Jl,Ui,Ji,Yr,yt,Sr,Ht,ki,Hr,te,We,kl,wt,Gi,Gl,Zi,Qr,Qt,Wi,Dr,Dt,gt,Lt,Xi,Lr,Ot,Bi,Or,Kt,_t,eo,qi,Kr,oe,Be,Zl,Mt,Pi,Wl,Ii,ea,to,xi,ta,Et,oa;return Pe=new $({}),Ie=new $({}),ae=new Ai({props:{$$slots:{default:[mm]},$$scope:{ctx:le}}}),Re=new $({}),Xe=new U({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC50b19iZXR0ZXJ0cmFuc2Zvcm1lcigp",highlighted:"model = model.to_bettertransformer()"}}),qe=new U({props:{code:"bW9kZWwlMjAlM0QlMjBtb2RlbC5yZXZlcnNlX2JldHRlcnRyYW5zZm9ybWVyKCklMEFtb2RlbC5zYXZlX3ByZXRyYWluZWQoJTIyc2F2ZWRfbW9kZWwlMjIp",highlighted:`model = model.reverse_bettertransformer()
model.save_pretrained(<span class="hljs-string">&quot;saved_model&quot;</span>)`}}),Ye=new $({}),Se=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMzUwbSUyMiklMEElMjMlMjBjb252ZXJ0JTIwdGhlJTIwbW9kZWwlMjB0byUyMEJldHRlclRyYW5zZm9ybWVyJTBBbW9kZWwudG9fYmV0dGVydHJhbnNmb3JtZXIoKSUwQSUwQSUyMyUyMFVzZSUyMGl0JTIwZm9yJTIwdHJhaW5pbmclMjBvciUyMGluZmVyZW5jZQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>)
<span class="hljs-comment"># convert the model to BetterTransformer</span>
model.to_bettertransformer()

<span class="hljs-comment"># Use it for training or inference</span>`}}),De=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMzUwbSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyKS50byglMjJjdWRhJTIyKSUwQSUyMyUyMGNvbnZlcnQlMjB0aGUlMjBtb2RlbCUyMHRvJTIwQmV0dGVyVHJhbnNmb3JtZXIlMEFtb2RlbC50b19iZXR0ZXJ0cmFuc2Zvcm1lcigpJTBBJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMkhlbGxvJTIwbXklMjBkb2clMjBpcyUyMGN1dGUlMjBhbmQlMjIlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTJCJTIwd2l0aCUyMHRvcmNoLmJhY2tlbmRzLmN1ZGEuc2RwX2tlcm5lbChlbmFibGVfZmxhc2glM0RUcnVlJTJDJTIwZW5hYmxlX21hdGglM0RGYWxzZSUyQyUyMGVuYWJsZV9tZW1fZWZmaWNpZW50JTNERmFsc2UpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(&quot;facebook/opt-350m&quot;)
model = AutoModelForCausalLM.from_pretrained(&quot;facebook/opt-350m&quot;).to(&quot;cuda&quot;)
# convert the model to BetterTransformer
model.to_bettertransformer()

input_text = &quot;Hello my dog is cute and&quot;
inputs = tokenizer(input_text, return_tensors=&quot;pt&quot;).to(&quot;cuda&quot;)

<span class="hljs-addition">+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):</span>
    outputs = model.generate(**inputs)

print(tokenizer.decode(outputs[0], skip_special_tokens=True))`}}),Le=new U({props:{code:"UnVudGltZUVycm9yJTNBJTIwTm8lMjBhdmFpbGFibGUlMjBrZXJuZWwuJTIwJTIwQWJvcnRpbmclMjBleGVjdXRpb24u",highlighted:"RuntimeError: No available kernel.  Aborting execution."}}),Oe=new U({props:{code:"cGlwMyUyMGluc3RhbGwlMjAtVSUyMC0tcHJlJTIwdG9yY2glMjB0b3JjaHZpc2lvbiUyMHRvcmNoYXVkaW8lMjAtLWluZGV4LXVybCUyMGh0dHBzJTNBJTJGJTJGZG93bmxvYWQucHl0b3JjaC5vcmclMkZ3aGwlMkZuaWdodGx5JTJGY3UxMTg=",highlighted:"pip3 install -U --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cu118"}}),et=new $({}),ue=new Ai({props:{$$slots:{default:[cm]},$$scope:{ctx:le}}}),tt=new $({}),ot=new $({}),lt=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`}}),rt=new $({}),at=new U({props:{code:"bW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`}}),st=new U({props:{code:"bWF4X21lbW9yeV9tYXBwaW5nJTIwJTNEJTIwJTdCMCUzQSUyMCUyMjYwME1CJTIyJTJDJTIwMSUzQSUyMCUyMjFHQiUyMiU3RCUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tM2IlMjIlMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUlMkMlMjBtYXhfbWVtb3J5JTNEbWF4X21lbW9yeV9tYXBwaW5nJTBBKQ==",highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;600MB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;1GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_4bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`}}),nt=new $({}),it=new $({}),Ee=new Ai({props:{$$slots:{default:[um]},$$scope:{ctx:le}}}),ct=new $({}),ut=new $({}),dt=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),ht=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMmI1JTIyJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSklMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBcHJvbXB0JTIwJTNEJTIwJTIySGVsbG8lMkMlMjBteSUyMGxsYW1hJTIwaXMlMjBjdXRlJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQW91dHB1dHMlMjAlM0QlMjB0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKGdlbmVyYXRlZF9pZHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_name)
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)

prompt = <span class="hljs-string">&quot;Hello, my llama is cute&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
generated_ids = model.generate(**inputs)
outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)`}}),bt=new $({}),vt=new U({props:{code:"bW9kZWxfbmFtZSUyMCUzRCUyMCUyMmJpZ3NjaWVuY2UlMkZibG9vbS0yYjUlMjIlMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),yt=new U({props:{code:"bWF4X21lbW9yeV9tYXBwaW5nJTIwJTNEJTIwJTdCMCUzQSUyMCUyMjFHQiUyMiUyQyUyMDElM0ElMjAlMjIyR0IlMjIlN0QlMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTNiJTIyJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlJTJDJTIwbWF4X21lbW9yeSUzRG1heF9tZW1vcnlfbWFwcGluZyUwQSk=",highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;1GB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;2GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`}}),wt=new $({}),Mt=new $({}),Et=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMEEpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMzUwbSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcpJTBBJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMkhlbGxvJTIwbXklMjBkb2clMjBpcyUyMGN1dGUlMjBhbmQlMjIlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBd2l0aCUyMHRvcmNoLmJhY2tlbmRzLmN1ZGEuc2RwX2tlcm5lbChlbmFibGVfZmxhc2glM0RUcnVlJTJDJTIwZW5hYmxlX21hdGglM0RGYWxzZSUyQyUyMGVuYWJsZV9tZW1fZWZmaWNpZW50JTNERmFsc2UpJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzKSUwQSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0cyU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_compute_dtype=torch.float16
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, quantization_config=quantization_config)

input_text = <span class="hljs-string">&quot;Hello my dog is cute and&quot;</span>
inputs = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-keyword">with</span> torch.backends.cuda.sdp_kernel(enable_flash=<span class="hljs-literal">True</span>, enable_math=<span class="hljs-literal">False</span>, enable_mem_efficient=<span class="hljs-literal">False</span>):
    outputs = model.generate(**inputs)

<span class="hljs-built_in">print</span>(tokenizer.decode(outputs[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`}}),{c(){w=l("meta"),M=m(),g=l("h1"),_=l("a"),vo=l("span"),u(Pe.$$.fragment),Ta=m(),yo=l("span"),$a=s("Efficient Inference on a Single GPU"),Fl=m(),G=l("p"),Ua=s("In addition to this guide, relevant information can be found as well in "),$t=l("a"),Ja=s("the guide for training on a single GPU"),ka=s(" and "),Ut=l("a"),Ga=s("the guide for inference on CPUs"),Za=s("."),Al=m(),V=l("h2"),re=l("a"),wo=l("span"),u(Ie.$$.fragment),Wa=m(),go=l("span"),Ba=s("BetterTransformer"),Nl=m(),xe=l("p"),je=l("a"),Pa=s("BetterTransformer"),Ia=s(" converts \u{1F917} Transformers models to use the PyTorch-native fastpath execution, which calls optimized kernels like Flash Attention under the hood."),Vl=m(),Jt=l("p"),xa=s("BetterTransformer is also supported for faster inference on single and multi-GPU for text, image, and audio models."),Xl=m(),u(ae.$$.fragment),ql=m(),X=l("h3"),se=l("a"),_o=l("span"),u(Re.$$.fragment),ja=m(),Mo=l("span"),Ra=s("Encoder models"),zl=m(),Z=l("p"),Ca=s("PyTorch-native "),Ce=l("a"),Eo=l("code"),Fa=s("nn.MultiHeadAttention"),Aa=s(" attention fastpath, called BetterTransformer, can be used with Transformers through the integration in the "),Fe=l("a"),Na=s("\u{1F917} Optimum library"),Va=s("."),Yl=m(),W=l("p"),Xa=s("PyTorch\u2019s attention fastpath allows to speed up inference through kernel fusions and the use of "),Ae=l("a"),qa=s("nested tensors"),za=s(". Detailed benchmarks can be found in "),Ne=l("a"),Ya=s("this blog post"),Sa=s("."),Sl=m(),B=l("p"),Ha=s("After installing the "),Ve=l("a"),To=l("code"),Qa=s("optimum"),Da=s(" package, to use Better Transformer during inference, the relevant internal modules are replaced by calling "),kt=l("a"),La=s("to_bettertransformer()"),Oa=s(":"),Hl=m(),u(Xe.$$.fragment),Ql=m(),ne=l("p"),Ka=s("The method "),Gt=l("a"),es=s("reverse_bettertransformer()"),ts=s(" allows to go back to the original modeling, which should be used before saving the model in order to use the canonical transformers modeling:"),Dl=m(),u(qe.$$.fragment),Ll=m(),P=l("p"),os=s("Have a look at this "),ze=l("a"),ls=s("blog post"),rs=s(" to learn more about what is possible to do with "),$o=l("code"),as=s("BetterTransformer"),ss=s(" API for encoder models."),Ol=m(),q=l("h3"),ie=l("a"),Uo=l("span"),u(Ye.$$.fragment),ns=m(),Jo=l("span"),is=s("Decoder models"),Kl=m(),pe=l("p"),ps=s("For text models, especially decoder-based models (GPT, T5, Llama, etc.), the BetterTransformer API converts all attention operations to use the "),fe=l("a"),ko=l("code"),fs=s("torch.nn.functional.scaled_dot_product_attention"),ms=s(" operator"),cs=s(" (SDPA) that is only available in PyTorch 2.0 and onwards."),er=m(),Zt=l("p"),us=s("To convert a model to BetterTransformer:"),tr=m(),u(Se.$$.fragment),or=m(),I=l("p"),ds=s("SDPA can also call "),He=l("a"),hs=s("Flash Attention"),bs=s(" kernels under the hood. To enable Flash Attention or to check that it is available in a given setting (hardware, problem size), use "),Qe=l("a"),Go=l("code"),vs=s("torch.backends.cuda.sdp_kernel"),ys=s(" as a context manager:"),lr=m(),u(De.$$.fragment),rr=m(),Wt=l("p"),ws=s("If you see a bug with a traceback saying"),ar=m(),u(Le.$$.fragment),sr=m(),Bt=l("p"),gs=s("try using the PyTorch nightly version, which may have a broader coverage for Flash Attention:"),nr=m(),u(Oe.$$.fragment),ir=m(),x=l("p"),_s=s("Have a look at "),Ke=l("a"),Ms=s("this detailed blogpost"),Es=s(" to read more about what is possible to do with "),Zo=l("code"),Ts=s("BetterTransformer"),$s=s(" + SDPA API."),pr=m(),z=l("h2"),me=l("a"),Wo=l("span"),u(et.$$.fragment),Us=m(),Pt=l("span"),Bo=l("code"),Js=s("bitsandbytes"),ks=s(" integration for FP4 mixed-precision inference"),fr=m(),ce=l("p"),Gs=s("You can install "),Po=l("code"),Zs=s("bitsandbytes"),Ws=s(" and benefit from easy model compression on GPUs. Using FP4 quantization you can expect to reduce up to 8x the model size compared to its native full precision version. Check out below how to get started."),mr=m(),u(ue.$$.fragment),cr=m(),Y=l("h3"),de=l("a"),Io=l("span"),u(tt.$$.fragment),Bs=m(),xo=l("span"),Ps=s("Requirements"),ur=m(),j=l("ul"),jo=l("li"),he=l("p"),Is=s("Latest "),Ro=l("code"),xs=s("bitsandbytes"),js=s(` library
`),Co=l("code"),Rs=s("pip install bitsandbytes>=0.39.0"),Cs=m(),Fo=l("li"),be=l("p"),Fs=s("Install latest "),Ao=l("code"),As=s("accelerate"),Ns=s(` from source
`),No=l("code"),Vs=s("pip install git+https://github.com/huggingface/accelerate.git"),Xs=m(),Vo=l("li"),ve=l("p"),qs=s("Install latest "),Xo=l("code"),zs=s("transformers"),Ys=s(` from source
`),qo=l("code"),Ss=s("pip install git+https://github.com/huggingface/transformers.git"),dr=m(),S=l("h3"),ye=l("a"),zo=l("span"),u(ot.$$.fragment),Hs=m(),Yo=l("span"),Qs=s("Running FP4 models - single GPU setup - Quickstart"),hr=m(),It=l("p"),Ds=s("You can quickly run a FP4 model on a single GPU by running the following code:"),br=m(),u(lt.$$.fragment),vr=m(),R=l("p"),Ls=s("Note that "),So=l("code"),Os=s("device_map"),Ks=s(" is optional but setting "),Ho=l("code"),en=s("device_map = 'auto'"),tn=s(" is prefered for inference as it will dispatch efficiently the model on the available ressources."),yr=m(),H=l("h3"),we=l("a"),Qo=l("span"),u(rt.$$.fragment),on=m(),Do=l("span"),ln=s("Running FP4 models - multi GPU setup"),wr=m(),xt=l("p"),rn=s("The way to load your mixed 4-bit model in multiple GPUs is as follows (same command as single GPU setup):"),gr=m(),u(at.$$.fragment),_r=m(),C=l("p"),an=s("But you can control the GPU RAM you want to allocate on each GPU using "),Lo=l("code"),sn=s("accelerate"),nn=s(". Use the "),Oo=l("code"),pn=s("max_memory"),fn=s(" argument as follows:"),Mr=m(),u(st.$$.fragment),Er=m(),jt=l("p"),mn=s("In this example, the first GPU will use 600MB of memory and the second 1GB."),Tr=m(),Q=l("h3"),ge=l("a"),Ko=l("span"),u(nt.$$.fragment),cn=m(),el=l("span"),un=s("Advanced usage"),$r=m(),_e=l("p"),dn=s("For more advanced usage of this method, please have a look at the "),Rt=l("a"),hn=s("quantization"),bn=s(" documentation page."),Ur=m(),D=l("h2"),Me=l("a"),tl=l("span"),u(it.$$.fragment),vn=m(),Ct=l("span"),ol=l("code"),yn=s("bitsandbytes"),wn=s(" integration for Int8 mixed-precision matrix decomposition"),Jr=m(),u(Ee.$$.fragment),kr=m(),E=l("p"),gn=s("From the paper "),pt=l("a"),ll=l("code"),_n=s("LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),Mn=s(`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),rl=l("code"),En=s("nn.Linear"),Tn=s(" size by 2 for "),al=l("code"),$n=s("float16"),Un=s(" and "),sl=l("code"),Jn=s("bfloat16"),kn=s(" weights and by 4 for "),nl=l("code"),Gn=s("float32"),Zn=s(" weights, with close to no impact to the quality by operating on the outliers in half-precision."),Gr=m(),Ft=l("p"),At=l("img"),Zr=m(),F=l("p"),Wn=s(`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),ft=l("a"),Bn=s("paper"),Pn=s(" or our "),mt=l("a"),In=s("blogpost about the integration"),xn=s("."),Wr=m(),Nt=l("p"),Vt=l("img"),Br=m(),Te=l("p"),jn=s(`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),Xt=l("a"),Rn=s("Google colab"),Cn=s("."),Pr=m(),L=l("h3"),$e=l("a"),il=l("span"),u(ct.$$.fragment),Fn=m(),pl=l("span"),An=s("Requirements"),Ir=m(),A=l("ul"),O=l("li"),Nn=s("If you have "),fl=l("code"),Vn=s("bitsandbytes<0.37.0"),Xn=s(", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),ml=l("code"),qn=s("bitsandbytes>=0.37.0"),zn=s(", all GPUs should be supported."),Yn=m(),Ue=l("li"),Sn=s("Install the correct version of "),cl=l("code"),Hn=s("bitsandbytes"),Qn=s(` by running:
`),ul=l("code"),Dn=s("pip install bitsandbytes>=0.31.5"),Ln=m(),Je=l("li"),On=s("Install "),dl=l("code"),Kn=s("accelerate"),ei=m(),hl=l("code"),ti=s("pip install accelerate>=0.12.0"),xr=m(),K=l("h3"),ke=l("a"),bl=l("span"),u(ut.$$.fragment),oi=m(),vl=l("span"),li=s("Running mixed-Int8 models - single GPU setup"),jr=m(),qt=l("p"),ri=s("After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),Rr=m(),u(dt.$$.fragment),Cr=m(),zt=l("p"),ai=s("For text generation, we recommend:"),Fr=m(),Ge=l("ul"),T=l("li"),si=s("using the model\u2019s "),yl=l("code"),ni=s("generate()"),ii=s(" method instead of the "),wl=l("code"),pi=s("pipeline()"),fi=s(" function. Although inference is possible with the "),gl=l("code"),mi=s("pipeline()"),ci=s(" function, it is not optimized for mixed-8bit models, and will be slower than using the "),_l=l("code"),ui=s("generate()"),di=s(" method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),Ml=l("code"),hi=s("pipeline()"),bi=s(" function for mixed-8bit models."),vi=m(),El=l("li"),yi=s("placing all inputs on the same device as the model."),Ar=m(),Yt=l("p"),wi=s("Here is a simple example:"),Nr=m(),u(ht.$$.fragment),Vr=m(),ee=l("h3"),Ze=l("a"),Tl=l("span"),u(bt.$$.fragment),gi=m(),$l=l("span"),_i=s("Running mixed-int8 models - multi GPU setup"),Xr=m(),St=l("p"),Mi=s("The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),qr=m(),u(vt.$$.fragment),zr=m(),N=l("p"),Ei=s("But you can control the GPU RAM you want to allocate on each GPU using "),Ul=l("code"),Ti=s("accelerate"),$i=s(". Use the "),Jl=l("code"),Ui=s("max_memory"),Ji=s(" argument as follows:"),Yr=m(),u(yt.$$.fragment),Sr=m(),Ht=l("p"),ki=s("In this example, the first GPU will use 1GB of memory and the second 2GB."),Hr=m(),te=l("h3"),We=l("a"),kl=l("span"),u(wt.$$.fragment),Gi=m(),Gl=l("span"),Zi=s("Colab demos"),Qr=m(),Qt=l("p"),Wi=s(`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),Dr=m(),Dt=l("p"),gt=l("a"),Lt=l("img"),Lr=m(),Ot=l("p"),Bi=s("Or this demo for BLOOM-3B:"),Or=m(),Kt=l("p"),_t=l("a"),eo=l("img"),Kr=m(),oe=l("h2"),Be=l("a"),Zl=l("span"),u(Mt.$$.fragment),Pi=m(),Wl=l("span"),Ii=s("Advanced usage: mixing FP4 (or Int8) and BetterTransformer"),ea=m(),to=l("p"),xi=s("You can combine the different methods described above to get the best performance for your model. For example, you can use BetterTransformer with FP4 mixed-precision inference + flash attention:"),ta=m(),u(Et.$$.fragment),this.h()},l(e){const i=pm('[data-svelte="svelte-1phssyn"]',document.head);w=r(i,"META",{name:!0,content:!0}),i.forEach(t),M=c(e),g=r(e,"H1",{class:!0});var Tt=a(g);_=r(Tt,"A",{id:!0,class:!0,href:!0});var Bl=a(_);vo=r(Bl,"SPAN",{});var Pl=a(vo);d(Pe.$$.fragment,Pl),Pl.forEach(t),Bl.forEach(t),Ta=c(Tt),yo=r(Tt,"SPAN",{});var zi=a(yo);$a=n(zi,"Efficient Inference on a Single GPU"),zi.forEach(t),Tt.forEach(t),Fl=c(e),G=r(e,"P",{});var oo=a(G);Ua=n(oo,"In addition to this guide, relevant information can be found as well in "),$t=r(oo,"A",{href:!0});var Yi=a($t);Ja=n(Yi,"the guide for training on a single GPU"),Yi.forEach(t),ka=n(oo," and "),Ut=r(oo,"A",{href:!0});var Si=a(Ut);Ga=n(Si,"the guide for inference on CPUs"),Si.forEach(t),Za=n(oo,"."),oo.forEach(t),Al=c(e),V=r(e,"H2",{class:!0});var la=a(V);re=r(la,"A",{id:!0,class:!0,href:!0});var Hi=a(re);wo=r(Hi,"SPAN",{});var Qi=a(wo);d(Ie.$$.fragment,Qi),Qi.forEach(t),Hi.forEach(t),Wa=c(la),go=r(la,"SPAN",{});var Di=a(go);Ba=n(Di,"BetterTransformer"),Di.forEach(t),la.forEach(t),Nl=c(e),xe=r(e,"P",{});var ji=a(xe);je=r(ji,"A",{href:!0,rel:!0});var Li=a(je);Pa=n(Li,"BetterTransformer"),Li.forEach(t),Ia=n(ji," converts \u{1F917} Transformers models to use the PyTorch-native fastpath execution, which calls optimized kernels like Flash Attention under the hood."),ji.forEach(t),Vl=c(e),Jt=r(e,"P",{});var Oi=a(Jt);xa=n(Oi,"BetterTransformer is also supported for faster inference on single and multi-GPU for text, image, and audio models."),Oi.forEach(t),Xl=c(e),d(ae.$$.fragment,e),ql=c(e),X=r(e,"H3",{class:!0});var ra=a(X);se=r(ra,"A",{id:!0,class:!0,href:!0});var Ki=a(se);_o=r(Ki,"SPAN",{});var ep=a(_o);d(Re.$$.fragment,ep),ep.forEach(t),Ki.forEach(t),ja=c(ra),Mo=r(ra,"SPAN",{});var tp=a(Mo);Ra=n(tp,"Encoder models"),tp.forEach(t),ra.forEach(t),zl=c(e),Z=r(e,"P",{});var lo=a(Z);Ca=n(lo,"PyTorch-native "),Ce=r(lo,"A",{href:!0,rel:!0});var op=a(Ce);Eo=r(op,"CODE",{});var lp=a(Eo);Fa=n(lp,"nn.MultiHeadAttention"),lp.forEach(t),op.forEach(t),Aa=n(lo," attention fastpath, called BetterTransformer, can be used with Transformers through the integration in the "),Fe=r(lo,"A",{href:!0,rel:!0});var rp=a(Fe);Na=n(rp,"\u{1F917} Optimum library"),rp.forEach(t),Va=n(lo,"."),lo.forEach(t),Yl=c(e),W=r(e,"P",{});var ro=a(W);Xa=n(ro,"PyTorch\u2019s attention fastpath allows to speed up inference through kernel fusions and the use of "),Ae=r(ro,"A",{href:!0,rel:!0});var ap=a(Ae);qa=n(ap,"nested tensors"),ap.forEach(t),za=n(ro,". Detailed benchmarks can be found in "),Ne=r(ro,"A",{href:!0,rel:!0});var sp=a(Ne);Ya=n(sp,"this blog post"),sp.forEach(t),Sa=n(ro,"."),ro.forEach(t),Sl=c(e),B=r(e,"P",{});var ao=a(B);Ha=n(ao,"After installing the "),Ve=r(ao,"A",{href:!0,rel:!0});var np=a(Ve);To=r(np,"CODE",{});var ip=a(To);Qa=n(ip,"optimum"),ip.forEach(t),np.forEach(t),Da=n(ao," package, to use Better Transformer during inference, the relevant internal modules are replaced by calling "),kt=r(ao,"A",{href:!0});var pp=a(kt);La=n(pp,"to_bettertransformer()"),pp.forEach(t),Oa=n(ao,":"),ao.forEach(t),Hl=c(e),d(Xe.$$.fragment,e),Ql=c(e),ne=r(e,"P",{});var aa=a(ne);Ka=n(aa,"The method "),Gt=r(aa,"A",{href:!0});var fp=a(Gt);es=n(fp,"reverse_bettertransformer()"),fp.forEach(t),ts=n(aa," allows to go back to the original modeling, which should be used before saving the model in order to use the canonical transformers modeling:"),aa.forEach(t),Dl=c(e),d(qe.$$.fragment,e),Ll=c(e),P=r(e,"P",{});var so=a(P);os=n(so,"Have a look at this "),ze=r(so,"A",{href:!0,rel:!0});var mp=a(ze);ls=n(mp,"blog post"),mp.forEach(t),rs=n(so," to learn more about what is possible to do with "),$o=r(so,"CODE",{});var cp=a($o);as=n(cp,"BetterTransformer"),cp.forEach(t),ss=n(so," API for encoder models."),so.forEach(t),Ol=c(e),q=r(e,"H3",{class:!0});var sa=a(q);ie=r(sa,"A",{id:!0,class:!0,href:!0});var up=a(ie);Uo=r(up,"SPAN",{});var dp=a(Uo);d(Ye.$$.fragment,dp),dp.forEach(t),up.forEach(t),ns=c(sa),Jo=r(sa,"SPAN",{});var hp=a(Jo);is=n(hp,"Decoder models"),hp.forEach(t),sa.forEach(t),Kl=c(e),pe=r(e,"P",{});var na=a(pe);ps=n(na,"For text models, especially decoder-based models (GPT, T5, Llama, etc.), the BetterTransformer API converts all attention operations to use the "),fe=r(na,"A",{href:!0,rel:!0});var Ri=a(fe);ko=r(Ri,"CODE",{});var bp=a(ko);fs=n(bp,"torch.nn.functional.scaled_dot_product_attention"),bp.forEach(t),ms=n(Ri," operator"),Ri.forEach(t),cs=n(na," (SDPA) that is only available in PyTorch 2.0 and onwards."),na.forEach(t),er=c(e),Zt=r(e,"P",{});var vp=a(Zt);us=n(vp,"To convert a model to BetterTransformer:"),vp.forEach(t),tr=c(e),d(Se.$$.fragment,e),or=c(e),I=r(e,"P",{});var no=a(I);ds=n(no,"SDPA can also call "),He=r(no,"A",{href:!0,rel:!0});var yp=a(He);hs=n(yp,"Flash Attention"),yp.forEach(t),bs=n(no," kernels under the hood. To enable Flash Attention or to check that it is available in a given setting (hardware, problem size), use "),Qe=r(no,"A",{href:!0,rel:!0});var wp=a(Qe);Go=r(wp,"CODE",{});var gp=a(Go);vs=n(gp,"torch.backends.cuda.sdp_kernel"),gp.forEach(t),wp.forEach(t),ys=n(no," as a context manager:"),no.forEach(t),lr=c(e),d(De.$$.fragment,e),rr=c(e),Wt=r(e,"P",{});var _p=a(Wt);ws=n(_p,"If you see a bug with a traceback saying"),_p.forEach(t),ar=c(e),d(Le.$$.fragment,e),sr=c(e),Bt=r(e,"P",{});var Mp=a(Bt);gs=n(Mp,"try using the PyTorch nightly version, which may have a broader coverage for Flash Attention:"),Mp.forEach(t),nr=c(e),d(Oe.$$.fragment,e),ir=c(e),x=r(e,"P",{});var io=a(x);_s=n(io,"Have a look at "),Ke=r(io,"A",{href:!0,rel:!0});var Ep=a(Ke);Ms=n(Ep,"this detailed blogpost"),Ep.forEach(t),Es=n(io," to read more about what is possible to do with "),Zo=r(io,"CODE",{});var Tp=a(Zo);Ts=n(Tp,"BetterTransformer"),Tp.forEach(t),$s=n(io," + SDPA API."),io.forEach(t),pr=c(e),z=r(e,"H2",{class:!0});var ia=a(z);me=r(ia,"A",{id:!0,class:!0,href:!0});var $p=a(me);Wo=r($p,"SPAN",{});var Up=a(Wo);d(et.$$.fragment,Up),Up.forEach(t),$p.forEach(t),Us=c(ia),Pt=r(ia,"SPAN",{});var Ci=a(Pt);Bo=r(Ci,"CODE",{});var Jp=a(Bo);Js=n(Jp,"bitsandbytes"),Jp.forEach(t),ks=n(Ci," integration for FP4 mixed-precision inference"),Ci.forEach(t),ia.forEach(t),fr=c(e),ce=r(e,"P",{});var pa=a(ce);Gs=n(pa,"You can install "),Po=r(pa,"CODE",{});var kp=a(Po);Zs=n(kp,"bitsandbytes"),kp.forEach(t),Ws=n(pa," and benefit from easy model compression on GPUs. Using FP4 quantization you can expect to reduce up to 8x the model size compared to its native full precision version. Check out below how to get started."),pa.forEach(t),mr=c(e),d(ue.$$.fragment,e),cr=c(e),Y=r(e,"H3",{class:!0});var fa=a(Y);de=r(fa,"A",{id:!0,class:!0,href:!0});var Gp=a(de);Io=r(Gp,"SPAN",{});var Zp=a(Io);d(tt.$$.fragment,Zp),Zp.forEach(t),Gp.forEach(t),Bs=c(fa),xo=r(fa,"SPAN",{});var Wp=a(xo);Ps=n(Wp,"Requirements"),Wp.forEach(t),fa.forEach(t),ur=c(e),j=r(e,"UL",{});var po=a(j);jo=r(po,"LI",{});var Bp=a(jo);he=r(Bp,"P",{});var Il=a(he);Is=n(Il,"Latest "),Ro=r(Il,"CODE",{});var Pp=a(Ro);xs=n(Pp,"bitsandbytes"),Pp.forEach(t),js=n(Il,` library
`),Co=r(Il,"CODE",{});var Ip=a(Co);Rs=n(Ip,"pip install bitsandbytes>=0.39.0"),Ip.forEach(t),Il.forEach(t),Bp.forEach(t),Cs=c(po),Fo=r(po,"LI",{});var xp=a(Fo);be=r(xp,"P",{});var xl=a(be);Fs=n(xl,"Install latest "),Ao=r(xl,"CODE",{});var jp=a(Ao);As=n(jp,"accelerate"),jp.forEach(t),Ns=n(xl,` from source
`),No=r(xl,"CODE",{});var Rp=a(No);Vs=n(Rp,"pip install git+https://github.com/huggingface/accelerate.git"),Rp.forEach(t),xl.forEach(t),xp.forEach(t),Xs=c(po),Vo=r(po,"LI",{});var Cp=a(Vo);ve=r(Cp,"P",{});var jl=a(ve);qs=n(jl,"Install latest "),Xo=r(jl,"CODE",{});var Fp=a(Xo);zs=n(Fp,"transformers"),Fp.forEach(t),Ys=n(jl,` from source
`),qo=r(jl,"CODE",{});var Ap=a(qo);Ss=n(Ap,"pip install git+https://github.com/huggingface/transformers.git"),Ap.forEach(t),jl.forEach(t),Cp.forEach(t),po.forEach(t),dr=c(e),S=r(e,"H3",{class:!0});var ma=a(S);ye=r(ma,"A",{id:!0,class:!0,href:!0});var Np=a(ye);zo=r(Np,"SPAN",{});var Vp=a(zo);d(ot.$$.fragment,Vp),Vp.forEach(t),Np.forEach(t),Hs=c(ma),Yo=r(ma,"SPAN",{});var Xp=a(Yo);Qs=n(Xp,"Running FP4 models - single GPU setup - Quickstart"),Xp.forEach(t),ma.forEach(t),hr=c(e),It=r(e,"P",{});var qp=a(It);Ds=n(qp,"You can quickly run a FP4 model on a single GPU by running the following code:"),qp.forEach(t),br=c(e),d(lt.$$.fragment,e),vr=c(e),R=r(e,"P",{});var fo=a(R);Ls=n(fo,"Note that "),So=r(fo,"CODE",{});var zp=a(So);Os=n(zp,"device_map"),zp.forEach(t),Ks=n(fo," is optional but setting "),Ho=r(fo,"CODE",{});var Yp=a(Ho);en=n(Yp,"device_map = 'auto'"),Yp.forEach(t),tn=n(fo," is prefered for inference as it will dispatch efficiently the model on the available ressources."),fo.forEach(t),yr=c(e),H=r(e,"H3",{class:!0});var ca=a(H);we=r(ca,"A",{id:!0,class:!0,href:!0});var Sp=a(we);Qo=r(Sp,"SPAN",{});var Hp=a(Qo);d(rt.$$.fragment,Hp),Hp.forEach(t),Sp.forEach(t),on=c(ca),Do=r(ca,"SPAN",{});var Qp=a(Do);ln=n(Qp,"Running FP4 models - multi GPU setup"),Qp.forEach(t),ca.forEach(t),wr=c(e),xt=r(e,"P",{});var Dp=a(xt);rn=n(Dp,"The way to load your mixed 4-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Dp.forEach(t),gr=c(e),d(at.$$.fragment,e),_r=c(e),C=r(e,"P",{});var mo=a(C);an=n(mo,"But you can control the GPU RAM you want to allocate on each GPU using "),Lo=r(mo,"CODE",{});var Lp=a(Lo);sn=n(Lp,"accelerate"),Lp.forEach(t),nn=n(mo,". Use the "),Oo=r(mo,"CODE",{});var Op=a(Oo);pn=n(Op,"max_memory"),Op.forEach(t),fn=n(mo," argument as follows:"),mo.forEach(t),Mr=c(e),d(st.$$.fragment,e),Er=c(e),jt=r(e,"P",{});var Kp=a(jt);mn=n(Kp,"In this example, the first GPU will use 600MB of memory and the second 1GB."),Kp.forEach(t),Tr=c(e),Q=r(e,"H3",{class:!0});var ua=a(Q);ge=r(ua,"A",{id:!0,class:!0,href:!0});var ef=a(ge);Ko=r(ef,"SPAN",{});var tf=a(Ko);d(nt.$$.fragment,tf),tf.forEach(t),ef.forEach(t),cn=c(ua),el=r(ua,"SPAN",{});var of=a(el);un=n(of,"Advanced usage"),of.forEach(t),ua.forEach(t),$r=c(e),_e=r(e,"P",{});var da=a(_e);dn=n(da,"For more advanced usage of this method, please have a look at the "),Rt=r(da,"A",{href:!0});var lf=a(Rt);hn=n(lf,"quantization"),lf.forEach(t),bn=n(da," documentation page."),da.forEach(t),Ur=c(e),D=r(e,"H2",{class:!0});var ha=a(D);Me=r(ha,"A",{id:!0,class:!0,href:!0});var rf=a(Me);tl=r(rf,"SPAN",{});var af=a(tl);d(it.$$.fragment,af),af.forEach(t),rf.forEach(t),vn=c(ha),Ct=r(ha,"SPAN",{});var Fi=a(Ct);ol=r(Fi,"CODE",{});var sf=a(ol);yn=n(sf,"bitsandbytes"),sf.forEach(t),wn=n(Fi," integration for Int8 mixed-precision matrix decomposition"),Fi.forEach(t),ha.forEach(t),Jr=c(e),d(Ee.$$.fragment,e),kr=c(e),E=r(e,"P",{});var J=a(E);gn=n(J,"From the paper "),pt=r(J,"A",{href:!0,rel:!0});var nf=a(pt);ll=r(nf,"CODE",{});var pf=a(ll);_n=n(pf,"LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),pf.forEach(t),nf.forEach(t),Mn=n(J,`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),rl=r(J,"CODE",{});var ff=a(rl);En=n(ff,"nn.Linear"),ff.forEach(t),Tn=n(J," size by 2 for "),al=r(J,"CODE",{});var mf=a(al);$n=n(mf,"float16"),mf.forEach(t),Un=n(J," and "),sl=r(J,"CODE",{});var cf=a(sl);Jn=n(cf,"bfloat16"),cf.forEach(t),kn=n(J," weights and by 4 for "),nl=r(J,"CODE",{});var uf=a(nl);Gn=n(uf,"float32"),uf.forEach(t),Zn=n(J," weights, with close to no impact to the quality by operating on the outliers in half-precision."),J.forEach(t),Gr=c(e),Ft=r(e,"P",{});var df=a(Ft);At=r(df,"IMG",{src:!0,alt:!0}),df.forEach(t),Zr=c(e),F=r(e,"P",{});var co=a(F);Wn=n(co,`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),ft=r(co,"A",{href:!0,rel:!0});var hf=a(ft);Bn=n(hf,"paper"),hf.forEach(t),Pn=n(co," or our "),mt=r(co,"A",{href:!0,rel:!0});var bf=a(mt);In=n(bf,"blogpost about the integration"),bf.forEach(t),xn=n(co,"."),co.forEach(t),Wr=c(e),Nt=r(e,"P",{});var vf=a(Nt);Vt=r(vf,"IMG",{src:!0,alt:!0}),vf.forEach(t),Br=c(e),Te=r(e,"P",{});var ba=a(Te);jn=n(ba,`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),Xt=r(ba,"A",{href:!0});var yf=a(Xt);Rn=n(yf,"Google colab"),yf.forEach(t),Cn=n(ba,"."),ba.forEach(t),Pr=c(e),L=r(e,"H3",{class:!0});var va=a(L);$e=r(va,"A",{id:!0,class:!0,href:!0});var wf=a($e);il=r(wf,"SPAN",{});var gf=a(il);d(ct.$$.fragment,gf),gf.forEach(t),wf.forEach(t),Fn=c(va),pl=r(va,"SPAN",{});var _f=a(pl);An=n(_f,"Requirements"),_f.forEach(t),va.forEach(t),Ir=c(e),A=r(e,"UL",{});var uo=a(A);O=r(uo,"LI",{});var ho=a(O);Nn=n(ho,"If you have "),fl=r(ho,"CODE",{});var Mf=a(fl);Vn=n(Mf,"bitsandbytes<0.37.0"),Mf.forEach(t),Xn=n(ho,", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),ml=r(ho,"CODE",{});var Ef=a(ml);qn=n(Ef,"bitsandbytes>=0.37.0"),Ef.forEach(t),zn=n(ho,", all GPUs should be supported."),ho.forEach(t),Yn=c(uo),Ue=r(uo,"LI",{});var Rl=a(Ue);Sn=n(Rl,"Install the correct version of "),cl=r(Rl,"CODE",{});var Tf=a(cl);Hn=n(Tf,"bitsandbytes"),Tf.forEach(t),Qn=n(Rl,` by running:
`),ul=r(Rl,"CODE",{});var $f=a(ul);Dn=n($f,"pip install bitsandbytes>=0.31.5"),$f.forEach(t),Rl.forEach(t),Ln=c(uo),Je=r(uo,"LI",{});var Cl=a(Je);On=n(Cl,"Install "),dl=r(Cl,"CODE",{});var Uf=a(dl);Kn=n(Uf,"accelerate"),Uf.forEach(t),ei=c(Cl),hl=r(Cl,"CODE",{});var Jf=a(hl);ti=n(Jf,"pip install accelerate>=0.12.0"),Jf.forEach(t),Cl.forEach(t),uo.forEach(t),xr=c(e),K=r(e,"H3",{class:!0});var ya=a(K);ke=r(ya,"A",{id:!0,class:!0,href:!0});var kf=a(ke);bl=r(kf,"SPAN",{});var Gf=a(bl);d(ut.$$.fragment,Gf),Gf.forEach(t),kf.forEach(t),oi=c(ya),vl=r(ya,"SPAN",{});var Zf=a(vl);li=n(Zf,"Running mixed-Int8 models - single GPU setup"),Zf.forEach(t),ya.forEach(t),jr=c(e),qt=r(e,"P",{});var Wf=a(qt);ri=n(Wf,"After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),Wf.forEach(t),Rr=c(e),d(dt.$$.fragment,e),Cr=c(e),zt=r(e,"P",{});var Bf=a(zt);ai=n(Bf,"For text generation, we recommend:"),Bf.forEach(t),Fr=c(e),Ge=r(e,"UL",{});var wa=a(Ge);T=r(wa,"LI",{});var k=a(T);si=n(k,"using the model\u2019s "),yl=r(k,"CODE",{});var Pf=a(yl);ni=n(Pf,"generate()"),Pf.forEach(t),ii=n(k," method instead of the "),wl=r(k,"CODE",{});var If=a(wl);pi=n(If,"pipeline()"),If.forEach(t),fi=n(k," function. Although inference is possible with the "),gl=r(k,"CODE",{});var xf=a(gl);mi=n(xf,"pipeline()"),xf.forEach(t),ci=n(k," function, it is not optimized for mixed-8bit models, and will be slower than using the "),_l=r(k,"CODE",{});var jf=a(_l);ui=n(jf,"generate()"),jf.forEach(t),di=n(k," method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),Ml=r(k,"CODE",{});var Rf=a(Ml);hi=n(Rf,"pipeline()"),Rf.forEach(t),bi=n(k," function for mixed-8bit models."),k.forEach(t),vi=c(wa),El=r(wa,"LI",{});var Cf=a(El);yi=n(Cf,"placing all inputs on the same device as the model."),Cf.forEach(t),wa.forEach(t),Ar=c(e),Yt=r(e,"P",{});var Ff=a(Yt);wi=n(Ff,"Here is a simple example:"),Ff.forEach(t),Nr=c(e),d(ht.$$.fragment,e),Vr=c(e),ee=r(e,"H3",{class:!0});var ga=a(ee);Ze=r(ga,"A",{id:!0,class:!0,href:!0});var Af=a(Ze);Tl=r(Af,"SPAN",{});var Nf=a(Tl);d(bt.$$.fragment,Nf),Nf.forEach(t),Af.forEach(t),gi=c(ga),$l=r(ga,"SPAN",{});var Vf=a($l);_i=n(Vf,"Running mixed-int8 models - multi GPU setup"),Vf.forEach(t),ga.forEach(t),Xr=c(e),St=r(e,"P",{});var Xf=a(St);Mi=n(Xf,"The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Xf.forEach(t),qr=c(e),d(vt.$$.fragment,e),zr=c(e),N=r(e,"P",{});var bo=a(N);Ei=n(bo,"But you can control the GPU RAM you want to allocate on each GPU using "),Ul=r(bo,"CODE",{});var qf=a(Ul);Ti=n(qf,"accelerate"),qf.forEach(t),$i=n(bo,". Use the "),Jl=r(bo,"CODE",{});var zf=a(Jl);Ui=n(zf,"max_memory"),zf.forEach(t),Ji=n(bo," argument as follows:"),bo.forEach(t),Yr=c(e),d(yt.$$.fragment,e),Sr=c(e),Ht=r(e,"P",{});var Yf=a(Ht);ki=n(Yf,"In this example, the first GPU will use 1GB of memory and the second 2GB."),Yf.forEach(t),Hr=c(e),te=r(e,"H3",{class:!0});var _a=a(te);We=r(_a,"A",{id:!0,class:!0,href:!0});var Sf=a(We);kl=r(Sf,"SPAN",{});var Hf=a(kl);d(wt.$$.fragment,Hf),Hf.forEach(t),Sf.forEach(t),Gi=c(_a),Gl=r(_a,"SPAN",{});var Qf=a(Gl);Zi=n(Qf,"Colab demos"),Qf.forEach(t),_a.forEach(t),Qr=c(e),Qt=r(e,"P",{});var Df=a(Qt);Wi=n(Df,`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),Df.forEach(t),Dr=c(e),Dt=r(e,"P",{});var Lf=a(Dt);gt=r(Lf,"A",{href:!0,rel:!0});var Of=a(gt);Lt=r(Of,"IMG",{src:!0,alt:!0}),Of.forEach(t),Lf.forEach(t),Lr=c(e),Ot=r(e,"P",{});var Kf=a(Ot);Bi=n(Kf,"Or this demo for BLOOM-3B:"),Kf.forEach(t),Or=c(e),Kt=r(e,"P",{});var em=a(Kt);_t=r(em,"A",{href:!0,rel:!0});var tm=a(_t);eo=r(tm,"IMG",{src:!0,alt:!0}),tm.forEach(t),em.forEach(t),Kr=c(e),oe=r(e,"H2",{class:!0});var Ma=a(oe);Be=r(Ma,"A",{id:!0,class:!0,href:!0});var om=a(Be);Zl=r(om,"SPAN",{});var lm=a(Zl);d(Mt.$$.fragment,lm),lm.forEach(t),om.forEach(t),Pi=c(Ma),Wl=r(Ma,"SPAN",{});var rm=a(Wl);Ii=n(rm,"Advanced usage: mixing FP4 (or Int8) and BetterTransformer"),rm.forEach(t),Ma.forEach(t),ea=c(e),to=r(e,"P",{});var am=a(to);xi=n(am,"You can combine the different methods described above to get the best performance for your model. For example, you can use BetterTransformer with FP4 mixed-precision inference + flash attention:"),am.forEach(t),ta=c(e),d(Et.$$.fragment,e),this.h()},h(){f(w,"name","hf:doc:metadata"),f(w,"content",JSON.stringify(hm)),f(_,"id","efficient-inference-on-a-single-gpu"),f(_,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(_,"href","#efficient-inference-on-a-single-gpu"),f(g,"class","relative group"),f($t,"href","perf_train_gpu_one"),f(Ut,"href","perf_infer_cpu"),f(re,"id","bettertransformer"),f(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(re,"href","#bettertransformer"),f(V,"class","relative group"),f(je,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),f(je,"rel","nofollow"),f(se,"id","encoder-models"),f(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(se,"href","#encoder-models"),f(X,"class","relative group"),f(Ce,"href","https://pytorch.org/blog/a-better-transformer-for-fast-transformer-encoder-inference/"),f(Ce,"rel","nofollow"),f(Fe,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),f(Fe,"rel","nofollow"),f(Ae,"href","https://pytorch.org/docs/stable/nested.html"),f(Ae,"rel","nofollow"),f(Ne,"href","https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2"),f(Ne,"rel","nofollow"),f(Ve,"href","https://github.com/huggingface/optimum"),f(Ve,"rel","nofollow"),f(kt,"href","/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.to_bettertransformer"),f(Gt,"href","/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.reverse_bettertransformer"),f(ze,"href","https://medium.com/pytorch/bettertransformer-out-of-the-box-performance-for-huggingface-transformers-3fbe27d50ab2"),f(ze,"rel","nofollow"),f(ie,"id","decoder-models"),f(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ie,"href","#decoder-models"),f(q,"class","relative group"),f(fe,"href","https://pytorch.org/docs/master/generated/torch.nn.functional.scaled_dot_product_attention"),f(fe,"rel","nofollow"),f(He,"href","https://arxiv.org/abs/2205.14135"),f(He,"rel","nofollow"),f(Qe,"href","https://pytorch.org/docs/master/backends.html#torch.backends.cuda.sdp_kernel"),f(Qe,"rel","nofollow"),f(Ke,"href","https://pytorch.org/blog/out-of-the-box-acceleration/"),f(Ke,"rel","nofollow"),f(me,"id","bitsandbytes-integration-for-fp4-mixedprecision-inference"),f(me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(me,"href","#bitsandbytes-integration-for-fp4-mixedprecision-inference"),f(z,"class","relative group"),f(de,"id","requirements-for-fp4-mixedprecision-inference"),f(de,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(de,"href","#requirements-for-fp4-mixedprecision-inference"),f(Y,"class","relative group"),f(ye,"id","running-fp4-models-single-gpu-setup-quickstart"),f(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ye,"href","#running-fp4-models-single-gpu-setup-quickstart"),f(S,"class","relative group"),f(we,"id","running-fp4-models-multi-gpu-setup"),f(we,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(we,"href","#running-fp4-models-multi-gpu-setup"),f(H,"class","relative group"),f(ge,"id","advanced-usage"),f(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ge,"href","#advanced-usage"),f(Q,"class","relative group"),f(Rt,"href","main_classes/quantization"),f(Me,"id","bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),f(Me,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Me,"href","#bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),f(D,"class","relative group"),f(pt,"href","https://arxiv.org/abs/2208.07339"),f(pt,"rel","nofollow"),Ea(At.src,Ni="https://cdn-uploads.huggingface.co/production/uploads/1659861207959-62441d1d9fdefb55a0b7d12c.png")||f(At,"src",Ni),f(At,"alt","HFxbitsandbytes.png"),f(ft,"href","https://arxiv.org/abs/2208.07339"),f(ft,"rel","nofollow"),f(mt,"href","https://huggingface.co/blog/hf-bitsandbytes-integration"),f(mt,"rel","nofollow"),Ea(Vt.src,Vi="https://cdn-uploads.huggingface.co/production/uploads/1660567469965-62441d1d9fdefb55a0b7d12c.gif")||f(Vt,"src",Vi),f(Vt,"alt","MixedInt8.gif"),f(Xt,"href","#colab-demos"),f($e,"id","requirements-for-int8-mixedprecision-matrix-decomposition"),f($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f($e,"href","#requirements-for-int8-mixedprecision-matrix-decomposition"),f(L,"class","relative group"),f(ke,"id","running-mixedint8-models-single-gpu-setup"),f(ke,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ke,"href","#running-mixedint8-models-single-gpu-setup"),f(K,"class","relative group"),f(Ze,"id","running-mixedint8-models-multi-gpu-setup"),f(Ze,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ze,"href","#running-mixedint8-models-multi-gpu-setup"),f(ee,"class","relative group"),f(We,"id","colab-demos"),f(We,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(We,"href","#colab-demos"),f(te,"class","relative group"),Ea(Lt.src,Xi="https://colab.research.google.com/assets/colab-badge.svg")||f(Lt,"src",Xi),f(Lt,"alt","Open In Colab: T5-11b demo"),f(gt,"href","https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing"),f(gt,"rel","nofollow"),Ea(eo.src,qi="https://colab.research.google.com/assets/colab-badge.svg")||f(eo,"src",qi),f(eo,"alt","Open In Colab: BLOOM-3b demo"),f(_t,"href","https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing"),f(_t,"rel","nofollow"),f(Be,"id","advanced-usage-mixing-fp4-or-int8-and-bettertransformer"),f(Be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Be,"href","#advanced-usage-mixing-fp4-or-int8-and-bettertransformer"),f(oe,"class","relative group")},m(e,i){o(document.head,w),p(e,M,i),p(e,g,i),o(g,_),o(_,vo),h(Pe,vo,null),o(g,Ta),o(g,yo),o(yo,$a),p(e,Fl,i),p(e,G,i),o(G,Ua),o(G,$t),o($t,Ja),o(G,ka),o(G,Ut),o(Ut,Ga),o(G,Za),p(e,Al,i),p(e,V,i),o(V,re),o(re,wo),h(Ie,wo,null),o(V,Wa),o(V,go),o(go,Ba),p(e,Nl,i),p(e,xe,i),o(xe,je),o(je,Pa),o(xe,Ia),p(e,Vl,i),p(e,Jt,i),o(Jt,xa),p(e,Xl,i),h(ae,e,i),p(e,ql,i),p(e,X,i),o(X,se),o(se,_o),h(Re,_o,null),o(X,ja),o(X,Mo),o(Mo,Ra),p(e,zl,i),p(e,Z,i),o(Z,Ca),o(Z,Ce),o(Ce,Eo),o(Eo,Fa),o(Z,Aa),o(Z,Fe),o(Fe,Na),o(Z,Va),p(e,Yl,i),p(e,W,i),o(W,Xa),o(W,Ae),o(Ae,qa),o(W,za),o(W,Ne),o(Ne,Ya),o(W,Sa),p(e,Sl,i),p(e,B,i),o(B,Ha),o(B,Ve),o(Ve,To),o(To,Qa),o(B,Da),o(B,kt),o(kt,La),o(B,Oa),p(e,Hl,i),h(Xe,e,i),p(e,Ql,i),p(e,ne,i),o(ne,Ka),o(ne,Gt),o(Gt,es),o(ne,ts),p(e,Dl,i),h(qe,e,i),p(e,Ll,i),p(e,P,i),o(P,os),o(P,ze),o(ze,ls),o(P,rs),o(P,$o),o($o,as),o(P,ss),p(e,Ol,i),p(e,q,i),o(q,ie),o(ie,Uo),h(Ye,Uo,null),o(q,ns),o(q,Jo),o(Jo,is),p(e,Kl,i),p(e,pe,i),o(pe,ps),o(pe,fe),o(fe,ko),o(ko,fs),o(fe,ms),o(pe,cs),p(e,er,i),p(e,Zt,i),o(Zt,us),p(e,tr,i),h(Se,e,i),p(e,or,i),p(e,I,i),o(I,ds),o(I,He),o(He,hs),o(I,bs),o(I,Qe),o(Qe,Go),o(Go,vs),o(I,ys),p(e,lr,i),h(De,e,i),p(e,rr,i),p(e,Wt,i),o(Wt,ws),p(e,ar,i),h(Le,e,i),p(e,sr,i),p(e,Bt,i),o(Bt,gs),p(e,nr,i),h(Oe,e,i),p(e,ir,i),p(e,x,i),o(x,_s),o(x,Ke),o(Ke,Ms),o(x,Es),o(x,Zo),o(Zo,Ts),o(x,$s),p(e,pr,i),p(e,z,i),o(z,me),o(me,Wo),h(et,Wo,null),o(z,Us),o(z,Pt),o(Pt,Bo),o(Bo,Js),o(Pt,ks),p(e,fr,i),p(e,ce,i),o(ce,Gs),o(ce,Po),o(Po,Zs),o(ce,Ws),p(e,mr,i),h(ue,e,i),p(e,cr,i),p(e,Y,i),o(Y,de),o(de,Io),h(tt,Io,null),o(Y,Bs),o(Y,xo),o(xo,Ps),p(e,ur,i),p(e,j,i),o(j,jo),o(jo,he),o(he,Is),o(he,Ro),o(Ro,xs),o(he,js),o(he,Co),o(Co,Rs),o(j,Cs),o(j,Fo),o(Fo,be),o(be,Fs),o(be,Ao),o(Ao,As),o(be,Ns),o(be,No),o(No,Vs),o(j,Xs),o(j,Vo),o(Vo,ve),o(ve,qs),o(ve,Xo),o(Xo,zs),o(ve,Ys),o(ve,qo),o(qo,Ss),p(e,dr,i),p(e,S,i),o(S,ye),o(ye,zo),h(ot,zo,null),o(S,Hs),o(S,Yo),o(Yo,Qs),p(e,hr,i),p(e,It,i),o(It,Ds),p(e,br,i),h(lt,e,i),p(e,vr,i),p(e,R,i),o(R,Ls),o(R,So),o(So,Os),o(R,Ks),o(R,Ho),o(Ho,en),o(R,tn),p(e,yr,i),p(e,H,i),o(H,we),o(we,Qo),h(rt,Qo,null),o(H,on),o(H,Do),o(Do,ln),p(e,wr,i),p(e,xt,i),o(xt,rn),p(e,gr,i),h(at,e,i),p(e,_r,i),p(e,C,i),o(C,an),o(C,Lo),o(Lo,sn),o(C,nn),o(C,Oo),o(Oo,pn),o(C,fn),p(e,Mr,i),h(st,e,i),p(e,Er,i),p(e,jt,i),o(jt,mn),p(e,Tr,i),p(e,Q,i),o(Q,ge),o(ge,Ko),h(nt,Ko,null),o(Q,cn),o(Q,el),o(el,un),p(e,$r,i),p(e,_e,i),o(_e,dn),o(_e,Rt),o(Rt,hn),o(_e,bn),p(e,Ur,i),p(e,D,i),o(D,Me),o(Me,tl),h(it,tl,null),o(D,vn),o(D,Ct),o(Ct,ol),o(ol,yn),o(Ct,wn),p(e,Jr,i),h(Ee,e,i),p(e,kr,i),p(e,E,i),o(E,gn),o(E,pt),o(pt,ll),o(ll,_n),o(E,Mn),o(E,rl),o(rl,En),o(E,Tn),o(E,al),o(al,$n),o(E,Un),o(E,sl),o(sl,Jn),o(E,kn),o(E,nl),o(nl,Gn),o(E,Zn),p(e,Gr,i),p(e,Ft,i),o(Ft,At),p(e,Zr,i),p(e,F,i),o(F,Wn),o(F,ft),o(ft,Bn),o(F,Pn),o(F,mt),o(mt,In),o(F,xn),p(e,Wr,i),p(e,Nt,i),o(Nt,Vt),p(e,Br,i),p(e,Te,i),o(Te,jn),o(Te,Xt),o(Xt,Rn),o(Te,Cn),p(e,Pr,i),p(e,L,i),o(L,$e),o($e,il),h(ct,il,null),o(L,Fn),o(L,pl),o(pl,An),p(e,Ir,i),p(e,A,i),o(A,O),o(O,Nn),o(O,fl),o(fl,Vn),o(O,Xn),o(O,ml),o(ml,qn),o(O,zn),o(A,Yn),o(A,Ue),o(Ue,Sn),o(Ue,cl),o(cl,Hn),o(Ue,Qn),o(Ue,ul),o(ul,Dn),o(A,Ln),o(A,Je),o(Je,On),o(Je,dl),o(dl,Kn),o(Je,ei),o(Je,hl),o(hl,ti),p(e,xr,i),p(e,K,i),o(K,ke),o(ke,bl),h(ut,bl,null),o(K,oi),o(K,vl),o(vl,li),p(e,jr,i),p(e,qt,i),o(qt,ri),p(e,Rr,i),h(dt,e,i),p(e,Cr,i),p(e,zt,i),o(zt,ai),p(e,Fr,i),p(e,Ge,i),o(Ge,T),o(T,si),o(T,yl),o(yl,ni),o(T,ii),o(T,wl),o(wl,pi),o(T,fi),o(T,gl),o(gl,mi),o(T,ci),o(T,_l),o(_l,ui),o(T,di),o(T,Ml),o(Ml,hi),o(T,bi),o(Ge,vi),o(Ge,El),o(El,yi),p(e,Ar,i),p(e,Yt,i),o(Yt,wi),p(e,Nr,i),h(ht,e,i),p(e,Vr,i),p(e,ee,i),o(ee,Ze),o(Ze,Tl),h(bt,Tl,null),o(ee,gi),o(ee,$l),o($l,_i),p(e,Xr,i),p(e,St,i),o(St,Mi),p(e,qr,i),h(vt,e,i),p(e,zr,i),p(e,N,i),o(N,Ei),o(N,Ul),o(Ul,Ti),o(N,$i),o(N,Jl),o(Jl,Ui),o(N,Ji),p(e,Yr,i),h(yt,e,i),p(e,Sr,i),p(e,Ht,i),o(Ht,ki),p(e,Hr,i),p(e,te,i),o(te,We),o(We,kl),h(wt,kl,null),o(te,Gi),o(te,Gl),o(Gl,Zi),p(e,Qr,i),p(e,Qt,i),o(Qt,Wi),p(e,Dr,i),p(e,Dt,i),o(Dt,gt),o(gt,Lt),p(e,Lr,i),p(e,Ot,i),o(Ot,Bi),p(e,Or,i),p(e,Kt,i),o(Kt,_t),o(_t,eo),p(e,Kr,i),p(e,oe,i),o(oe,Be),o(Be,Zl),h(Mt,Zl,null),o(oe,Pi),o(oe,Wl),o(Wl,Ii),p(e,ea,i),p(e,to,i),o(to,xi),p(e,ta,i),h(Et,e,i),oa=!0},p(e,[i]){const Tt={};i&2&&(Tt.$$scope={dirty:i,ctx:e}),ae.$set(Tt);const Bl={};i&2&&(Bl.$$scope={dirty:i,ctx:e}),ue.$set(Bl);const Pl={};i&2&&(Pl.$$scope={dirty:i,ctx:e}),Ee.$set(Pl)},i(e){oa||(b(Pe.$$.fragment,e),b(Ie.$$.fragment,e),b(ae.$$.fragment,e),b(Re.$$.fragment,e),b(Xe.$$.fragment,e),b(qe.$$.fragment,e),b(Ye.$$.fragment,e),b(Se.$$.fragment,e),b(De.$$.fragment,e),b(Le.$$.fragment,e),b(Oe.$$.fragment,e),b(et.$$.fragment,e),b(ue.$$.fragment,e),b(tt.$$.fragment,e),b(ot.$$.fragment,e),b(lt.$$.fragment,e),b(rt.$$.fragment,e),b(at.$$.fragment,e),b(st.$$.fragment,e),b(nt.$$.fragment,e),b(it.$$.fragment,e),b(Ee.$$.fragment,e),b(ct.$$.fragment,e),b(ut.$$.fragment,e),b(dt.$$.fragment,e),b(ht.$$.fragment,e),b(bt.$$.fragment,e),b(vt.$$.fragment,e),b(yt.$$.fragment,e),b(wt.$$.fragment,e),b(Mt.$$.fragment,e),b(Et.$$.fragment,e),oa=!0)},o(e){v(Pe.$$.fragment,e),v(Ie.$$.fragment,e),v(ae.$$.fragment,e),v(Re.$$.fragment,e),v(Xe.$$.fragment,e),v(qe.$$.fragment,e),v(Ye.$$.fragment,e),v(Se.$$.fragment,e),v(De.$$.fragment,e),v(Le.$$.fragment,e),v(Oe.$$.fragment,e),v(et.$$.fragment,e),v(ue.$$.fragment,e),v(tt.$$.fragment,e),v(ot.$$.fragment,e),v(lt.$$.fragment,e),v(rt.$$.fragment,e),v(at.$$.fragment,e),v(st.$$.fragment,e),v(nt.$$.fragment,e),v(it.$$.fragment,e),v(Ee.$$.fragment,e),v(ct.$$.fragment,e),v(ut.$$.fragment,e),v(dt.$$.fragment,e),v(ht.$$.fragment,e),v(bt.$$.fragment,e),v(vt.$$.fragment,e),v(yt.$$.fragment,e),v(wt.$$.fragment,e),v(Mt.$$.fragment,e),v(Et.$$.fragment,e),oa=!1},d(e){t(w),e&&t(M),e&&t(g),y(Pe),e&&t(Fl),e&&t(G),e&&t(Al),e&&t(V),y(Ie),e&&t(Nl),e&&t(xe),e&&t(Vl),e&&t(Jt),e&&t(Xl),y(ae,e),e&&t(ql),e&&t(X),y(Re),e&&t(zl),e&&t(Z),e&&t(Yl),e&&t(W),e&&t(Sl),e&&t(B),e&&t(Hl),y(Xe,e),e&&t(Ql),e&&t(ne),e&&t(Dl),y(qe,e),e&&t(Ll),e&&t(P),e&&t(Ol),e&&t(q),y(Ye),e&&t(Kl),e&&t(pe),e&&t(er),e&&t(Zt),e&&t(tr),y(Se,e),e&&t(or),e&&t(I),e&&t(lr),y(De,e),e&&t(rr),e&&t(Wt),e&&t(ar),y(Le,e),e&&t(sr),e&&t(Bt),e&&t(nr),y(Oe,e),e&&t(ir),e&&t(x),e&&t(pr),e&&t(z),y(et),e&&t(fr),e&&t(ce),e&&t(mr),y(ue,e),e&&t(cr),e&&t(Y),y(tt),e&&t(ur),e&&t(j),e&&t(dr),e&&t(S),y(ot),e&&t(hr),e&&t(It),e&&t(br),y(lt,e),e&&t(vr),e&&t(R),e&&t(yr),e&&t(H),y(rt),e&&t(wr),e&&t(xt),e&&t(gr),y(at,e),e&&t(_r),e&&t(C),e&&t(Mr),y(st,e),e&&t(Er),e&&t(jt),e&&t(Tr),e&&t(Q),y(nt),e&&t($r),e&&t(_e),e&&t(Ur),e&&t(D),y(it),e&&t(Jr),y(Ee,e),e&&t(kr),e&&t(E),e&&t(Gr),e&&t(Ft),e&&t(Zr),e&&t(F),e&&t(Wr),e&&t(Nt),e&&t(Br),e&&t(Te),e&&t(Pr),e&&t(L),y(ct),e&&t(Ir),e&&t(A),e&&t(xr),e&&t(K),y(ut),e&&t(jr),e&&t(qt),e&&t(Rr),y(dt,e),e&&t(Cr),e&&t(zt),e&&t(Fr),e&&t(Ge),e&&t(Ar),e&&t(Yt),e&&t(Nr),y(ht,e),e&&t(Vr),e&&t(ee),y(bt),e&&t(Xr),e&&t(St),e&&t(qr),y(vt,e),e&&t(zr),e&&t(N),e&&t(Yr),y(yt,e),e&&t(Sr),e&&t(Ht),e&&t(Hr),e&&t(te),y(wt),e&&t(Qr),e&&t(Qt),e&&t(Dr),e&&t(Dt),e&&t(Lr),e&&t(Ot),e&&t(Or),e&&t(Kt),e&&t(Kr),e&&t(oe),y(Mt),e&&t(ea),e&&t(to),e&&t(ta),y(Et,e)}}}const hm={local:"efficient-inference-on-a-single-gpu",sections:[{local:"bettertransformer",sections:[{local:"encoder-models",title:"Encoder models"},{local:"decoder-models",title:"Decoder models"}],title:"BetterTransformer"},{local:"bitsandbytes-integration-for-fp4-mixedprecision-inference",sections:[{local:"requirements-for-fp4-mixedprecision-inference",title:"Requirements "},{local:"running-fp4-models-single-gpu-setup-quickstart",title:"Running FP4 models - single GPU setup - Quickstart"},{local:"running-fp4-models-multi-gpu-setup",title:"Running FP4 models - multi GPU setup"},{local:"advanced-usage",title:"Advanced usage"}],title:"`bitsandbytes` integration for FP4 mixed-precision inference"},{local:"bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition",sections:[{local:"requirements-for-int8-mixedprecision-matrix-decomposition",title:"Requirements "},{local:"running-mixedint8-models-single-gpu-setup",title:"Running mixed-Int8 models - single GPU setup"},{local:"running-mixedint8-models-multi-gpu-setup",title:"Running mixed-int8 models - multi GPU setup"},{local:"colab-demos",title:"Colab demos"}],title:"`bitsandbytes` integration for Int8 mixed-precision matrix decomposition"},{local:"advanced-usage-mixing-fp4-or-int8-and-bettertransformer",title:"Advanced usage: mixing FP4 (or Int8) and BetterTransformer"}],title:"Efficient Inference on a Single GPU"};function bm(le){return fm(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class _m extends sm{constructor(w){super();nm(this,w,bm,dm,im,{})}}export{_m as default,hm as metadata};
