import{S as ol,i as rl,s as al,e as r,k as p,w,t as i,M as ll,c as a,d as o,m as u,a as l,x as y,h as s,b as m,N as fo,G as t,g as f,y as E,q as $,o as x,B as P,v as il}from"../chunks/vendor-hf-doc-builder.js";import{T as sl}from"../chunks/Tip-hf-doc-builder.js";import{I as ce}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as mo}from"../chunks/CodeBlock-hf-doc-builder.js";function nl($t){let d,B;return{c(){d=r("p"),B=i("Note that this feature can also be used in a multi GPU setup.")},l(h){d=a(h,"P",{});var b=l(d);B=s(b,"Note that this feature can also be used in a multi GPU setup."),b.forEach(o)},m(h,b){f(h,d,b),t(d,B)},d(h){h&&o(d)}}}function fl($t){let d,B,h,b,Ne,Z,po,De,uo,xt,k,co,he,ho,_o,_e,bo,vo,Pt,q,S,ze,J,go,be,He,wo,yo,kt,A,Eo,Re,$o,xo,Q,Po,ko,At,U,F,We,Y,Ao,ve,Xe,Go,Co,Gt,N,Ct,c,Mo,ee,Ve,qo,Uo,Ke,To,Io,Ze,Oo,Lo,Je,jo,Bo,Qe,So,Fo,Mt,ge,we,Rr,qt,G,No,te,Do,zo,oe,Ho,Ro,Ut,ye,Ee,Wr,Tt,D,Wo,$e,Xo,Vo,It,T,z,Ye,re,Ko,et,Zo,Ot,C,I,Jo,tt,Qo,Yo,ot,er,tr,or,H,rr,rt,ar,lr,at,ir,sr,R,nr,lt,fr,mr,it,pr,Lt,O,W,st,ae,ur,nt,dr,jt,xe,cr,Bt,le,St,Pe,hr,Ft,X,_,_r,ft,br,vr,mt,gr,wr,pt,yr,Er,ut,$r,xr,dt,Pr,kr,Ar,ct,Gr,Nt,ke,Cr,Dt,ie,zt,L,V,ht,se,Mr,_t,qr,Ht,Ae,Ur,Rt,ne,Wt,M,Tr,bt,Ir,Or,vt,Lr,jr,Xt,fe,Vt,Ge,Br,Kt,j,K,gt,me,Sr,wt,Fr,Zt,Ce,Nr,Jt,Me,pe,qe,Xr,Qt,Ue,Dr,Yt,Te,ue,Ie,Vr,eo;return Z=new ce({}),J=new ce({}),Y=new ce({}),N=new sl({props:{$$slots:{default:[nl]},$$scope:{ctx:$t}}}),re=new ce({}),ae=new ce({}),le=new mo({props:{code:`from transformers import AutoModelForCausalLM

model_name = "bigscience/bloom-2b5"
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),ie=new mo({props:{code:`from transformers import AutoModelForCausalLM, AutoTokenizer

model_name = "bigscience/bloom-2b5"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)

text = "Hello, my llama is cute"
inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
generated_ids = model.generate(**inputs)
outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)`,highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_name)
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)

text = <span class="hljs-string">&quot;Hello, my llama is cute&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
generated_ids = model.generate(**inputs)
outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)`}}),se=new ce({}),ne=new mo({props:{code:`model_name = "bigscience/bloom-2b5"
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", load_in_8bit=True)`,highlighted:`model_name = <span class="hljs-string">&quot;bigscience/bloom-2b5&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),fe=new mo({props:{code:`max_memory_mapping = {0: "1GB", 1: "2GB"}
model_name = "bigscience/bloom-3b"
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map="auto", load_in_8bit=True, max_memory=max_memory_mapping
)`,highlighted:`max_memory_mapping = {<span class="hljs-number">0</span>: <span class="hljs-string">&quot;1GB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;2GB&quot;</span>}
model_name = <span class="hljs-string">&quot;bigscience/bloom-3b&quot;</span>
model_8bit = AutoModelForCausalLM.from_pretrained(
    model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, max_memory=max_memory_mapping
)`}}),me=new ce({}),{c(){d=r("meta"),B=p(),h=r("h1"),b=r("a"),Ne=r("span"),w(Z.$$.fragment),po=p(),De=r("span"),uo=i("Efficient Inference on a Single GPU"),xt=p(),k=r("p"),co=i("This document will be completed soon with information on how to infer on a single GPU. In the meantime you can check out "),he=r("a"),ho=i("the guide for training on a single GPU"),_o=i(" and "),_e=r("a"),bo=i("the guide for inference on CPUs"),vo=i("."),Pt=p(),q=r("h2"),S=r("a"),ze=r("span"),w(J.$$.fragment),go=p(),be=r("span"),He=r("code"),wo=i("BetterTransformer"),yo=i(" for faster inference"),kt=p(),A=r("p"),Eo=i("We have recently integrated "),Re=r("code"),$o=i("BetterTransformer"),xo=i(" for faster inference on GPU for text, image and audio models. Check the documentation about this integration "),Q=r("a"),Po=i("here"),ko=i(" for more details."),At=p(),U=r("h2"),F=r("a"),We=r("span"),w(Y.$$.fragment),Ao=p(),ve=r("span"),Xe=r("code"),Go=i("bitsandbytes"),Co=i(" integration for Int8 mixed-precision matrix decomposition"),Gt=p(),w(N.$$.fragment),Ct=p(),c=r("p"),Mo=i("From the paper "),ee=r("a"),Ve=r("code"),qo=i("LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),Uo=i(`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),Ke=r("code"),To=i("nn.Linear"),Io=i(" size by 2 for "),Ze=r("code"),Oo=i("float16"),Lo=i(" and "),Je=r("code"),jo=i("bfloat16"),Bo=i(" weights and by 4 for "),Qe=r("code"),So=i("float32"),Fo=i(" weights, with close to no impact to the quality by operating on the outliers in half-precision."),Mt=p(),ge=r("p"),we=r("img"),qt=p(),G=r("p"),No=i(`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),te=r("a"),Do=i("paper"),zo=i(" or our "),oe=r("a"),Ho=i("blogpost about the integration"),Ro=i("."),Ut=p(),ye=r("p"),Ee=r("img"),Tt=p(),D=r("p"),Wo=i(`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),$e=r("a"),Xo=i("Google colab"),Vo=i("."),It=p(),T=r("h3"),z=r("a"),Ye=r("span"),w(re.$$.fragment),Ko=p(),et=r("span"),Zo=i("Requirements"),Ot=p(),C=r("ul"),I=r("li"),Jo=i("If you have "),tt=r("code"),Qo=i("bitsandbytes<0.37.0"),Yo=i(", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),ot=r("code"),er=i("bitsandbytes>=0.37.0"),tr=i(", all GPUs should be supported."),or=p(),H=r("li"),rr=i("Install the correct version of "),rt=r("code"),ar=i("bitsandbytes"),lr=i(` by running:
`),at=r("code"),ir=i("pip install bitsandbytes>=0.31.5"),sr=p(),R=r("li"),nr=i("Install "),lt=r("code"),fr=i("accelerate"),mr=p(),it=r("code"),pr=i("pip install accelerate>=0.12.0"),Lt=p(),O=r("h3"),W=r("a"),st=r("span"),w(ae.$$.fragment),ur=p(),nt=r("span"),dr=i("Running mixed-Int8 models - single GPU setup"),jt=p(),xe=r("p"),cr=i("After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),Bt=p(),w(le.$$.fragment),St=p(),Pe=r("p"),hr=i("For text generation, we recommend:"),Ft=p(),X=r("ul"),_=r("li"),_r=i("using the model\u2019s "),ft=r("code"),br=i("generate()"),vr=i(" method instead of the "),mt=r("code"),gr=i("pipeline()"),wr=i(" function. Although inference is possible with the "),pt=r("code"),yr=i("pipeline()"),Er=i(" function, it is not optimized for mixed-8bit models, and will be slower than using the "),ut=r("code"),$r=i("generate()"),xr=i(" method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),dt=r("code"),Pr=i("pipeline()"),kr=i(" function for mixed-8bit models."),Ar=p(),ct=r("li"),Gr=i("placing all inputs on the same device as the model."),Nt=p(),ke=r("p"),Cr=i("Here is a simple example:"),Dt=p(),w(ie.$$.fragment),zt=p(),L=r("h3"),V=r("a"),ht=r("span"),w(se.$$.fragment),Mr=p(),_t=r("span"),qr=i("Running mixed-int8 models - multi GPU setup"),Ht=p(),Ae=r("p"),Ur=i("The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),Rt=p(),w(ne.$$.fragment),Wt=p(),M=r("p"),Tr=i("But you can control the GPU RAM you want to allocate on each GPU using "),bt=r("code"),Ir=i("accelerate"),Or=i(". Use the "),vt=r("code"),Lr=i("max_memory"),jr=i(" argument as follows:"),Xt=p(),w(fe.$$.fragment),Vt=p(),Ge=r("p"),Br=i("In this example, the first GPU will use 1GB of memory and the second 2GB."),Kt=p(),j=r("h3"),K=r("a"),gt=r("span"),w(me.$$.fragment),Sr=p(),wt=r("span"),Fr=i("Colab demos"),Zt=p(),Ce=r("p"),Nr=i(`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),Jt=p(),Me=r("p"),pe=r("a"),qe=r("img"),Qt=p(),Ue=r("p"),Dr=i("Or this demo for BLOOM-3B:"),Yt=p(),Te=r("p"),ue=r("a"),Ie=r("img"),this.h()},l(e){const n=ll('[data-svelte="svelte-1phssyn"]',document.head);d=a(n,"META",{name:!0,content:!0}),n.forEach(o),B=u(e),h=a(e,"H1",{class:!0});var de=l(h);b=a(de,"A",{id:!0,class:!0,href:!0});var Kr=l(b);Ne=a(Kr,"SPAN",{});var Zr=l(Ne);y(Z.$$.fragment,Zr),Zr.forEach(o),Kr.forEach(o),po=u(de),De=a(de,"SPAN",{});var Jr=l(De);uo=s(Jr,"Efficient Inference on a Single GPU"),Jr.forEach(o),de.forEach(o),xt=u(e),k=a(e,"P",{});var Oe=l(k);co=s(Oe,"This document will be completed soon with information on how to infer on a single GPU. In the meantime you can check out "),he=a(Oe,"A",{href:!0});var Qr=l(he);ho=s(Qr,"the guide for training on a single GPU"),Qr.forEach(o),_o=s(Oe," and "),_e=a(Oe,"A",{href:!0});var Yr=l(_e);bo=s(Yr,"the guide for inference on CPUs"),Yr.forEach(o),vo=s(Oe,"."),Oe.forEach(o),Pt=u(e),q=a(e,"H2",{class:!0});var to=l(q);S=a(to,"A",{id:!0,class:!0,href:!0});var ea=l(S);ze=a(ea,"SPAN",{});var ta=l(ze);y(J.$$.fragment,ta),ta.forEach(o),ea.forEach(o),go=u(to),be=a(to,"SPAN",{});var zr=l(be);He=a(zr,"CODE",{});var oa=l(He);wo=s(oa,"BetterTransformer"),oa.forEach(o),yo=s(zr," for faster inference"),zr.forEach(o),to.forEach(o),kt=u(e),A=a(e,"P",{});var Le=l(A);Eo=s(Le,"We have recently integrated "),Re=a(Le,"CODE",{});var ra=l(Re);$o=s(ra,"BetterTransformer"),ra.forEach(o),xo=s(Le," for faster inference on GPU for text, image and audio models. Check the documentation about this integration "),Q=a(Le,"A",{href:!0,rel:!0});var aa=l(Q);Po=s(aa,"here"),aa.forEach(o),ko=s(Le," for more details."),Le.forEach(o),At=u(e),U=a(e,"H2",{class:!0});var oo=l(U);F=a(oo,"A",{id:!0,class:!0,href:!0});var la=l(F);We=a(la,"SPAN",{});var ia=l(We);y(Y.$$.fragment,ia),ia.forEach(o),la.forEach(o),Ao=u(oo),ve=a(oo,"SPAN",{});var Hr=l(ve);Xe=a(Hr,"CODE",{});var sa=l(Xe);Go=s(sa,"bitsandbytes"),sa.forEach(o),Co=s(Hr," integration for Int8 mixed-precision matrix decomposition"),Hr.forEach(o),oo.forEach(o),Gt=u(e),y(N.$$.fragment,e),Ct=u(e),c=a(e,"P",{});var v=l(c);Mo=s(v,"From the paper "),ee=a(v,"A",{href:!0,rel:!0});var na=l(ee);Ve=a(na,"CODE",{});var fa=l(Ve);qo=s(fa,"LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale"),fa.forEach(o),na.forEach(o),Uo=s(v,`, we support Hugging Face integration for all models in the Hub with a few lines of code.
The method reduces `),Ke=a(v,"CODE",{});var ma=l(Ke);To=s(ma,"nn.Linear"),ma.forEach(o),Io=s(v," size by 2 for "),Ze=a(v,"CODE",{});var pa=l(Ze);Oo=s(pa,"float16"),pa.forEach(o),Lo=s(v," and "),Je=a(v,"CODE",{});var ua=l(Je);jo=s(ua,"bfloat16"),ua.forEach(o),Bo=s(v," weights and by 4 for "),Qe=a(v,"CODE",{});var da=l(Qe);So=s(da,"float32"),da.forEach(o),Fo=s(v," weights, with close to no impact to the quality by operating on the outliers in half-precision."),v.forEach(o),Mt=u(e),ge=a(e,"P",{});var ca=l(ge);we=a(ca,"IMG",{src:!0,alt:!0}),ca.forEach(o),qt=u(e),G=a(e,"P",{});var je=l(G);No=s(je,`Int8 mixed-precision matrix decomposition works by separating a matrix multiplication into two streams: (1) a systematic feature outlier stream matrix multiplied in fp16 (0.01%), (2) a regular stream of int8 matrix multiplication (99.9%). With this method, int8 inference with no predictive degradation is possible for very large models.
For more details regarding the method, check out the `),te=a(je,"A",{href:!0,rel:!0});var ha=l(te);Do=s(ha,"paper"),ha.forEach(o),zo=s(je," or our "),oe=a(je,"A",{href:!0,rel:!0});var _a=l(oe);Ho=s(_a,"blogpost about the integration"),_a.forEach(o),Ro=s(je,"."),je.forEach(o),Ut=u(e),ye=a(e,"P",{});var ba=l(ye);Ee=a(ba,"IMG",{src:!0,alt:!0}),ba.forEach(o),Tt=u(e),D=a(e,"P",{});var ro=l(D);Wo=s(ro,`Note, that you would require a GPU to run mixed-8bit models as the kernels have been compiled for GPUs only. Make sure that you have enough GPU memory to store the quarter (or half if your model weights are in half precision) of the model before using this feature.
Below are some notes to help you use this module, or follow the demos on `),$e=a(ro,"A",{href:!0});var va=l($e);Xo=s(va,"Google colab"),va.forEach(o),Vo=s(ro,"."),ro.forEach(o),It=u(e),T=a(e,"H3",{class:!0});var ao=l(T);z=a(ao,"A",{id:!0,class:!0,href:!0});var ga=l(z);Ye=a(ga,"SPAN",{});var wa=l(Ye);y(re.$$.fragment,wa),wa.forEach(o),ga.forEach(o),Ko=u(ao),et=a(ao,"SPAN",{});var ya=l(et);Zo=s(ya,"Requirements"),ya.forEach(o),ao.forEach(o),Ot=u(e),C=a(e,"UL",{});var Be=l(C);I=a(Be,"LI",{});var Se=l(I);Jo=s(Se,"If you have "),tt=a(Se,"CODE",{});var Ea=l(tt);Qo=s(Ea,"bitsandbytes<0.37.0"),Ea.forEach(o),Yo=s(Se,", make sure you run on NVIDIA GPUs that support 8-bit tensor cores (Turing, Ampere or newer architectures - e.g. T4, RTX20s RTX30s, A40-A100). For "),ot=a(Se,"CODE",{});var $a=l(ot);er=s($a,"bitsandbytes>=0.37.0"),$a.forEach(o),tr=s(Se,", all GPUs should be supported."),Se.forEach(o),or=u(Be),H=a(Be,"LI",{});var yt=l(H);rr=s(yt,"Install the correct version of "),rt=a(yt,"CODE",{});var xa=l(rt);ar=s(xa,"bitsandbytes"),xa.forEach(o),lr=s(yt,` by running:
`),at=a(yt,"CODE",{});var Pa=l(at);ir=s(Pa,"pip install bitsandbytes>=0.31.5"),Pa.forEach(o),yt.forEach(o),sr=u(Be),R=a(Be,"LI",{});var Et=l(R);nr=s(Et,"Install "),lt=a(Et,"CODE",{});var ka=l(lt);fr=s(ka,"accelerate"),ka.forEach(o),mr=u(Et),it=a(Et,"CODE",{});var Aa=l(it);pr=s(Aa,"pip install accelerate>=0.12.0"),Aa.forEach(o),Et.forEach(o),Be.forEach(o),Lt=u(e),O=a(e,"H3",{class:!0});var lo=l(O);W=a(lo,"A",{id:!0,class:!0,href:!0});var Ga=l(W);st=a(Ga,"SPAN",{});var Ca=l(st);y(ae.$$.fragment,Ca),Ca.forEach(o),Ga.forEach(o),ur=u(lo),nt=a(lo,"SPAN",{});var Ma=l(nt);dr=s(Ma,"Running mixed-Int8 models - single GPU setup"),Ma.forEach(o),lo.forEach(o),jt=u(e),xe=a(e,"P",{});var qa=l(xe);cr=s(qa,"After installing the required libraries, the way to load your mixed 8-bit model is as follows:"),qa.forEach(o),Bt=u(e),y(le.$$.fragment,e),St=u(e),Pe=a(e,"P",{});var Ua=l(Pe);hr=s(Ua,"For text generation, we recommend:"),Ua.forEach(o),Ft=u(e),X=a(e,"UL",{});var io=l(X);_=a(io,"LI",{});var g=l(_);_r=s(g,"using the model\u2019s "),ft=a(g,"CODE",{});var Ta=l(ft);br=s(Ta,"generate()"),Ta.forEach(o),vr=s(g," method instead of the "),mt=a(g,"CODE",{});var Ia=l(mt);gr=s(Ia,"pipeline()"),Ia.forEach(o),wr=s(g," function. Although inference is possible with the "),pt=a(g,"CODE",{});var Oa=l(pt);yr=s(Oa,"pipeline()"),Oa.forEach(o),Er=s(g," function, it is not optimized for mixed-8bit models, and will be slower than using the "),ut=a(g,"CODE",{});var La=l(ut);$r=s(La,"generate()"),La.forEach(o),xr=s(g," method. Moreover, some sampling strategies are like nucleaus sampling are not supported by the "),dt=a(g,"CODE",{});var ja=l(dt);Pr=s(ja,"pipeline()"),ja.forEach(o),kr=s(g," function for mixed-8bit models."),g.forEach(o),Ar=u(io),ct=a(io,"LI",{});var Ba=l(ct);Gr=s(Ba,"placing all inputs on the same device as the model."),Ba.forEach(o),io.forEach(o),Nt=u(e),ke=a(e,"P",{});var Sa=l(ke);Cr=s(Sa,"Here is a simple example:"),Sa.forEach(o),Dt=u(e),y(ie.$$.fragment,e),zt=u(e),L=a(e,"H3",{class:!0});var so=l(L);V=a(so,"A",{id:!0,class:!0,href:!0});var Fa=l(V);ht=a(Fa,"SPAN",{});var Na=l(ht);y(se.$$.fragment,Na),Na.forEach(o),Fa.forEach(o),Mr=u(so),_t=a(so,"SPAN",{});var Da=l(_t);qr=s(Da,"Running mixed-int8 models - multi GPU setup"),Da.forEach(o),so.forEach(o),Ht=u(e),Ae=a(e,"P",{});var za=l(Ae);Ur=s(za,"The way to load your mixed 8-bit model in multiple GPUs is as follows (same command as single GPU setup):"),za.forEach(o),Rt=u(e),y(ne.$$.fragment,e),Wt=u(e),M=a(e,"P",{});var Fe=l(M);Tr=s(Fe,"But you can control the GPU RAM you want to allocate on each GPU using "),bt=a(Fe,"CODE",{});var Ha=l(bt);Ir=s(Ha,"accelerate"),Ha.forEach(o),Or=s(Fe,". Use the "),vt=a(Fe,"CODE",{});var Ra=l(vt);Lr=s(Ra,"max_memory"),Ra.forEach(o),jr=s(Fe," argument as follows:"),Fe.forEach(o),Xt=u(e),y(fe.$$.fragment,e),Vt=u(e),Ge=a(e,"P",{});var Wa=l(Ge);Br=s(Wa,"In this example, the first GPU will use 1GB of memory and the second 2GB."),Wa.forEach(o),Kt=u(e),j=a(e,"H3",{class:!0});var no=l(j);K=a(no,"A",{id:!0,class:!0,href:!0});var Xa=l(K);gt=a(Xa,"SPAN",{});var Va=l(gt);y(me.$$.fragment,Va),Va.forEach(o),Xa.forEach(o),Sr=u(no),wt=a(no,"SPAN",{});var Ka=l(wt);Fr=s(Ka,"Colab demos"),Ka.forEach(o),no.forEach(o),Zt=u(e),Ce=a(e,"P",{});var Za=l(Ce);Nr=s(Za,`With this method you can infer on models that were not possible to infer on a Google Colab before.
Check out the demo for running T5-11b (42GB in fp32)! Using 8-bit quantization on Google Colab:`),Za.forEach(o),Jt=u(e),Me=a(e,"P",{});var Ja=l(Me);pe=a(Ja,"A",{href:!0,rel:!0});var Qa=l(pe);qe=a(Qa,"IMG",{src:!0,alt:!0}),Qa.forEach(o),Ja.forEach(o),Qt=u(e),Ue=a(e,"P",{});var Ya=l(Ue);Dr=s(Ya,"Or this demo for BLOOM-3B:"),Ya.forEach(o),Yt=u(e),Te=a(e,"P",{});var el=l(Te);ue=a(el,"A",{href:!0,rel:!0});var tl=l(ue);Ie=a(tl,"IMG",{src:!0,alt:!0}),tl.forEach(o),el.forEach(o),this.h()},h(){m(d,"name","hf:doc:metadata"),m(d,"content",JSON.stringify(ml)),m(b,"id","efficient-inference-on-a-single-gpu"),m(b,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(b,"href","#efficient-inference-on-a-single-gpu"),m(h,"class","relative group"),m(he,"href","perf_train_gpu_one"),m(_e,"href","perf_infer_cpu"),m(S,"id","bettertransformer-for-faster-inference"),m(S,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(S,"href","#bettertransformer-for-faster-inference"),m(q,"class","relative group"),m(Q,"href","https://huggingface.co/docs/optimum/bettertransformer/overview"),m(Q,"rel","nofollow"),m(F,"id","bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),m(F,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(F,"href","#bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition"),m(U,"class","relative group"),m(ee,"href","https://arxiv.org/abs/2208.07339"),m(ee,"rel","nofollow"),fo(we.src,Rr="https://s3.amazonaws.com/moonup/production/uploads/1659861207959-62441d1d9fdefb55a0b7d12c.png")||m(we,"src",Rr),m(we,"alt","HFxbitsandbytes.png"),m(te,"href","https://arxiv.org/abs/2208.07339"),m(te,"rel","nofollow"),m(oe,"href","https://huggingface.co/blog/hf-bitsandbytes-integration"),m(oe,"rel","nofollow"),fo(Ee.src,Wr="https://s3.amazonaws.com/moonup/production/uploads/1660567469965-62441d1d9fdefb55a0b7d12c.gif")||m(Ee,"src",Wr),m(Ee,"alt","MixedInt8.gif"),m($e,"href","#colab-demos"),m(z,"id","requirements"),m(z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(z,"href","#requirements"),m(T,"class","relative group"),m(W,"id","running-mixedint8-models-single-gpu-setup"),m(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(W,"href","#running-mixedint8-models-single-gpu-setup"),m(O,"class","relative group"),m(V,"id","running-mixedint8-models-multi-gpu-setup"),m(V,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(V,"href","#running-mixedint8-models-multi-gpu-setup"),m(L,"class","relative group"),m(K,"id","colab-demos"),m(K,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(K,"href","#colab-demos"),m(j,"class","relative group"),fo(qe.src,Xr="https://colab.research.google.com/assets/colab-badge.svg")||m(qe,"src",Xr),m(qe,"alt","Open In Colab: T5-11b demo"),m(pe,"href","https://colab.research.google.com/drive/1YORPWx4okIHXnjW7MSAidXN29mPVNT7F?usp=sharing"),m(pe,"rel","nofollow"),fo(Ie.src,Vr="https://colab.research.google.com/assets/colab-badge.svg")||m(Ie,"src",Vr),m(Ie,"alt","Open In Colab: BLOOM-3b demo"),m(ue,"href","https://colab.research.google.com/drive/1qOjXfQIAULfKvZqwCen8-MoWKGdSatZ4?usp=sharing"),m(ue,"rel","nofollow")},m(e,n){t(document.head,d),f(e,B,n),f(e,h,n),t(h,b),t(b,Ne),E(Z,Ne,null),t(h,po),t(h,De),t(De,uo),f(e,xt,n),f(e,k,n),t(k,co),t(k,he),t(he,ho),t(k,_o),t(k,_e),t(_e,bo),t(k,vo),f(e,Pt,n),f(e,q,n),t(q,S),t(S,ze),E(J,ze,null),t(q,go),t(q,be),t(be,He),t(He,wo),t(be,yo),f(e,kt,n),f(e,A,n),t(A,Eo),t(A,Re),t(Re,$o),t(A,xo),t(A,Q),t(Q,Po),t(A,ko),f(e,At,n),f(e,U,n),t(U,F),t(F,We),E(Y,We,null),t(U,Ao),t(U,ve),t(ve,Xe),t(Xe,Go),t(ve,Co),f(e,Gt,n),E(N,e,n),f(e,Ct,n),f(e,c,n),t(c,Mo),t(c,ee),t(ee,Ve),t(Ve,qo),t(c,Uo),t(c,Ke),t(Ke,To),t(c,Io),t(c,Ze),t(Ze,Oo),t(c,Lo),t(c,Je),t(Je,jo),t(c,Bo),t(c,Qe),t(Qe,So),t(c,Fo),f(e,Mt,n),f(e,ge,n),t(ge,we),f(e,qt,n),f(e,G,n),t(G,No),t(G,te),t(te,Do),t(G,zo),t(G,oe),t(oe,Ho),t(G,Ro),f(e,Ut,n),f(e,ye,n),t(ye,Ee),f(e,Tt,n),f(e,D,n),t(D,Wo),t(D,$e),t($e,Xo),t(D,Vo),f(e,It,n),f(e,T,n),t(T,z),t(z,Ye),E(re,Ye,null),t(T,Ko),t(T,et),t(et,Zo),f(e,Ot,n),f(e,C,n),t(C,I),t(I,Jo),t(I,tt),t(tt,Qo),t(I,Yo),t(I,ot),t(ot,er),t(I,tr),t(C,or),t(C,H),t(H,rr),t(H,rt),t(rt,ar),t(H,lr),t(H,at),t(at,ir),t(C,sr),t(C,R),t(R,nr),t(R,lt),t(lt,fr),t(R,mr),t(R,it),t(it,pr),f(e,Lt,n),f(e,O,n),t(O,W),t(W,st),E(ae,st,null),t(O,ur),t(O,nt),t(nt,dr),f(e,jt,n),f(e,xe,n),t(xe,cr),f(e,Bt,n),E(le,e,n),f(e,St,n),f(e,Pe,n),t(Pe,hr),f(e,Ft,n),f(e,X,n),t(X,_),t(_,_r),t(_,ft),t(ft,br),t(_,vr),t(_,mt),t(mt,gr),t(_,wr),t(_,pt),t(pt,yr),t(_,Er),t(_,ut),t(ut,$r),t(_,xr),t(_,dt),t(dt,Pr),t(_,kr),t(X,Ar),t(X,ct),t(ct,Gr),f(e,Nt,n),f(e,ke,n),t(ke,Cr),f(e,Dt,n),E(ie,e,n),f(e,zt,n),f(e,L,n),t(L,V),t(V,ht),E(se,ht,null),t(L,Mr),t(L,_t),t(_t,qr),f(e,Ht,n),f(e,Ae,n),t(Ae,Ur),f(e,Rt,n),E(ne,e,n),f(e,Wt,n),f(e,M,n),t(M,Tr),t(M,bt),t(bt,Ir),t(M,Or),t(M,vt),t(vt,Lr),t(M,jr),f(e,Xt,n),E(fe,e,n),f(e,Vt,n),f(e,Ge,n),t(Ge,Br),f(e,Kt,n),f(e,j,n),t(j,K),t(K,gt),E(me,gt,null),t(j,Sr),t(j,wt),t(wt,Fr),f(e,Zt,n),f(e,Ce,n),t(Ce,Nr),f(e,Jt,n),f(e,Me,n),t(Me,pe),t(pe,qe),f(e,Qt,n),f(e,Ue,n),t(Ue,Dr),f(e,Yt,n),f(e,Te,n),t(Te,ue),t(ue,Ie),eo=!0},p(e,[n]){const de={};n&2&&(de.$$scope={dirty:n,ctx:e}),N.$set(de)},i(e){eo||($(Z.$$.fragment,e),$(J.$$.fragment,e),$(Y.$$.fragment,e),$(N.$$.fragment,e),$(re.$$.fragment,e),$(ae.$$.fragment,e),$(le.$$.fragment,e),$(ie.$$.fragment,e),$(se.$$.fragment,e),$(ne.$$.fragment,e),$(fe.$$.fragment,e),$(me.$$.fragment,e),eo=!0)},o(e){x(Z.$$.fragment,e),x(J.$$.fragment,e),x(Y.$$.fragment,e),x(N.$$.fragment,e),x(re.$$.fragment,e),x(ae.$$.fragment,e),x(le.$$.fragment,e),x(ie.$$.fragment,e),x(se.$$.fragment,e),x(ne.$$.fragment,e),x(fe.$$.fragment,e),x(me.$$.fragment,e),eo=!1},d(e){o(d),e&&o(B),e&&o(h),P(Z),e&&o(xt),e&&o(k),e&&o(Pt),e&&o(q),P(J),e&&o(kt),e&&o(A),e&&o(At),e&&o(U),P(Y),e&&o(Gt),P(N,e),e&&o(Ct),e&&o(c),e&&o(Mt),e&&o(ge),e&&o(qt),e&&o(G),e&&o(Ut),e&&o(ye),e&&o(Tt),e&&o(D),e&&o(It),e&&o(T),P(re),e&&o(Ot),e&&o(C),e&&o(Lt),e&&o(O),P(ae),e&&o(jt),e&&o(xe),e&&o(Bt),P(le,e),e&&o(St),e&&o(Pe),e&&o(Ft),e&&o(X),e&&o(Nt),e&&o(ke),e&&o(Dt),P(ie,e),e&&o(zt),e&&o(L),P(se),e&&o(Ht),e&&o(Ae),e&&o(Rt),P(ne,e),e&&o(Wt),e&&o(M),e&&o(Xt),P(fe,e),e&&o(Vt),e&&o(Ge),e&&o(Kt),e&&o(j),P(me),e&&o(Zt),e&&o(Ce),e&&o(Jt),e&&o(Me),e&&o(Qt),e&&o(Ue),e&&o(Yt),e&&o(Te)}}}const ml={local:"efficient-inference-on-a-single-gpu",sections:[{local:"bettertransformer-for-faster-inference",title:"`BetterTransformer` for faster inference"},{local:"bitsandbytes-integration-for-int8-mixedprecision-matrix-decomposition",sections:[{local:"requirements",title:"Requirements"},{local:"running-mixedint8-models-single-gpu-setup",title:"Running mixed-Int8 models - single GPU setup"},{local:"running-mixedint8-models-multi-gpu-setup",title:"Running mixed-int8 models - multi GPU setup"},{local:"colab-demos",title:"Colab demos"}],title:"`bitsandbytes` integration for Int8 mixed-precision matrix decomposition"}],title:"Efficient Inference on a Single GPU"};function pl($t){return il(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class _l extends ol{constructor(d){super();rl(this,d,pl,fl,al,{})}}export{_l as default,ml as metadata};
