import{S as i0,i as d0,s as h0,e as a,k as h,w as c,t as s,M as u0,c as l,d as o,m as u,a as r,x as p,h as n,b as f,G as t,g as d,y as m,q as b,o as v,B as y,v as f0}from"../../chunks/vendor-hf-doc-builder.js";import{T as Qu}from"../../chunks/Tip-hf-doc-builder.js";import{D as el}from"../../chunks/Docstring-hf-doc-builder.js";import{C as E}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as g}from"../../chunks/IconCopyLink-hf-doc-builder.js";function c0(K){let _;return{c(){_=s("GPTQ quantization only works for text model for now. Futhermore, the quantization process can a lot of time depending on one's hardware (175B model = 4 gpu hours using NVIDIA A100). Please check on the hub if there is not a GPTQ quantized version of the model. If not, you can submit a demand on github.")},l($){_=n($,"GPTQ quantization only works for text model for now. Futhermore, the quantization process can a lot of time depending on one's hardware (175B model = 4 gpu hours using NVIDIA A100). Please check on the hub if there is not a GPTQ quantized version of the model. If not, you can submit a demand on github.")},m($,w){d($,_,w)},d($){$&&o(_)}}}function p0(K){let _,$;return{c(){_=a("p"),$=s("Note that once a model has been loaded in 4-bit it is currently not possible to push the quantized weights on the Hub. Note also that you cannot train 4-bit weights as this is not supported yet. However you can use 4-bit models to train extra parameters, this will be covered in the next section.")},l(w){_=l(w,"P",{});var k=r(_);$=n(k,"Note that once a model has been loaded in 4-bit it is currently not possible to push the quantized weights on the Hub. Note also that you cannot train 4-bit weights as this is not supported yet. However you can use 4-bit models to train extra parameters, this will be covered in the next section."),k.forEach(o)},m(w,k){d(w,_,k),t(_,$)},d(w){w&&o(_)}}}function m0(K){let _,$,w,k,$e,j,Eo,ke,Me,dt,Q,A,Te,H;return{c(){_=a("p"),$=s("Note that once a model has been loaded in 8-bit it is currently not possible to push the quantized weights on the Hub except if you use the latest "),w=a("code"),k=s("transformers"),$e=s(" and "),j=a("code"),Eo=s("bitsandbytes"),ke=s(`. Note also that you cannot train 8-bit weights as this is not supported yet. However you can use 8-bit models to train extra parameters, this will be covered in the next section.
Note also that `),Me=a("code"),dt=s("device_map"),Q=s(" is optional but setting "),A=a("code"),Te=s("device_map = 'auto'"),H=s(" is prefered for inference as it will dispatch efficiently the model on the available ressources.")},l(Ce){_=l(Ce,"P",{});var T=r(_);$=n(T,"Note that once a model has been loaded in 8-bit it is currently not possible to push the quantized weights on the Hub except if you use the latest "),w=l(T,"CODE",{});var ht=r(w);k=n(ht,"transformers"),ht.forEach(o),$e=n(T," and "),j=l(T,"CODE",{});var tl=r(j);Eo=n(tl,"bitsandbytes"),tl.forEach(o),ke=n(T,`. Note also that you cannot train 8-bit weights as this is not supported yet. However you can use 8-bit models to train extra parameters, this will be covered in the next section.
Note also that `),Me=l(T,"CODE",{});var ol=r(Me);dt=n(ol,"device_map"),ol.forEach(o),Q=n(T," is optional but setting "),A=l(T,"CODE",{});var $o=r(A);Te=n($o,"device_map = 'auto'"),$o.forEach(o),H=n(T," is prefered for inference as it will dispatch efficiently the model on the available ressources."),T.forEach(o)},m(Ce,T){d(Ce,_,T),t(_,$),t(_,w),t(w,k),t(_,$e),t(_,j),t(j,Eo),t(_,ke),t(_,Me),t(Me,dt),t(_,Q),t(_,A),t(A,Te),t(_,H)},d(Ce){Ce&&o(_)}}}function b0(K){let _,$;return{c(){_=a("p"),$=s("Pushing 8bit models on the Hub is strongely encouraged for large models. This will allow the community to benefit from the memory footprint reduction and loading for example large models on a Google Colab.")},l(w){_=l(w,"P",{});var k=r(_);$=n(k,"Pushing 8bit models on the Hub is strongely encouraged for large models. This will allow the community to benefit from the memory footprint reduction and loading for example large models on a Google Colab."),k.forEach(o)},m(w,k){d(w,_,k),t(_,$)},d(w){w&&o(_)}}}function v0(K){let _,$,w,k,$e,j,Eo,ke,Me,dt,Q,A,Te,H,Ce,T,ht,tl,ol,$o,ut,Bu,Vl,Fu,xu,Ai,al,Xu,Ni,ee,ko,Iu,Mo,Ru,Vu,Du,qe,Yu,Dl,Su,Hu,To,Lu,Ou,Ku,Co,ef,qo,Yl,tf,of,Ui,ze,ft,Sl,zo,af,Hl,lf,Wi,ll,rf,Zi,F,Ll,ct,sf,Ol,nf,df,Kl,hf,uf,er,pt,ff,tr,cf,pf,or,mf,bf,ar,mt,vf,lr,yf,_f,rr,gf,wf,sr,bt,Ef,nr,$f,kf,ir,Mf,Pi,rl,Tf,Gi,Je,vt,dr,Jo,Cf,hr,qf,ji,yt,zf,ur,Jf,Af,Qi,sl,Nf,Bi,Ae,_t,fr,Ao,Uf,cr,Wf,Fi,N,Zf,nl,Pf,Gf,pr,jf,Qf,mr,Bf,Ff,br,xf,Xf,xi,No,Xi,il,If,Ii,Uo,Ri,Ne,gt,vr,Wo,Rf,yr,Vf,Vi,te,Df,_r,Yf,Sf,gr,Hf,Lf,Di,Zo,Yi,dl,Of,Si,wt,Kf,wr,ec,tc,Hi,Po,Li,C,oc,Er,ac,lc,$r,rc,sc,Go,nc,ic,kr,dc,hc,Mr,uc,fc,Oi,Et,Ki,Ue,$t,Tr,jo,cc,Cr,pc,ed,kt,mc,qr,bc,vc,td,Qo,od,Mt,yc,zr,_c,gc,ad,Bo,ld,oe,wc,Jr,Ec,$c,Ar,kc,Mc,rd,Fo,sd,We,Tt,Nr,xo,Tc,Ur,Cc,nd,ae,qc,Wr,zc,Jc,Zr,Ac,Nc,id,Xo,dd,le,Uc,Pr,Wc,Zc,Gr,Pc,Gc,hd,Io,ud,Ze,Ct,jr,Ro,jc,Qr,Qc,fd,re,Bc,Br,Fc,xc,hl,Xc,Ic,cd,Vo,pd,ul,Rc,md,Pe,qt,Fr,Do,Vc,xr,Dc,bd,zt,Yc,Yo,Xr,Sc,Hc,vd,Ge,Jt,Ir,So,Lc,Rr,Oc,yd,At,Kc,Ho,ep,tp,_d,je,Nt,Vr,Lo,op,Dr,ap,gd,L,Oo,lp,Ko,rp,Yr,sp,np,ip,Ut,ea,dp,Sr,hp,wd,Qe,Wt,Hr,ta,up,fl,Lr,fp,cp,Ed,x,pp,Or,mp,bp,Kr,vp,yp,es,_p,gp,$d,se,wp,oa,Ep,$p,aa,kp,Mp,kd,ne,Tp,ts,Cp,qp,os,zp,Jp,Md,Zt,Ap,la,Np,Up,Td,Pt,Wp,as,Zp,Pp,Cd,Be,Gt,ls,ra,Gp,rs,jp,qd,U,Qp,ss,Bp,Fp,ns,xp,Xp,cl,Ip,Rp,is,Vp,Dp,zd,sa,Jd,W,Yp,ds,Sp,Hp,hs,Lp,Op,us,Kp,em,fs,tm,om,Ad,na,Nd,Fe,jt,cs,ia,am,ps,lm,Ud,xe,Qt,ms,da,rm,bs,sm,Wd,pl,nm,Zd,ie,vs,Bt,im,ys,dm,hm,_s,um,fm,gs,Ft,cm,ws,pm,mm,Es,bm,vm,$s,xt,ym,ks,_m,gm,Ms,wm,Pd,Xe,Xt,Ts,ha,Em,Cs,$m,Gd,X,qs,It,zs,km,Mm,ua,Tm,Cm,qm,Js,Z,fa,zm,As,Jm,Am,Nm,Ns,Um,Wm,Us,Zm,Pm,ca,Gm,jm,Ws,Qm,Bm,Fm,Zs,de,Ps,xm,Xm,pa,Im,Rm,Gs,Vm,Dm,Ym,js,I,Qs,Sm,Hm,Bs,Lm,Om,Fs,Km,eb,xs,tb,ob,jd,Ie,Rt,Xs,ma,ab,Is,lb,Qd,he,rb,Rs,sb,nb,Vs,ib,db,Bd,ba,Fd,Vt,xd,Re,Dt,Ds,va,hb,Ys,ub,Xd,ue,fb,Ss,cb,pb,Hs,mb,bb,Id,ya,Rd,Yt,vb,ml,yb,_b,Vd,St,gb,Ls,wb,Eb,Dd,_a,Yd,bl,$b,Sd,Ht,Hd,Ve,Lt,Os,ga,kb,Ks,Mb,Ld,vl,Tb,Od,De,Ot,en,wa,Cb,tn,qb,Kd,fe,zb,on,Jb,Ab,an,Nb,Ub,eh,Ea,th,Ye,Kt,ln,$a,Wb,rn,Zb,oh,yl,Pb,ah,ka,lh,Se,eo,sn,Ma,Gb,nn,jb,rh,_l,Qb,sh,Ta,nh,He,to,dn,Ca,Bb,hn,Fb,ih,R,xb,un,Xb,Ib,fn,Rb,Vb,cn,Db,Yb,dh,qa,hh,oo,uh,Le,ao,pn,za,Sb,mn,Hb,fh,ce,Lb,bn,Ob,Kb,vn,ev,tv,ch,Ja,ph,q,ov,yn,av,lv,_n,rv,sv,gn,nv,iv,wn,dv,hv,En,uv,fv,mh,Oe,lo,$n,Aa,cv,kn,pv,bh,gl,mv,vh,Ke,ro,Mn,Na,bv,so,vv,Tn,yv,_v,Cn,gv,yh,P,wv,qn,Ev,$v,zn,kv,Mv,Jn,Tv,Cv,An,qv,zv,_h,G,Jv,wl,Av,Nv,Nn,Uv,Wv,Un,Zv,Pv,Wn,Gv,jv,gh,Ua,wh,pe,Qv,Zn,Bv,Fv,Pn,xv,Xv,Eh,Wa,$h,El,Iv,kh,Za,Mh,$l,Rv,Th,et,no,Gn,Pa,Vv,kl,Dv,jn,Yv,Ch,me,Sv,Qn,Hv,Lv,Bn,Ov,Kv,qh,Ga,zh,tt,io,Fn,ja,ey,xn,ty,Jh,ot,oy,Xn,ay,ly,In,ry,Ah,Qa,Nh,at,ho,Rn,Ba,sy,Vn,ny,Uh,V,iy,Dn,dy,hy,Yn,uy,fy,Fa,Sn,cy,py,Wh,z,my,Hn,by,vy,Ln,yy,_y,On,gy,wy,Kn,Ey,$y,ei,ky,My,Zh,lt,uo,ti,xa,Ty,oi,Cy,Ph,M,Xa,qy,Ia,zy,ai,Jy,Ay,Ny,rt,Uy,li,Wy,Zy,ri,Py,Gy,jy,B,Qy,si,By,Fy,ni,xy,Xy,ii,Iy,Ry,di,Vy,Dy,Yy,fo,Ra,Sy,st,Hy,hi,Ly,Oy,ui,Ky,e_,t_,co,Va,o_,fi,a_,l_,po,Da,r_,Ya,s_,ci,n_,i_,d_,mo,Sa,h_,pi,u_,Gh,nt,bo,mi,Ha,f_,Ml,c_,bi,p_,jh,be,m_,La,b_,v_,vi,y_,__,Qh;return j=new g({}),H=new g({}),zo=new g({}),Jo=new g({}),Ao=new g({}),No=new E({props:{code:"bW9kZWxfaWQlMjAlM0QlMjAlMjJmYWNlYm9vayUyRm9wdC0xMjVtJTIyJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBZ3B0cV9jb25maWclMjAlM0QlMjBHUFRRQ29uZmlnKGJpdHMlM0Q0JTJDJTIwZGF0YXNldCUyMCUzRCUyMCUyMmM0JTIyJTJDJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyKQ==",highlighted:`model_id = <span class="hljs-string">&quot;facebook/opt-125m&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
gptq_config = GPTQConfig(bits=<span class="hljs-number">4</span>, dataset = <span class="hljs-string">&quot;c4&quot;</span>, tokenizer=tokenizer)`}}),Uo=new E({props:{code:"ZGF0YXNldCUyMCUzRCUyMCU1QiUyMmF1dG8tZ3B0cSUyMGlzJTIwYW4lMjBlYXN5LXRvLXVzZSUyMG1vZGVsJTIwcXVhbnRpemF0aW9uJTIwbGlicmFyeSUyMHdpdGglMjB1c2VyLWZyaWVuZGx5JTIwYXBpcyUyQyUyMGJhc2VkJTIwb24lMjBHUFRRJTIwYWxnb3JpdGhtLiUyMiU1RCUwQXF1YW50aXphdGlvbiUyMCUzRCUyMEdQVFFDb25maWcoYml0cyUzRDQlMkMlMjBkYXRhc2V0JTIwJTNEJTIwZGF0YXNldCUyQyUyMHRva2VuaXplciUzRHRva2VuaXplcik=",highlighted:`dataset = [<span class="hljs-string">&quot;auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm.&quot;</span>]
quantization = GPTQConfig(bits=<span class="hljs-number">4</span>, dataset = dataset, tokenizer=tokenizer)`}}),Wo=new g({}),Zo=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEZ3B0cV9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=gptq_config)`}}),Po=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMHF1YW50aXphdGlvbl9jb25maWclM0RncHRxX2NvbmZpZyk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=gptq_config)`}}),Et=new Qu({props:{warning:!0,$$slots:{default:[c0]},$$scope:{ctx:K}}}),jo=new g({}),Qo=new E({props:{code:"cXVhbnRpemVkX21vZGVsLnB1c2hfdG9faHViKCUyMm9wdC0xMjVtLWdwdHElMjIpJTBBdG9rZW5pemVyLnB1c2hfdG9faHViKCUyMm9wdC0xMjVtLWdwdHElMjIp",highlighted:`quantized_model.push_to_hub(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)
tokenizer.push_to_hub(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)`}}),Bo=new E({props:{code:"cXVhbnRpemVkX21vZGVsLnNhdmVfcHJldHJhaW5lZCglMjJvcHQtMTI1bS1ncHRxJTIyKSUwQXRva2VuaXplci5zYXZlX3ByZXRyYWluZWQoJTIyb3B0LTEyNW0tZ3B0cSUyMik=",highlighted:`quantized_model.save_pretrained(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)
tokenizer.save_pretrained(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)`}}),Fo=new E({props:{code:"cXVhbnRpemVkX21vZGVsLnRvKCUyMmNwdSUyMiklMEFxdWFudGl6ZWRfbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKCUyMm9wdC0xMjVtLWdwdHElMjIp",highlighted:`quantized_model.to(<span class="hljs-string">&quot;cpu&quot;</span>)
quantized_model.save_pretrained(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)`}}),xo=new g({}),Xo=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyJTdCeW91cl91c2VybmFtZSU3RCUyRm9wdC0xMjVtLWdwdHElMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/opt-125m-gptq&quot;</span>)`}}),Io=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyJTdCeW91cl91c2VybmFtZSU3RCUyRm9wdC0xMjVtLWdwdHElMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/opt-125m-gptq&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`}}),Ro=new g({}),Vo=new E({props:{code:"aW1wb3J0JTIwdG9yY2glMEFncHRxX2NvbmZpZyUyMCUzRCUyMEdQVFFDb25maWcoYml0cyUzRDQlMkMlMjBkaXNhYmxlX2V4bGxhbWElM0RGYWxzZSklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjIlN0J5b3VyX3VzZXJuYW1lJTdEJTJGb3B0LTEyNW0tZ3B0cSUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMGdwdHFfY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
gptq_config = GPTQConfig(bits=<span class="hljs-number">4</span>, disable_exllama=<span class="hljs-literal">False</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/opt-125m-gptq&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config = gptq_config)`}}),Do=new g({}),So=new g({}),Lo=new g({}),Oo=new el({props:{name:"class transformers.GPTQConfig",anchor:"transformers.GPTQConfig",parameters:[{name:"bits",val:": int"},{name:"tokenizer",val:": typing.Any = None"},{name:"dataset",val:": typing.Union[str, typing.List[str], NoneType] = None"},{name:"group_size",val:": int = 128"},{name:"damp_percent",val:": float = 0.1"},{name:"desc_act",val:": bool = False"},{name:"sym",val:": bool = True"},{name:"true_sequential",val:": bool = True"},{name:"use_cuda_fp16",val:": bool = False"},{name:"model_seqlen",val:": typing.Optional[int] = None"},{name:"block_name_to_quantize",val:": typing.Optional[str] = None"},{name:"module_name_preceding_first_block",val:": typing.Optional[typing.List[str]] = None"},{name:"batch_size",val:": int = 1"},{name:"pad_token_id",val:": typing.Optional[int] = None"},{name:"disable_exllama",val:": bool = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTQConfig.bits",description:`<strong>bits</strong> (<code>int</code>) &#x2014;
The number of bits to quantize to, supported numbers are (2, 3, 4, 8).`,name:"bits"},{anchor:"transformers.GPTQConfig.tokenizer",description:`<strong>tokenizer</strong> (<code>str</code> or <code>PreTrainedTokenizerBase</code>, <em>optional</em>) &#x2014;
The tokenizer used to process the dataset. You can pass either:<ul>
<li>A custom tokenizer object.</li>
<li>A string, the <em>model id</em> of a predefined tokenizer hosted inside a model repo on huggingface.co.
Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or namespaced under a
user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>A path to a <em>directory</em> containing vocabulary files required by the tokenizer, for instance saved
using the <a href="/docs/transformers/v4.32.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.save_pretrained">save_pretrained()</a> method, e.g., <code>./my_model_directory/</code>.</li>
</ul>`,name:"tokenizer"},{anchor:"transformers.GPTQConfig.dataset",description:`<strong>dataset</strong> (<code>Union[List[str]]</code>, <em>optional</em>) &#x2014;
The dataset used for quantization. You can provide your own dataset in a list of string or just use the
original datasets used in GPTQ paper [&#x2018;wikitext2&#x2019;,&#x2018;c4&#x2019;,&#x2018;c4-new&#x2019;,&#x2018;ptb&#x2019;,&#x2018;ptb-new&#x2019;]`,name:"dataset"},{anchor:"transformers.GPTQConfig.group_size",description:`<strong>group_size</strong> (<code>int</code>, <em>optional</em>, defaults to 128) &#x2014;
The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization.`,name:"group_size"},{anchor:"transformers.GPTQConfig.damp_percent",description:`<strong>damp_percent</strong> (<code>float</code>, <em>optional</em>, defaults to 0.1) &#x2014;
The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.1.`,name:"damp_percent"},{anchor:"transformers.GPTQConfig.desc_act",description:`<strong>desc_act</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to quantize columns in order of decreasing activation size. Setting it to False can significantly
speed up inference but the perplexity may become slightly worse. Also known as act-order.`,name:"desc_act"},{anchor:"transformers.GPTQConfig.sym",description:`<strong>sym</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to use symetric quantization.`,name:"sym"},{anchor:"transformers.GPTQConfig.true_sequential",description:`<strong>true_sequential</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to perform sequential quantization even within a single Transformer block. Instead of quantizing
the entire block at once, we perform layer-wise quantization. As a result, each layer undergoes
quantization using inputs that have passed through the previously quantized layers.`,name:"true_sequential"},{anchor:"transformers.GPTQConfig.use_cuda_fp16",description:`<strong>use_cuda_fp16</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16.`,name:"use_cuda_fp16"},{anchor:"transformers.GPTQConfig.model_seqlen",description:`<strong>model_seqlen</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The maximum sequence length that the model can take.`,name:"model_seqlen"},{anchor:"transformers.GPTQConfig.block_name_to_quantize",description:`<strong>block_name_to_quantize</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The transformers block name to quantize.`,name:"block_name_to_quantize"},{anchor:"transformers.GPTQConfig.module_name_preceding_first_block",description:`<strong>module_name_preceding_first_block</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
The layers that are preceding the first Transformer block.`,name:"module_name_preceding_first_block"},{anchor:"transformers.GPTQConfig.batch_size",description:`<strong>batch_size</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The batch size used when processing the dataset`,name:"batch_size"},{anchor:"transformers.GPTQConfig.pad_token_id",description:`<strong>pad_token_id</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The pad token id. Needed to prepare the dataset when <code>batch_size</code> &gt; 1.`,name:"pad_token_id"},{anchor:"transformers.GPTQConfig.disable_exllama",description:`<strong>disable_exllama</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to use exllama backend. Only works with <code>bits</code> = 4.`,name:"disable_exllama"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/utils/quantization_config.py#L303"}}),ea=new el({props:{name:"post_init",anchor:"transformers.GPTQConfig.post_init",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/utils/quantization_config.py#L394"}}),ta=new g({}),ra=new g({}),sa=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTIwbG9hZF9pbl84Yml0JTNEVHJ1ZSklMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTM1MG0lMjIlMkMlMjBsb2FkX2luXzRiaXQlM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_8bit = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)
model_4bit = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`}}),na=new E({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTM1MG0lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDMyKSUwQW1vZGVsXzhiaXQubW9kZWwuZGVjb2Rlci5sYXllcnMlNUItMSU1RC5maW5hbF9sYXllcl9ub3JtLndlaWdodC5kdHlwZQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>model_8bit = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, torch_dtype=torch.float32)
<span class="hljs-meta">&gt;&gt;&gt; </span>model_8bit.model.decoder.layers[-<span class="hljs-number">1</span>].final_layer_norm.weight.dtype
torch.float32`}}),ia=new g({}),da=new g({}),ha=new g({}),ma=new g({}),ba=new E({props:{code:"JTIzJTIwcGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fNGJpdCUzRFRydWUp",highlighted:`<span class="hljs-comment"># pip install transformers accelerate bitsandbytes</span>
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`}}),Vt=new Qu({props:{warning:!0,$$slots:{default:[p0]},$$scope:{ctx:K}}}),va=new g({}),ya=new E({props:{code:"JTIzJTIwcGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMGxvYWRfaW5fOGJpdCUzRFRydWUp",highlighted:`<span class="hljs-comment"># pip install transformers accelerate bitsandbytes</span>
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`}}),_a=new E({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())'}}),Ht=new Qu({props:{warning:!0,$$slots:{default:[m0]},$$scope:{ctx:K}}}),ga=new g({}),wa=new g({}),Ea=new E({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`}}),$a=new g({}),ka=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW5mNF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBbW9kZWxfbmY0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRG5mNF9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

nf4_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

model_nf4 = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=nf4_config)`}}),Ma=new g({}),Ta=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQW1vZGVsX2RvdWJsZV9xdWFudCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCUyQyUyMHF1YW50aXphdGlvbl9jb25maWclM0Rkb3VibGVfcXVhbnRfY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

model_double_quant = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=double_quant_config)`}}),Ca=new g({}),qa=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmJpZ3NjaWVuY2UlMkZibG9vbS01NjBtJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmJpZ3NjaWVuY2UlMkZibG9vbS01NjBtJTIyKSUwQSUwQW1vZGVsLnB1c2hfdG9faHViKCUyMmJsb29tLTU2MG0tOGJpdCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>)

model.push_to_hub(<span class="hljs-string">&quot;bloom-560m-8bit&quot;</span>)`}}),oo=new Qu({props:{warning:!0,$$slots:{default:[b0]},$$scope:{ctx:K}}}),za=new g({}),Ja=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMiU3QnlvdXJfdXNlcm5hbWUlN0QlMkZibG9vbS01NjBtLThiaXQlMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/bloom-560m-8bit&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`}}),Aa=new g({}),Na=new g({}),Ua=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcobGxtX2ludDhfZW5hYmxlX2ZwMzJfY3B1X29mZmxvYWQlM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=<span class="hljs-literal">True</span>)`}}),Wa=new E({props:{code:"ZGV2aWNlX21hcCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLndvcmRfZW1iZWRkaW5ncyUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci53b3JkX2VtYmVkZGluZ3NfbGF5ZXJub3JtJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMmxtX2hlYWQlMjIlM0ElMjAlMjJjcHUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci5oJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLmxuX2YlMjIlM0ElMjAwJTJDJTBBJTdE",highlighted:`device_map = {
    <span class="hljs-string">&quot;transformer.word_embeddings&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.word_embeddings_layernorm&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;lm_head&quot;</span>: <span class="hljs-string">&quot;cpu&quot;</span>,
    <span class="hljs-string">&quot;transformer.h&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.ln_f&quot;</span>: <span class="hljs-number">0</span>,
}`}}),Za=new E({props:{code:"bW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    device_map=device_map,
    quantization_config=quantization_config,
)`}}),Pa=new g({}),Ga=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbGxtX2ludDhfdGhyZXNob2xkJTNEMTAlMkMlMEEpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjBtb2RlbF9pZCUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0RkZXZpY2VfbWFwJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMEEpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)
tokenizer = AutoTokenizer.from_pretrained(model_id)`}}),ja=new g({}),Qa=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbGxtX2ludDhfc2tpcF9tb2R1bGVzJTNEJTVCJTIybG1faGVhZCUyMiU1RCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_skip_modules=[<span class="hljs-string">&quot;lm_head&quot;</span>],
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)
tokenizer = AutoTokenizer.from_pretrained(model_id)`}}),Ba=new g({}),xa=new g({}),Xa=new el({props:{name:"class transformers.BitsAndBytesConfig",anchor:"transformers.BitsAndBytesConfig",parameters:[{name:"load_in_8bit",val:" = False"},{name:"load_in_4bit",val:" = False"},{name:"llm_int8_threshold",val:" = 6.0"},{name:"llm_int8_skip_modules",val:" = None"},{name:"llm_int8_enable_fp32_cpu_offload",val:" = False"},{name:"llm_int8_has_fp16_weight",val:" = False"},{name:"bnb_4bit_compute_dtype",val:" = None"},{name:"bnb_4bit_quant_type",val:" = 'fp4'"},{name:"bnb_4bit_use_double_quant",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.BitsAndBytesConfig.load_in_8bit",description:`<strong>load_in_8bit</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used to enable 8-bit quantization with LLM.int8().`,name:"load_in_8bit"},{anchor:"transformers.BitsAndBytesConfig.load_in_4bit",description:`<strong>load_in_4bit</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used to enable 4-bit quantization by replacing the Linear layers with FP4/NF4 layers from
<code>bitsandbytes</code>.`,name:"load_in_4bit"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_threshold",description:`<strong>llm_int8_threshold</strong> (<code>float</code>, <em>optional</em>, defaults to 6) &#x2014;
This corresponds to the outlier threshold for outlier detection as described in <code>LLM.int8() : 8-bit Matrix Multiplication for Transformers at Scale</code> paper: <a href="https://arxiv.org/abs/2208.07339" rel="nofollow">https://arxiv.org/abs/2208.07339</a> Any hidden states value
that is above this threshold will be considered an outlier and the operation on those values will be done
in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but
there are some exceptional systematic outliers that are very differently distributed for large models.
These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of
magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6,
but a lower threshold might be needed for more unstable models (small models, fine-tuning).`,name:"llm_int8_threshold"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_skip_modules",description:`<strong>llm_int8_skip_modules</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
An explicit list of the modules that we do not want to convert in 8-bit. This is useful for models such as
Jukebox that has several heads in different places and not necessarily at the last position. For example
for <code>CausalLM</code> models, the last <code>lm_head</code> is kept in its original <code>dtype</code>.`,name:"llm_int8_skip_modules"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_enable_fp32_cpu_offload",description:`<strong>llm_int8_enable_fp32_cpu_offload</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used for advanced use cases and users that are aware of this feature. If you want to split
your model in different parts and run some parts in int8 on GPU and some parts in fp32 on CPU, you can use
this flag. This is useful for offloading large models such as <code>google/flan-t5-xxl</code>. Note that the int8
operations will not be run on CPU.`,name:"llm_int8_enable_fp32_cpu_offload"},{anchor:"transformers.BitsAndBytesConfig.llm_int8_has_fp16_weight",description:`<strong>llm_int8_has_fp16_weight</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag runs LLM.int8() with 16-bit main weights. This is useful for fine-tuning as the weights do not
have to be converted back and forth for the backward pass.`,name:"llm_int8_has_fp16_weight"},{anchor:"transformers.BitsAndBytesConfig.bnb_4bit_compute_dtype",description:`<strong>bnb_4bit_compute_dtype</strong> (<code>torch.dtype</code> or str, <em>optional</em>, defaults to <code>torch.float32</code>) &#x2014;
This sets the computational type which might be different than the input time. For example, inputs might be
fp32, but computation can be set to bf16 for speedups.`,name:"bnb_4bit_compute_dtype"},{anchor:"transformers.BitsAndBytesConfig.bnb_4bit_quant_type",description:`<strong>bnb_4bit_quant_type</strong> (<code>str</code>, {fp4, nf4}, defaults to <code>fp4</code>) &#x2014;
This sets the quantization data type in the bnb.nn.Linear4Bit layers. Options are FP4 and NF4 data types
which are specified by <code>fp4</code> or <code>nf4</code>.`,name:"bnb_4bit_quant_type"},{anchor:"transformers.BitsAndBytesConfig.bnb_4bit_use_double_quant",description:`<strong>bnb_4bit_use_double_quant</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
This flag is used for nested quantization where the quantization constants from the first quantization are
quantized again.`,name:"bnb_4bit_use_double_quant"},{anchor:"transformers.BitsAndBytesConfig.kwargs",description:`<strong>kwargs</strong> (<code>Dict[str, Any]</code>, <em>optional</em>) &#x2014;
Additional parameters from which to initialize the configuration object.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/utils/quantization_config.py#L130"}}),Ra=new el({props:{name:"is_quantizable",anchor:"transformers.BitsAndBytesConfig.is_quantizable",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/utils/quantization_config.py#L245"}}),Va=new el({props:{name:"post_init",anchor:"transformers.BitsAndBytesConfig.post_init",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/utils/quantization_config.py#L214"}}),Da=new el({props:{name:"quantization_method",anchor:"transformers.BitsAndBytesConfig.quantization_method",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/utils/quantization_config.py#L251"}}),Sa=new el({props:{name:"to_diff_dict",anchor:"transformers.BitsAndBytesConfig.to_diff_dict",parameters:[],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/utils/quantization_config.py#L279",returnDescription:`
<p>Dictionary of all the attributes that make up this configuration instance,</p>
`,returnType:`
<p><code>Dict[str, Any]</code></p>
`}}),Ha=new g({}),{c(){_=a("meta"),$=h(),w=a("h1"),k=a("a"),$e=a("span"),c(j.$$.fragment),Eo=h(),ke=a("span"),Me=s("Quantize \u{1F917} Transformers models"),dt=h(),Q=a("h2"),A=a("a"),Te=a("span"),c(H.$$.fragment),Ce=h(),T=a("span"),ht=a("code"),tl=s("AutoGPTQ"),ol=s(" Integration"),$o=h(),ut=a("p"),Bu=s("\u{1F917} Transformers has integrated "),Vl=a("code"),Fu=s("optimum"),xu=s(" API to perform GPTQ quantization on language models. You can load and quantize your model in 8, 4, 3 or even 2 bits without a big drop of performance and faster inference speed! This is supported by most GPU hardwares."),Ai=h(),al=a("p"),Xu=s("To learn more about the the quantization model, check out:"),Ni=h(),ee=a("ul"),ko=a("li"),Iu=s("the "),Mo=a("a"),Ru=s("GPTQ"),Vu=s(" paper"),Du=h(),qe=a("li"),Yu=s("the "),Dl=a("code"),Su=s("optimum"),Hu=h(),To=a("a"),Lu=s("guide"),Ou=s(" on GPTQ quantization"),Ku=h(),Co=a("li"),ef=s("the "),qo=a("a"),Yl=a("code"),tf=s("AutoGPTQ"),of=s(" library used as the backend"),Ui=h(),ze=a("h3"),ft=a("a"),Sl=a("span"),c(zo.$$.fragment),af=h(),Hl=a("span"),lf=s("Requirements"),Wi=h(),ll=a("p"),rf=s("You need to have the following requirements installed to run the code below:"),Zi=h(),F=a("ul"),Ll=a("li"),ct=a("p"),sf=s("Install latest "),Ol=a("code"),nf=s("AutoGPTQ"),df=s(` library
`),Kl=a("code"),hf=s("pip install auto-gptq"),uf=h(),er=a("li"),pt=a("p"),ff=s("Install latest "),tr=a("code"),cf=s("optimum"),pf=s(` from source
`),or=a("code"),mf=s("pip install git+https://github.com/huggingface/optimum.git"),bf=h(),ar=a("li"),mt=a("p"),vf=s("Install latest "),lr=a("code"),yf=s("transformers"),_f=s(` from source
`),rr=a("code"),gf=s("pip install git+https://github.com/huggingface/transformers.git"),wf=h(),sr=a("li"),bt=a("p"),Ef=s("Install latest "),nr=a("code"),$f=s("accelerate"),kf=s(` library
`),ir=a("code"),Mf=s("pip install --upgrade accelerate"),Pi=h(),rl=a("p"),Tf=s("Note that GPTQ integration supports for now only text models and you may encounter unexpected behaviour for vision, speech or multi-modal models."),Gi=h(),Je=a("h3"),vt=a("a"),dr=a("span"),c(Jo.$$.fragment),Cf=h(),hr=a("span"),qf=s("Load and quantize a model"),ji=h(),yt=a("p"),zf=s("GPTQ is a quantization method that requires weights calibration before using the quantized models. If you want to quantize transformers model from scratch, it might take some time before producing the quantized model (~5 min on a Google colab for "),ur=a("code"),Jf=s("facebook/opt-350m"),Af=s(" model)."),Qi=h(),sl=a("p"),Nf=s("Hence, there are two different scenarios where you want to use GPTQ-quantized models. The first use case would be to load models that has been already quantized by other users that are available on the Hub, the second use case would be to quantize your model from scratch and save it or push it on the Hub so that other users can also use it."),Bi=h(),Ae=a("h4"),_t=a("a"),fr=a("span"),c(Ao.$$.fragment),Uf=h(),cr=a("span"),Wf=s("GPTQ Configuration"),Fi=h(),N=a("p"),Zf=s("In order to load and quantize a model, you need to create a "),nl=a("a"),Pf=s("GPTQConfig"),Gf=s(". You need to pass the number of "),pr=a("code"),jf=s("bits"),Qf=s(", a "),mr=a("code"),Bf=s("dataset"),Ff=s(" in order to calibrate the quantization and the "),br=a("code"),xf=s("tokenizer"),Xf=s(" of the model in order prepare the dataset."),xi=h(),c(No.$$.fragment),Xi=h(),il=a("p"),If=s("Note that you can pass your own dataset as a list of string. However, it is highly recommended to use the dataset from the GPTQ paper."),Ii=h(),c(Uo.$$.fragment),Ri=h(),Ne=a("h4"),gt=a("a"),vr=a("span"),c(Wo.$$.fragment),Rf=h(),yr=a("span"),Vf=s("Quantization"),Vi=h(),te=a("p"),Df=s("You can quantize a model by using "),_r=a("code"),Yf=s("from_pretrained"),Sf=s(" and setting the "),gr=a("code"),Hf=s("quantization_config"),Lf=s("."),Di=h(),c(Zo.$$.fragment),Yi=h(),dl=a("p"),Of=s("Note that you will need a GPU to quantize a model. We will put the model in the cpu and move the modules back and forth to the gpu in order to quantize them."),Si=h(),wt=a("p"),Kf=s("If you want to maximize your gpus usage while using cpu offload, you can set "),wr=a("code"),ec=s('device_map = "auto"'),tc=s("."),Hi=h(),c(Po.$$.fragment),Li=h(),C=a("p"),oc=s("Note that disk offload is not supported. Furthermore, if you are out of memory because of the dataset, you may have to pass "),Er=a("code"),ac=s("max_memory"),lc=s(" in "),$r=a("code"),rc=s("from_pretained"),sc=s(". Checkout this "),Go=a("a"),nc=s("guide"),ic=s(" to learn more about "),kr=a("code"),dc=s("device_map"),hc=s(" and "),Mr=a("code"),uc=s("max_memory"),fc=s("."),Oi=h(),c(Et.$$.fragment),Ki=h(),Ue=a("h3"),$t=a("a"),Tr=a("span"),c(jo.$$.fragment),cc=h(),Cr=a("span"),pc=s("Push quantized model to \u{1F917} Hub"),ed=h(),kt=a("p"),mc=s("You can push the quantized model like any \u{1F917} model to Hub with "),qr=a("code"),bc=s("push_to_hub"),vc=s(". The quantization config will be saved and pushed along the model."),td=h(),c(Qo.$$.fragment),od=h(),Mt=a("p"),yc=s("If you want to save your quantized model on your local machine, you can also do it with "),zr=a("code"),_c=s("save_pretrained"),gc=s(":"),ad=h(),c(Bo.$$.fragment),ld=h(),oe=a("p"),wc=s("Note that if you have quantized your model with a "),Jr=a("code"),Ec=s("device_map"),$c=s(", make sure to move the entire model to one of your gpus or the "),Ar=a("code"),kc=s("cpu"),Mc=s(" before saving it."),rd=h(),c(Fo.$$.fragment),sd=h(),We=a("h3"),Tt=a("a"),Nr=a("span"),c(xo.$$.fragment),Tc=h(),Ur=a("span"),Cc=s("Load a quantized model from the \u{1F917} Hub"),nd=h(),ae=a("p"),qc=s("You can load a quantized model from the Hub by using "),Wr=a("code"),zc=s("from_pretrained"),Jc=s(`.
Make sure that the pushed weights are quantized, by checking that the attribute `),Zr=a("code"),Ac=s("quantization_config"),Nc=s(" is present in the model configuration object."),id=h(),c(Xo.$$.fragment),dd=h(),le=a("p"),Uc=s("If you want to load a model faster and without allocating more memory than needed, the "),Pr=a("code"),Wc=s("device_map"),Zc=s(" argument also works with quantized model. Make sure that you have "),Gr=a("code"),Pc=s("accelerate"),Gc=s(" library installed."),hd=h(),c(Io.$$.fragment),ud=h(),Ze=a("h3"),Ct=a("a"),jr=a("span"),c(Ro.$$.fragment),jc=h(),Qr=a("span"),Qc=s("Exllama kernels for faster inference"),fd=h(),re=a("p"),Bc=s("For 4-bit model, you can use the exllama kernels in order to a faster inference speed. It is activated by default. You can change that behavior by passing "),Br=a("code"),Fc=s("disable_exllama"),xc=s(" in "),hl=a("a"),Xc=s("GPTQConfig"),Ic=s(". This will overwrite the quantization config stored in the config. Note that you will only be able to overwrite the attributes related to the kernels. Furthermore, you need to have the entire model on gpus if you want to use exllama kernels."),cd=h(),c(Vo.$$.fragment),pd=h(),ul=a("p"),Rc=s("Note that only 4-bit models are supported for now. Furthermore, it is recommended to deactivate the exllama kernels if you are finetuning a quantized model with peft."),md=h(),Pe=a("h4"),qt=a("a"),Fr=a("span"),c(Do.$$.fragment),Vc=h(),xr=a("span"),Dc=s("Fine-tune a quantized model"),bd=h(),zt=a("p"),Yc=s(`With the official support of adapters in the Hugging Face ecosystem, you can fine-tune models that have been quantized with GPTQ.
Please have a look at `),Yo=a("a"),Xr=a("code"),Sc=s("peft"),Hc=s(" library for more details."),vd=h(),Ge=a("h3"),Jt=a("a"),Ir=a("span"),c(So.$$.fragment),Lc=h(),Rr=a("span"),Oc=s("Example demo"),yd=h(),At=a("p"),Kc=s("Check out the Google Colab "),Ho=a("a"),ep=s("notebook"),tp=s(" to learn how to quantize your model with GPTQ and how finetune the quantized model with peft."),_d=h(),je=a("h3"),Nt=a("a"),Vr=a("span"),c(Lo.$$.fragment),op=h(),Dr=a("span"),ap=s("GPTQConfig"),gd=h(),L=a("div"),c(Oo.$$.fragment),lp=h(),Ko=a("p"),rp=s(`This is a wrapper class about all possible attributes and features that you can play with a model that has been
loaded using `),Yr=a("code"),sp=s("optimum"),np=s(" api for gptq quantization relying on auto_gptq backend."),ip=h(),Ut=a("div"),c(ea.$$.fragment),dp=h(),Sr=a("p"),hp=s("Safety checker that arguments are correct"),wd=h(),Qe=a("h2"),Wt=a("a"),Hr=a("span"),c(ta.$$.fragment),up=h(),fl=a("span"),Lr=a("code"),fp=s("bitsandbytes"),cp=s(" Integration"),Ed=h(),x=a("p"),pp=s("\u{1F917} Transformers is closely integrated with most used modules on "),Or=a("code"),mp=s("bitsandbytes"),bp=s(`. You can load your model in 8-bit precision with few lines of code.
This is supported by most of the GPU hardwares since the `),Kr=a("code"),vp=s("0.37.0"),yp=s(" release of "),es=a("code"),_p=s("bitsandbytes"),gp=s("."),$d=h(),se=a("p"),wp=s("Learn more about the quantization method in the "),oa=a("a"),Ep=s("LLM.int8()"),$p=s(" paper, or the "),aa=a("a"),kp=s("blogpost"),Mp=s(" about the collaboration."),kd=h(),ne=a("p"),Tp=s("Since its "),ts=a("code"),Cp=s("0.39.0"),qp=s(" release, you can load any model that supports "),os=a("code"),zp=s("device_map"),Jp=s(" using 4-bit quantization, leveraging FP4 data type."),Md=h(),Zt=a("p"),Ap=s("If you want to quantize your own pytorch model, check out this "),la=a("a"),Np=s("documentation"),Up=s(" from \u{1F917} Accelerate library."),Td=h(),Pt=a("p"),Wp=s("Here are the things you can do using "),as=a("code"),Zp=s("bitsandbytes"),Pp=s(" integration"),Cd=h(),Be=a("h3"),Gt=a("a"),ls=a("span"),c(ra.$$.fragment),Gp=h(),rs=a("span"),jp=s("General usage"),qd=h(),U=a("p"),Qp=s("You can quantize a model by using the "),ss=a("code"),Bp=s("load_in_8bit"),Fp=s(" or "),ns=a("code"),xp=s("load_in_4bit"),Xp=s(" argument when calling the "),cl=a("a"),Ip=s("from_pretrained()"),Rp=s(" method as long as your model supports loading with \u{1F917} Accelerate and contains "),is=a("code"),Vp=s("torch.nn.Linear"),Dp=s(" layers. This should work for any modality as well."),zd=h(),c(sa.$$.fragment),Jd=h(),W=a("p"),Yp=s("By default all other modules (e.g. "),ds=a("code"),Sp=s("torch.nn.LayerNorm"),Hp=s(") will be converted in "),hs=a("code"),Lp=s("torch.float16"),Op=s(", but if you want to change their "),us=a("code"),Kp=s("dtype"),em=s(" you can overwrite the "),fs=a("code"),tm=s("torch_dtype"),om=s(" argument:"),Ad=h(),c(na.$$.fragment),Nd=h(),Fe=a("h3"),jt=a("a"),cs=a("span"),c(ia.$$.fragment),am=h(),ps=a("span"),lm=s("FP4 quantization"),Ud=h(),xe=a("h4"),Qt=a("a"),ms=a("span"),c(da.$$.fragment),rm=h(),bs=a("span"),sm=s("Requirements"),Wd=h(),pl=a("p"),nm=s("Make sure that you have installed the requirements below before running any of the code snippets below."),Zd=h(),ie=a("ul"),vs=a("li"),Bt=a("p"),im=s("Latest "),ys=a("code"),dm=s("bitsandbytes"),hm=s(` library
`),_s=a("code"),um=s("pip install bitsandbytes>=0.39.0"),fm=h(),gs=a("li"),Ft=a("p"),cm=s("Install latest "),ws=a("code"),pm=s("accelerate"),mm=h(),Es=a("code"),bm=s("pip install --upgrade accelerate"),vm=h(),$s=a("li"),xt=a("p"),ym=s("Install latest "),ks=a("code"),_m=s("transformers"),gm=h(),Ms=a("code"),wm=s("pip install --upgrade transformers"),Pd=h(),Xe=a("h4"),Xt=a("a"),Ts=a("span"),c(ha.$$.fragment),Em=h(),Cs=a("span"),$m=s("Tips and best practices"),Gd=h(),X=a("ul"),qs=a("li"),It=a("p"),zs=a("strong"),km=s("Advanced usage:"),Mm=s(" Refer to "),ua=a("a"),Tm=s("this Google Colab notebook"),Cm=s(" for advanced usage of 4-bit quantization with all the possible options."),qm=h(),Js=a("li"),Z=a("p"),fa=a("strong"),zm=s("Faster inference with "),As=a("code"),Jm=s("batch_size=1"),Am=s(" :"),Nm=s(" Since the "),Ns=a("code"),Um=s("0.40.0"),Wm=s(" release of bitsandbytes, for "),Us=a("code"),Zm=s("batch_size=1"),Pm=s(" you can benefit from fast inference. Check out "),ca=a("a"),Gm=s("these release notes"),jm=s(" and make sure to have a version that is greater than "),Ws=a("code"),Qm=s("0.40.0"),Bm=s(" to benefit from this feature out of the box."),Fm=h(),Zs=a("li"),de=a("p"),Ps=a("strong"),xm=s("Training:"),Xm=s(" According to "),pa=a("a"),Im=s("QLoRA paper"),Rm=s(", for training 4-bit base models (e.g. using LoRA adapters) one should use "),Gs=a("code"),Vm=s("bnb_4bit_quant_type='nf4'"),Dm=s("."),Ym=h(),js=a("li"),I=a("p"),Qs=a("strong"),Sm=s("Inference:"),Hm=s(" For inference, "),Bs=a("code"),Lm=s("bnb_4bit_quant_type"),Om=s(" does not have a huge impact on the performance. However for consistency with the model\u2019s weights, make sure you use the same "),Fs=a("code"),Km=s("bnb_4bit_compute_dtype"),eb=s(" and "),xs=a("code"),tb=s("torch_dtype"),ob=s(" arguments."),jd=h(),Ie=a("h4"),Rt=a("a"),Xs=a("span"),c(ma.$$.fragment),ab=h(),Is=a("span"),lb=s("Load a large model in 4bit"),Qd=h(),he=a("p"),rb=s("By using "),Rs=a("code"),sb=s("load_in_4bit=True"),nb=s(" when calling the "),Vs=a("code"),ib=s(".from_pretrained"),db=s(" method, you can divide your memory use by 4 (roughly)."),Bd=h(),c(ba.$$.fragment),Fd=h(),c(Vt.$$.fragment),xd=h(),Re=a("h3"),Dt=a("a"),Ds=a("span"),c(va.$$.fragment),hb=h(),Ys=a("span"),ub=s("Load a large model in 8bit"),Xd=h(),ue=a("p"),fb=s("You can load a model by roughly halving the memory requirements by using "),Ss=a("code"),cb=s("load_in_8bit=True"),pb=s(" argument when calling "),Hs=a("code"),mb=s(".from_pretrained"),bb=s(" method"),Id=h(),c(ya.$$.fragment),Rd=h(),Yt=a("p"),vb=s("Then, use your model as you would usually use a "),ml=a("a"),yb=s("PreTrainedModel"),_b=s("."),Vd=h(),St=a("p"),gb=s("You can check the memory footprint of your model with "),Ls=a("code"),wb=s("get_memory_footprint"),Eb=s(" method."),Dd=h(),c(_a.$$.fragment),Yd=h(),bl=a("p"),$b=s("With this integration we were able to load large models on smaller devices and run them without any issue."),Sd=h(),c(Ht.$$.fragment),Hd=h(),Ve=a("h4"),Lt=a("a"),Os=a("span"),c(ga.$$.fragment),kb=h(),Ks=a("span"),Mb=s("Advanced use cases"),Ld=h(),vl=a("p"),Tb=s("Here we will cover some advanced use cases you can perform with FP4 quantization"),Od=h(),De=a("h5"),Ot=a("a"),en=a("span"),c(wa.$$.fragment),Cb=h(),tn=a("span"),qb=s("Change the compute dtype"),Kd=h(),fe=a("p"),zb=s("The compute dtype is used to change the dtype that will be used during computation. For example, hidden states could be in "),on=a("code"),Jb=s("float32"),Ab=s(" but computation can be set to bf16 for speedups. By default, the compute dtype is set to "),an=a("code"),Nb=s("float32"),Ub=s("."),eh=h(),c(Ea.$$.fragment),th=h(),Ye=a("h5"),Kt=a("a"),ln=a("span"),c($a.$$.fragment),Wb=h(),rn=a("span"),Zb=s("Using NF4 (Normal Float 4) data type"),oh=h(),yl=a("p"),Pb=s("You can also use the NF4 data type, which is a new 4bit datatype adapted for weights that have been initialized using a normal distribution. For that run:"),ah=h(),c(ka.$$.fragment),lh=h(),Se=a("h5"),eo=a("a"),sn=a("span"),c(Ma.$$.fragment),Gb=h(),nn=a("span"),jb=s("Use nested quantization for more memory efficient inference"),rh=h(),_l=a("p"),Qb=s("We also advise users to use the nested quantization technique. This saves more memory at no additional performance - from our empirical observations, this enables fine-tuning llama-13b model on an NVIDIA-T4 16GB with a sequence length of 1024, batch size of 1 and gradient accumulation steps of 4."),sh=h(),c(Ta.$$.fragment),nh=h(),He=a("h3"),to=a("a"),dn=a("span"),c(Ca.$$.fragment),Bb=h(),hn=a("span"),Fb=s("Push quantized models on the \u{1F917} Hub"),ih=h(),R=a("p"),xb=s("You can push a quantized model on the Hub by naively using "),un=a("code"),Xb=s("push_to_hub"),Ib=s(` method. This will first push the quantization configuration file, then push the quantized model weights.
Make sure to use `),fn=a("code"),Rb=s("bitsandbytes>0.37.2"),Vb=s(" (at this time of writing, we tested it on "),cn=a("code"),Db=s("bitsandbytes==0.38.0.post1"),Yb=s(") to be able to use this feature."),dh=h(),c(qa.$$.fragment),hh=h(),c(oo.$$.fragment),uh=h(),Le=a("h3"),ao=a("a"),pn=a("span"),c(za.$$.fragment),Sb=h(),mn=a("span"),Hb=s("Load a quantized model from the \u{1F917} Hub"),fh=h(),ce=a("p"),Lb=s("You can load a quantized model from the Hub by using "),bn=a("code"),Ob=s("from_pretrained"),Kb=s(" method. Make sure that the pushed weights are quantized, by checking that the attribute "),vn=a("code"),ev=s("quantization_config"),tv=s(" is present in the model configuration object."),ch=h(),c(Ja.$$.fragment),ph=h(),q=a("p"),ov=s("Note that in this case, you don\u2019t need to specify the arguments "),yn=a("code"),av=s("load_in_8bit=True"),lv=s(", but you need to make sure that "),_n=a("code"),rv=s("bitsandbytes"),sv=s(" and "),gn=a("code"),nv=s("accelerate"),iv=s(` are installed.
Note also that `),wn=a("code"),dv=s("device_map"),hv=s(" is optional but setting "),En=a("code"),uv=s("device_map = 'auto'"),fv=s(" is prefered for inference as it will dispatch efficiently the model on the available ressources."),mh=h(),Oe=a("h3"),lo=a("a"),$n=a("span"),c(Aa.$$.fragment),cv=h(),kn=a("span"),pv=s("Advanced use cases"),bh=h(),gl=a("p"),mv=s("This section is intended to advanced users, that want to explore what it is possible to do beyond loading and running 8-bit models."),vh=h(),Ke=a("h4"),ro=a("a"),Mn=a("span"),c(Na.$$.fragment),bv=h(),so=a("span"),vv=s("Offload between "),Tn=a("code"),yv=s("cpu"),_v=s(" and "),Cn=a("code"),gv=s("gpu"),yh=h(),P=a("p"),wv=s("One of the advanced use case of this is being able to load a model and dispatch the weights between "),qn=a("code"),Ev=s("CPU"),$v=s(" and "),zn=a("code"),kv=s("GPU"),Mv=s(". Note that the weights that will be dispatched on CPU "),Jn=a("strong"),Tv=s("will not"),Cv=s(" be converted in 8-bit, thus kept in "),An=a("code"),qv=s("float32"),zv=s(". This feature is intended for users that want to fit a very large model and dispatch the model between GPU and CPU."),_h=h(),G=a("p"),Jv=s("First, load a "),wl=a("a"),Av=s("BitsAndBytesConfig"),Nv=s(" from "),Nn=a("code"),Uv=s("transformers"),Wv=s(" and set the attribute "),Un=a("code"),Zv=s("llm_int8_enable_fp32_cpu_offload"),Pv=s(" to "),Wn=a("code"),Gv=s("True"),jv=s(":"),gh=h(),c(Ua.$$.fragment),wh=h(),pe=a("p"),Qv=s("Let\u2019s say you want to load "),Zn=a("code"),Bv=s("bigscience/bloom-1b7"),Fv=s(" model, and you have just enough GPU RAM to fit the entire model except the "),Pn=a("code"),xv=s("lm_head"),Xv=s(". Therefore write a custom device_map as follows:"),Eh=h(),c(Wa.$$.fragment),$h=h(),El=a("p"),Iv=s("And load your model as follows:"),kh=h(),c(Za.$$.fragment),Mh=h(),$l=a("p"),Rv=s("And that\u2019s it! Enjoy your model!"),Th=h(),et=a("h4"),no=a("a"),Gn=a("span"),c(Pa.$$.fragment),Vv=h(),kl=a("span"),Dv=s("Play with "),jn=a("code"),Yv=s("llm_int8_threshold"),Ch=h(),me=a("p"),Sv=s("You can play with the "),Qn=a("code"),Hv=s("llm_int8_threshold"),Lv=s(` argument to change the threshold of the outliers. An \u201Coutlier\u201D is a hidden state value that is greater than a certain threshold.
This corresponds to the outlier threshold for outlier detection as described in `),Bn=a("code"),Ov=s("LLM.int8()"),Kv=s(` paper. Any hidden states value that is above this threshold will be considered an outlier and the operation on those values will be done in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but there are some exceptional systematic outliers that are very differently distributed for large models. These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6, but a lower threshold might be needed for more unstable models (small models, fine-tuning).
This argument can impact the inference speed of the model. We suggest to play with this parameter to find which one is the best for your use case.`),qh=h(),c(Ga.$$.fragment),zh=h(),tt=a("h4"),io=a("a"),Fn=a("span"),c(ja.$$.fragment),ey=h(),xn=a("span"),ty=s("Skip the conversion of some modules"),Jh=h(),ot=a("p"),oy=s("Some models has several modules that needs to be not converted in 8-bit to ensure stability. For example Jukebox model has several "),Xn=a("code"),ay=s("lm_head"),ly=s(" modules that should be skipped. Play with "),In=a("code"),ry=s("llm_int8_skip_modules"),Ah=h(),c(Qa.$$.fragment),Nh=h(),at=a("h4"),ho=a("a"),Rn=a("span"),c(Ba.$$.fragment),sy=h(),Vn=a("span"),ny=s("Fine-tune a model that has been loaded in 8-bit"),Uh=h(),V=a("p"),iy=s(`With the official support of adapters in the Hugging Face ecosystem, you can fine-tune models that have been loaded in 8-bit.
This enables fine-tuning large models such as `),Dn=a("code"),dy=s("flan-t5-large"),hy=s(" or "),Yn=a("code"),uy=s("facebook/opt-6.7b"),fy=s(" in a single google Colab. Please have a look at "),Fa=a("a"),Sn=a("code"),cy=s("peft"),py=s(" library for more details."),Wh=h(),z=a("p"),my=s("Note that you don\u2019t need to pass "),Hn=a("code"),by=s("device_map"),vy=s(" when loading the model for training. It will automatically load your model on your GPU. You can also set the device map to a specific device if needed (e.g. "),Ln=a("code"),yy=s("cuda:0"),_y=s(", "),On=a("code"),gy=s("0"),wy=s(", "),Kn=a("code"),Ey=s("torch.device('cuda:0')"),$y=s("). Please note that "),ei=a("code"),ky=s("device_map=auto"),My=s(" should be used for inference only."),Zh=h(),lt=a("h3"),uo=a("a"),ti=a("span"),c(xa.$$.fragment),Ty=h(),oi=a("span"),Cy=s("BitsAndBytesConfig"),Ph=h(),M=a("div"),c(Xa.$$.fragment),qy=h(),Ia=a("p"),zy=s(`This is a wrapper class about all possible attributes and features that you can play with a model that has been
loaded using `),ai=a("code"),Jy=s("bitsandbytes"),Ay=s("."),Ny=h(),rt=a("p"),Uy=s("This replaces "),li=a("code"),Wy=s("load_in_8bit"),Zy=s(" or "),ri=a("code"),Py=s("load_in_4bit"),Gy=s("therefore both options are mutually exclusive."),jy=h(),B=a("p"),Qy=s("Currently only supports "),si=a("code"),By=s("LLM.int8()"),Fy=s(", "),ni=a("code"),xy=s("FP4"),Xy=s(", and "),ii=a("code"),Iy=s("NF4"),Ry=s(" quantization. If more methods are added to "),di=a("code"),Vy=s("bitsandbytes"),Dy=s(`,
then more arguments will be added to this class.`),Yy=h(),fo=a("div"),c(Ra.$$.fragment),Sy=h(),st=a("p"),Hy=s("Returns "),hi=a("code"),Ly=s("True"),Oy=s(" if the model is quantizable, "),ui=a("code"),Ky=s("False"),e_=s(" otherwise."),t_=h(),co=a("div"),c(Va.$$.fragment),o_=h(),fi=a("p"),a_=s("Safety checker that arguments are correct - also replaces some NoneType arguments with their default values."),l_=h(),po=a("div"),c(Da.$$.fragment),r_=h(),Ya=a("p"),s_=s(`This method returns the quantization method used for the model. If the model is not quantizable, it returns
`),ci=a("code"),n_=s("None"),i_=s("."),d_=h(),mo=a("div"),c(Sa.$$.fragment),h_=h(),pi=a("p"),u_=s(`Removes all attributes from config which correspond to the default config attributes for better readability and
serializes to a Python dictionary.`),Gh=h(),nt=a("h2"),bo=a("a"),mi=a("span"),c(Ha.$$.fragment),f_=h(),Ml=a("span"),c_=s("Quantization with \u{1F917} "),bi=a("code"),p_=s("optimum"),jh=h(),be=a("p"),m_=s("Please have a look at "),La=a("a"),b_=s("Optimum documentation"),v_=s(" to learn more about quantization methods that are supported by "),vi=a("code"),y_=s("optimum"),__=s(" and see if these are applicable for your use case."),this.h()},l(e){const i=u0('[data-svelte="svelte-1phssyn"]',document.head);_=l(i,"META",{name:!0,content:!0}),i.forEach(o),$=u(e),w=l(e,"H1",{class:!0});var Oa=r(w);k=l(Oa,"A",{id:!0,class:!0,href:!0});var yi=r(k);$e=l(yi,"SPAN",{});var _i=r($e);p(j.$$.fragment,_i),_i.forEach(o),yi.forEach(o),Eo=u(Oa),ke=l(Oa,"SPAN",{});var gi=r(ke);Me=n(gi,"Quantize \u{1F917} Transformers models"),gi.forEach(o),Oa.forEach(o),dt=u(e),Q=l(e,"H2",{class:!0});var Bh=r(Q);A=l(Bh,"A",{id:!0,class:!0,href:!0});var k_=r(A);Te=l(k_,"SPAN",{});var M_=r(Te);p(H.$$.fragment,M_),M_.forEach(o),k_.forEach(o),Ce=u(Bh),T=l(Bh,"SPAN",{});var g_=r(T);ht=l(g_,"CODE",{});var T_=r(ht);tl=n(T_,"AutoGPTQ"),T_.forEach(o),ol=n(g_," Integration"),g_.forEach(o),Bh.forEach(o),$o=u(e),ut=l(e,"P",{});var Fh=r(ut);Bu=n(Fh,"\u{1F917} Transformers has integrated "),Vl=l(Fh,"CODE",{});var C_=r(Vl);Fu=n(C_,"optimum"),C_.forEach(o),xu=n(Fh," API to perform GPTQ quantization on language models. You can load and quantize your model in 8, 4, 3 or even 2 bits without a big drop of performance and faster inference speed! This is supported by most GPU hardwares."),Fh.forEach(o),Ai=u(e),al=l(e,"P",{});var q_=r(al);Xu=n(q_,"To learn more about the the quantization model, check out:"),q_.forEach(o),Ni=u(e),ee=l(e,"UL",{});var Tl=r(ee);ko=l(Tl,"LI",{});var xh=r(ko);Iu=n(xh,"the "),Mo=l(xh,"A",{href:!0,rel:!0});var z_=r(Mo);Ru=n(z_,"GPTQ"),z_.forEach(o),Vu=n(xh," paper"),xh.forEach(o),Du=u(Tl),qe=l(Tl,"LI",{});var Cl=r(qe);Yu=n(Cl,"the "),Dl=l(Cl,"CODE",{});var J_=r(Dl);Su=n(J_,"optimum"),J_.forEach(o),Hu=u(Cl),To=l(Cl,"A",{href:!0,rel:!0});var A_=r(To);Lu=n(A_,"guide"),A_.forEach(o),Ou=n(Cl," on GPTQ quantization"),Cl.forEach(o),Ku=u(Tl),Co=l(Tl,"LI",{});var Xh=r(Co);ef=n(Xh,"the "),qo=l(Xh,"A",{href:!0,rel:!0});var N_=r(qo);Yl=l(N_,"CODE",{});var U_=r(Yl);tf=n(U_,"AutoGPTQ"),U_.forEach(o),N_.forEach(o),of=n(Xh," library used as the backend"),Xh.forEach(o),Tl.forEach(o),Ui=u(e),ze=l(e,"H3",{class:!0});var Ih=r(ze);ft=l(Ih,"A",{id:!0,class:!0,href:!0});var W_=r(ft);Sl=l(W_,"SPAN",{});var Z_=r(Sl);p(zo.$$.fragment,Z_),Z_.forEach(o),W_.forEach(o),af=u(Ih),Hl=l(Ih,"SPAN",{});var P_=r(Hl);lf=n(P_,"Requirements"),P_.forEach(o),Ih.forEach(o),Wi=u(e),ll=l(e,"P",{});var G_=r(ll);rf=n(G_,"You need to have the following requirements installed to run the code below:"),G_.forEach(o),Zi=u(e),F=l(e,"UL",{});var vo=r(F);Ll=l(vo,"LI",{});var j_=r(Ll);ct=l(j_,"P",{});var wi=r(ct);sf=n(wi,"Install latest "),Ol=l(wi,"CODE",{});var Q_=r(Ol);nf=n(Q_,"AutoGPTQ"),Q_.forEach(o),df=n(wi,` library
`),Kl=l(wi,"CODE",{});var B_=r(Kl);hf=n(B_,"pip install auto-gptq"),B_.forEach(o),wi.forEach(o),j_.forEach(o),uf=u(vo),er=l(vo,"LI",{});var F_=r(er);pt=l(F_,"P",{});var Ei=r(pt);ff=n(Ei,"Install latest "),tr=l(Ei,"CODE",{});var x_=r(tr);cf=n(x_,"optimum"),x_.forEach(o),pf=n(Ei,` from source
`),or=l(Ei,"CODE",{});var X_=r(or);mf=n(X_,"pip install git+https://github.com/huggingface/optimum.git"),X_.forEach(o),Ei.forEach(o),F_.forEach(o),bf=u(vo),ar=l(vo,"LI",{});var I_=r(ar);mt=l(I_,"P",{});var $i=r(mt);vf=n($i,"Install latest "),lr=l($i,"CODE",{});var R_=r(lr);yf=n(R_,"transformers"),R_.forEach(o),_f=n($i,` from source
`),rr=l($i,"CODE",{});var V_=r(rr);gf=n(V_,"pip install git+https://github.com/huggingface/transformers.git"),V_.forEach(o),$i.forEach(o),I_.forEach(o),wf=u(vo),sr=l(vo,"LI",{});var D_=r(sr);bt=l(D_,"P",{});var ki=r(bt);Ef=n(ki,"Install latest "),nr=l(ki,"CODE",{});var Y_=r(nr);$f=n(Y_,"accelerate"),Y_.forEach(o),kf=n(ki,` library
`),ir=l(ki,"CODE",{});var S_=r(ir);Mf=n(S_,"pip install --upgrade accelerate"),S_.forEach(o),ki.forEach(o),D_.forEach(o),vo.forEach(o),Pi=u(e),rl=l(e,"P",{});var H_=r(rl);Tf=n(H_,"Note that GPTQ integration supports for now only text models and you may encounter unexpected behaviour for vision, speech or multi-modal models."),H_.forEach(o),Gi=u(e),Je=l(e,"H3",{class:!0});var Rh=r(Je);vt=l(Rh,"A",{id:!0,class:!0,href:!0});var L_=r(vt);dr=l(L_,"SPAN",{});var O_=r(dr);p(Jo.$$.fragment,O_),O_.forEach(o),L_.forEach(o),Cf=u(Rh),hr=l(Rh,"SPAN",{});var K_=r(hr);qf=n(K_,"Load and quantize a model"),K_.forEach(o),Rh.forEach(o),ji=u(e),yt=l(e,"P",{});var Vh=r(yt);zf=n(Vh,"GPTQ is a quantization method that requires weights calibration before using the quantized models. If you want to quantize transformers model from scratch, it might take some time before producing the quantized model (~5 min on a Google colab for "),ur=l(Vh,"CODE",{});var eg=r(ur);Jf=n(eg,"facebook/opt-350m"),eg.forEach(o),Af=n(Vh," model)."),Vh.forEach(o),Qi=u(e),sl=l(e,"P",{});var tg=r(sl);Nf=n(tg,"Hence, there are two different scenarios where you want to use GPTQ-quantized models. The first use case would be to load models that has been already quantized by other users that are available on the Hub, the second use case would be to quantize your model from scratch and save it or push it on the Hub so that other users can also use it."),tg.forEach(o),Bi=u(e),Ae=l(e,"H4",{class:!0});var Dh=r(Ae);_t=l(Dh,"A",{id:!0,class:!0,href:!0});var og=r(_t);fr=l(og,"SPAN",{});var ag=r(fr);p(Ao.$$.fragment,ag),ag.forEach(o),og.forEach(o),Uf=u(Dh),cr=l(Dh,"SPAN",{});var lg=r(cr);Wf=n(lg,"GPTQ Configuration"),lg.forEach(o),Dh.forEach(o),Fi=u(e),N=l(e,"P",{});var ve=r(N);Zf=n(ve,"In order to load and quantize a model, you need to create a "),nl=l(ve,"A",{href:!0});var rg=r(nl);Pf=n(rg,"GPTQConfig"),rg.forEach(o),Gf=n(ve,". You need to pass the number of "),pr=l(ve,"CODE",{});var sg=r(pr);jf=n(sg,"bits"),sg.forEach(o),Qf=n(ve,", a "),mr=l(ve,"CODE",{});var ng=r(mr);Bf=n(ng,"dataset"),ng.forEach(o),Ff=n(ve," in order to calibrate the quantization and the "),br=l(ve,"CODE",{});var ig=r(br);xf=n(ig,"tokenizer"),ig.forEach(o),Xf=n(ve," of the model in order prepare the dataset."),ve.forEach(o),xi=u(e),p(No.$$.fragment,e),Xi=u(e),il=l(e,"P",{});var dg=r(il);If=n(dg,"Note that you can pass your own dataset as a list of string. However, it is highly recommended to use the dataset from the GPTQ paper."),dg.forEach(o),Ii=u(e),p(Uo.$$.fragment,e),Ri=u(e),Ne=l(e,"H4",{class:!0});var Yh=r(Ne);gt=l(Yh,"A",{id:!0,class:!0,href:!0});var hg=r(gt);vr=l(hg,"SPAN",{});var ug=r(vr);p(Wo.$$.fragment,ug),ug.forEach(o),hg.forEach(o),Rf=u(Yh),yr=l(Yh,"SPAN",{});var fg=r(yr);Vf=n(fg,"Quantization"),fg.forEach(o),Yh.forEach(o),Vi=u(e),te=l(e,"P",{});var ql=r(te);Df=n(ql,"You can quantize a model by using "),_r=l(ql,"CODE",{});var cg=r(_r);Yf=n(cg,"from_pretrained"),cg.forEach(o),Sf=n(ql," and setting the "),gr=l(ql,"CODE",{});var pg=r(gr);Hf=n(pg,"quantization_config"),pg.forEach(o),Lf=n(ql,"."),ql.forEach(o),Di=u(e),p(Zo.$$.fragment,e),Yi=u(e),dl=l(e,"P",{});var mg=r(dl);Of=n(mg,"Note that you will need a GPU to quantize a model. We will put the model in the cpu and move the modules back and forth to the gpu in order to quantize them."),mg.forEach(o),Si=u(e),wt=l(e,"P",{});var Sh=r(wt);Kf=n(Sh,"If you want to maximize your gpus usage while using cpu offload, you can set "),wr=l(Sh,"CODE",{});var bg=r(wr);ec=n(bg,'device_map = "auto"'),bg.forEach(o),tc=n(Sh,"."),Sh.forEach(o),Hi=u(e),p(Po.$$.fragment,e),Li=u(e),C=l(e,"P",{});var D=r(C);oc=n(D,"Note that disk offload is not supported. Furthermore, if you are out of memory because of the dataset, you may have to pass "),Er=l(D,"CODE",{});var vg=r(Er);ac=n(vg,"max_memory"),vg.forEach(o),lc=n(D," in "),$r=l(D,"CODE",{});var yg=r($r);rc=n(yg,"from_pretained"),yg.forEach(o),sc=n(D,". Checkout this "),Go=l(D,"A",{href:!0,rel:!0});var _g=r(Go);nc=n(_g,"guide"),_g.forEach(o),ic=n(D," to learn more about "),kr=l(D,"CODE",{});var gg=r(kr);dc=n(gg,"device_map"),gg.forEach(o),hc=n(D," and "),Mr=l(D,"CODE",{});var wg=r(Mr);uc=n(wg,"max_memory"),wg.forEach(o),fc=n(D,"."),D.forEach(o),Oi=u(e),p(Et.$$.fragment,e),Ki=u(e),Ue=l(e,"H3",{class:!0});var Hh=r(Ue);$t=l(Hh,"A",{id:!0,class:!0,href:!0});var Eg=r($t);Tr=l(Eg,"SPAN",{});var $g=r(Tr);p(jo.$$.fragment,$g),$g.forEach(o),Eg.forEach(o),cc=u(Hh),Cr=l(Hh,"SPAN",{});var kg=r(Cr);pc=n(kg,"Push quantized model to \u{1F917} Hub"),kg.forEach(o),Hh.forEach(o),ed=u(e),kt=l(e,"P",{});var Lh=r(kt);mc=n(Lh,"You can push the quantized model like any \u{1F917} model to Hub with "),qr=l(Lh,"CODE",{});var Mg=r(qr);bc=n(Mg,"push_to_hub"),Mg.forEach(o),vc=n(Lh,". The quantization config will be saved and pushed along the model."),Lh.forEach(o),td=u(e),p(Qo.$$.fragment,e),od=u(e),Mt=l(e,"P",{});var Oh=r(Mt);yc=n(Oh,"If you want to save your quantized model on your local machine, you can also do it with "),zr=l(Oh,"CODE",{});var Tg=r(zr);_c=n(Tg,"save_pretrained"),Tg.forEach(o),gc=n(Oh,":"),Oh.forEach(o),ad=u(e),p(Bo.$$.fragment,e),ld=u(e),oe=l(e,"P",{});var zl=r(oe);wc=n(zl,"Note that if you have quantized your model with a "),Jr=l(zl,"CODE",{});var Cg=r(Jr);Ec=n(Cg,"device_map"),Cg.forEach(o),$c=n(zl,", make sure to move the entire model to one of your gpus or the "),Ar=l(zl,"CODE",{});var qg=r(Ar);kc=n(qg,"cpu"),qg.forEach(o),Mc=n(zl," before saving it."),zl.forEach(o),rd=u(e),p(Fo.$$.fragment,e),sd=u(e),We=l(e,"H3",{class:!0});var Kh=r(We);Tt=l(Kh,"A",{id:!0,class:!0,href:!0});var zg=r(Tt);Nr=l(zg,"SPAN",{});var Jg=r(Nr);p(xo.$$.fragment,Jg),Jg.forEach(o),zg.forEach(o),Tc=u(Kh),Ur=l(Kh,"SPAN",{});var Ag=r(Ur);Cc=n(Ag,"Load a quantized model from the \u{1F917} Hub"),Ag.forEach(o),Kh.forEach(o),nd=u(e),ae=l(e,"P",{});var Jl=r(ae);qc=n(Jl,"You can load a quantized model from the Hub by using "),Wr=l(Jl,"CODE",{});var Ng=r(Wr);zc=n(Ng,"from_pretrained"),Ng.forEach(o),Jc=n(Jl,`.
Make sure that the pushed weights are quantized, by checking that the attribute `),Zr=l(Jl,"CODE",{});var Ug=r(Zr);Ac=n(Ug,"quantization_config"),Ug.forEach(o),Nc=n(Jl," is present in the model configuration object."),Jl.forEach(o),id=u(e),p(Xo.$$.fragment,e),dd=u(e),le=l(e,"P",{});var Al=r(le);Uc=n(Al,"If you want to load a model faster and without allocating more memory than needed, the "),Pr=l(Al,"CODE",{});var Wg=r(Pr);Wc=n(Wg,"device_map"),Wg.forEach(o),Zc=n(Al," argument also works with quantized model. Make sure that you have "),Gr=l(Al,"CODE",{});var Zg=r(Gr);Pc=n(Zg,"accelerate"),Zg.forEach(o),Gc=n(Al," library installed."),Al.forEach(o),hd=u(e),p(Io.$$.fragment,e),ud=u(e),Ze=l(e,"H3",{class:!0});var eu=r(Ze);Ct=l(eu,"A",{id:!0,class:!0,href:!0});var Pg=r(Ct);jr=l(Pg,"SPAN",{});var Gg=r(jr);p(Ro.$$.fragment,Gg),Gg.forEach(o),Pg.forEach(o),jc=u(eu),Qr=l(eu,"SPAN",{});var jg=r(Qr);Qc=n(jg,"Exllama kernels for faster inference"),jg.forEach(o),eu.forEach(o),fd=u(e),re=l(e,"P",{});var Nl=r(re);Bc=n(Nl,"For 4-bit model, you can use the exllama kernels in order to a faster inference speed. It is activated by default. You can change that behavior by passing "),Br=l(Nl,"CODE",{});var Qg=r(Br);Fc=n(Qg,"disable_exllama"),Qg.forEach(o),xc=n(Nl," in "),hl=l(Nl,"A",{href:!0});var Bg=r(hl);Xc=n(Bg,"GPTQConfig"),Bg.forEach(o),Ic=n(Nl,". This will overwrite the quantization config stored in the config. Note that you will only be able to overwrite the attributes related to the kernels. Furthermore, you need to have the entire model on gpus if you want to use exllama kernels."),Nl.forEach(o),cd=u(e),p(Vo.$$.fragment,e),pd=u(e),ul=l(e,"P",{});var Fg=r(ul);Rc=n(Fg,"Note that only 4-bit models are supported for now. Furthermore, it is recommended to deactivate the exllama kernels if you are finetuning a quantized model with peft."),Fg.forEach(o),md=u(e),Pe=l(e,"H4",{class:!0});var tu=r(Pe);qt=l(tu,"A",{id:!0,class:!0,href:!0});var xg=r(qt);Fr=l(xg,"SPAN",{});var Xg=r(Fr);p(Do.$$.fragment,Xg),Xg.forEach(o),xg.forEach(o),Vc=u(tu),xr=l(tu,"SPAN",{});var Ig=r(xr);Dc=n(Ig,"Fine-tune a quantized model"),Ig.forEach(o),tu.forEach(o),bd=u(e),zt=l(e,"P",{});var ou=r(zt);Yc=n(ou,`With the official support of adapters in the Hugging Face ecosystem, you can fine-tune models that have been quantized with GPTQ.
Please have a look at `),Yo=l(ou,"A",{href:!0,rel:!0});var Rg=r(Yo);Xr=l(Rg,"CODE",{});var Vg=r(Xr);Sc=n(Vg,"peft"),Vg.forEach(o),Rg.forEach(o),Hc=n(ou," library for more details."),ou.forEach(o),vd=u(e),Ge=l(e,"H3",{class:!0});var au=r(Ge);Jt=l(au,"A",{id:!0,class:!0,href:!0});var Dg=r(Jt);Ir=l(Dg,"SPAN",{});var Yg=r(Ir);p(So.$$.fragment,Yg),Yg.forEach(o),Dg.forEach(o),Lc=u(au),Rr=l(au,"SPAN",{});var Sg=r(Rr);Oc=n(Sg,"Example demo"),Sg.forEach(o),au.forEach(o),yd=u(e),At=l(e,"P",{});var lu=r(At);Kc=n(lu,"Check out the Google Colab "),Ho=l(lu,"A",{href:!0,rel:!0});var Hg=r(Ho);ep=n(Hg,"notebook"),Hg.forEach(o),tp=n(lu," to learn how to quantize your model with GPTQ and how finetune the quantized model with peft."),lu.forEach(o),_d=u(e),je=l(e,"H3",{class:!0});var ru=r(je);Nt=l(ru,"A",{id:!0,class:!0,href:!0});var Lg=r(Nt);Vr=l(Lg,"SPAN",{});var Og=r(Vr);p(Lo.$$.fragment,Og),Og.forEach(o),Lg.forEach(o),op=u(ru),Dr=l(ru,"SPAN",{});var Kg=r(Dr);ap=n(Kg,"GPTQConfig"),Kg.forEach(o),ru.forEach(o),gd=u(e),L=l(e,"DIV",{class:!0});var Ul=r(L);p(Oo.$$.fragment,Ul),lp=u(Ul),Ko=l(Ul,"P",{});var su=r(Ko);rp=n(su,`This is a wrapper class about all possible attributes and features that you can play with a model that has been
loaded using `),Yr=l(su,"CODE",{});var e1=r(Yr);sp=n(e1,"optimum"),e1.forEach(o),np=n(su," api for gptq quantization relying on auto_gptq backend."),su.forEach(o),ip=u(Ul),Ut=l(Ul,"DIV",{class:!0});var nu=r(Ut);p(ea.$$.fragment,nu),dp=u(nu),Sr=l(nu,"P",{});var t1=r(Sr);hp=n(t1,"Safety checker that arguments are correct"),t1.forEach(o),nu.forEach(o),Ul.forEach(o),wd=u(e),Qe=l(e,"H2",{class:!0});var iu=r(Qe);Wt=l(iu,"A",{id:!0,class:!0,href:!0});var o1=r(Wt);Hr=l(o1,"SPAN",{});var a1=r(Hr);p(ta.$$.fragment,a1),a1.forEach(o),o1.forEach(o),up=u(iu),fl=l(iu,"SPAN",{});var w_=r(fl);Lr=l(w_,"CODE",{});var l1=r(Lr);fp=n(l1,"bitsandbytes"),l1.forEach(o),cp=n(w_," Integration"),w_.forEach(o),iu.forEach(o),Ed=u(e),x=l(e,"P",{});var yo=r(x);pp=n(yo,"\u{1F917} Transformers is closely integrated with most used modules on "),Or=l(yo,"CODE",{});var r1=r(Or);mp=n(r1,"bitsandbytes"),r1.forEach(o),bp=n(yo,`. You can load your model in 8-bit precision with few lines of code.
This is supported by most of the GPU hardwares since the `),Kr=l(yo,"CODE",{});var s1=r(Kr);vp=n(s1,"0.37.0"),s1.forEach(o),yp=n(yo," release of "),es=l(yo,"CODE",{});var n1=r(es);_p=n(n1,"bitsandbytes"),n1.forEach(o),gp=n(yo,"."),yo.forEach(o),$d=u(e),se=l(e,"P",{});var Wl=r(se);wp=n(Wl,"Learn more about the quantization method in the "),oa=l(Wl,"A",{href:!0,rel:!0});var i1=r(oa);Ep=n(i1,"LLM.int8()"),i1.forEach(o),$p=n(Wl," paper, or the "),aa=l(Wl,"A",{href:!0,rel:!0});var d1=r(aa);kp=n(d1,"blogpost"),d1.forEach(o),Mp=n(Wl," about the collaboration."),Wl.forEach(o),kd=u(e),ne=l(e,"P",{});var Zl=r(ne);Tp=n(Zl,"Since its "),ts=l(Zl,"CODE",{});var h1=r(ts);Cp=n(h1,"0.39.0"),h1.forEach(o),qp=n(Zl," release, you can load any model that supports "),os=l(Zl,"CODE",{});var u1=r(os);zp=n(u1,"device_map"),u1.forEach(o),Jp=n(Zl," using 4-bit quantization, leveraging FP4 data type."),Zl.forEach(o),Md=u(e),Zt=l(e,"P",{});var du=r(Zt);Ap=n(du,"If you want to quantize your own pytorch model, check out this "),la=l(du,"A",{href:!0,rel:!0});var f1=r(la);Np=n(f1,"documentation"),f1.forEach(o),Up=n(du," from \u{1F917} Accelerate library."),du.forEach(o),Td=u(e),Pt=l(e,"P",{});var hu=r(Pt);Wp=n(hu,"Here are the things you can do using "),as=l(hu,"CODE",{});var c1=r(as);Zp=n(c1,"bitsandbytes"),c1.forEach(o),Pp=n(hu," integration"),hu.forEach(o),Cd=u(e),Be=l(e,"H3",{class:!0});var uu=r(Be);Gt=l(uu,"A",{id:!0,class:!0,href:!0});var p1=r(Gt);ls=l(p1,"SPAN",{});var m1=r(ls);p(ra.$$.fragment,m1),m1.forEach(o),p1.forEach(o),Gp=u(uu),rs=l(uu,"SPAN",{});var b1=r(rs);jp=n(b1,"General usage"),b1.forEach(o),uu.forEach(o),qd=u(e),U=l(e,"P",{});var ye=r(U);Qp=n(ye,"You can quantize a model by using the "),ss=l(ye,"CODE",{});var v1=r(ss);Bp=n(v1,"load_in_8bit"),v1.forEach(o),Fp=n(ye," or "),ns=l(ye,"CODE",{});var y1=r(ns);xp=n(y1,"load_in_4bit"),y1.forEach(o),Xp=n(ye," argument when calling the "),cl=l(ye,"A",{href:!0});var _1=r(cl);Ip=n(_1,"from_pretrained()"),_1.forEach(o),Rp=n(ye," method as long as your model supports loading with \u{1F917} Accelerate and contains "),is=l(ye,"CODE",{});var g1=r(is);Vp=n(g1,"torch.nn.Linear"),g1.forEach(o),Dp=n(ye," layers. This should work for any modality as well."),ye.forEach(o),zd=u(e),p(sa.$$.fragment,e),Jd=u(e),W=l(e,"P",{});var _e=r(W);Yp=n(_e,"By default all other modules (e.g. "),ds=l(_e,"CODE",{});var w1=r(ds);Sp=n(w1,"torch.nn.LayerNorm"),w1.forEach(o),Hp=n(_e,") will be converted in "),hs=l(_e,"CODE",{});var E1=r(hs);Lp=n(E1,"torch.float16"),E1.forEach(o),Op=n(_e,", but if you want to change their "),us=l(_e,"CODE",{});var $1=r(us);Kp=n($1,"dtype"),$1.forEach(o),em=n(_e," you can overwrite the "),fs=l(_e,"CODE",{});var k1=r(fs);tm=n(k1,"torch_dtype"),k1.forEach(o),om=n(_e," argument:"),_e.forEach(o),Ad=u(e),p(na.$$.fragment,e),Nd=u(e),Fe=l(e,"H3",{class:!0});var fu=r(Fe);jt=l(fu,"A",{id:!0,class:!0,href:!0});var M1=r(jt);cs=l(M1,"SPAN",{});var T1=r(cs);p(ia.$$.fragment,T1),T1.forEach(o),M1.forEach(o),am=u(fu),ps=l(fu,"SPAN",{});var C1=r(ps);lm=n(C1,"FP4 quantization"),C1.forEach(o),fu.forEach(o),Ud=u(e),xe=l(e,"H4",{class:!0});var cu=r(xe);Qt=l(cu,"A",{id:!0,class:!0,href:!0});var q1=r(Qt);ms=l(q1,"SPAN",{});var z1=r(ms);p(da.$$.fragment,z1),z1.forEach(o),q1.forEach(o),rm=u(cu),bs=l(cu,"SPAN",{});var J1=r(bs);sm=n(J1,"Requirements"),J1.forEach(o),cu.forEach(o),Wd=u(e),pl=l(e,"P",{});var A1=r(pl);nm=n(A1,"Make sure that you have installed the requirements below before running any of the code snippets below."),A1.forEach(o),Zd=u(e),ie=l(e,"UL",{});var Pl=r(ie);vs=l(Pl,"LI",{});var N1=r(vs);Bt=l(N1,"P",{});var Mi=r(Bt);im=n(Mi,"Latest "),ys=l(Mi,"CODE",{});var U1=r(ys);dm=n(U1,"bitsandbytes"),U1.forEach(o),hm=n(Mi,` library
`),_s=l(Mi,"CODE",{});var W1=r(_s);um=n(W1,"pip install bitsandbytes>=0.39.0"),W1.forEach(o),Mi.forEach(o),N1.forEach(o),fm=u(Pl),gs=l(Pl,"LI",{});var Z1=r(gs);Ft=l(Z1,"P",{});var Ti=r(Ft);cm=n(Ti,"Install latest "),ws=l(Ti,"CODE",{});var P1=r(ws);pm=n(P1,"accelerate"),P1.forEach(o),mm=u(Ti),Es=l(Ti,"CODE",{});var G1=r(Es);bm=n(G1,"pip install --upgrade accelerate"),G1.forEach(o),Ti.forEach(o),Z1.forEach(o),vm=u(Pl),$s=l(Pl,"LI",{});var j1=r($s);xt=l(j1,"P",{});var Ci=r(xt);ym=n(Ci,"Install latest "),ks=l(Ci,"CODE",{});var Q1=r(ks);_m=n(Q1,"transformers"),Q1.forEach(o),gm=u(Ci),Ms=l(Ci,"CODE",{});var B1=r(Ms);wm=n(B1,"pip install --upgrade transformers"),B1.forEach(o),Ci.forEach(o),j1.forEach(o),Pl.forEach(o),Pd=u(e),Xe=l(e,"H4",{class:!0});var pu=r(Xe);Xt=l(pu,"A",{id:!0,class:!0,href:!0});var F1=r(Xt);Ts=l(F1,"SPAN",{});var x1=r(Ts);p(ha.$$.fragment,x1),x1.forEach(o),F1.forEach(o),Em=u(pu),Cs=l(pu,"SPAN",{});var X1=r(Cs);$m=n(X1,"Tips and best practices"),X1.forEach(o),pu.forEach(o),Gd=u(e),X=l(e,"UL",{});var _o=r(X);qs=l(_o,"LI",{});var I1=r(qs);It=l(I1,"P",{});var qi=r(It);zs=l(qi,"STRONG",{});var R1=r(zs);km=n(R1,"Advanced usage:"),R1.forEach(o),Mm=n(qi," Refer to "),ua=l(qi,"A",{href:!0,rel:!0});var V1=r(ua);Tm=n(V1,"this Google Colab notebook"),V1.forEach(o),Cm=n(qi," for advanced usage of 4-bit quantization with all the possible options."),qi.forEach(o),I1.forEach(o),qm=u(_o),Js=l(_o,"LI",{});var D1=r(Js);Z=l(D1,"P",{});var O=r(Z);fa=l(O,"STRONG",{});var mu=r(fa);zm=n(mu,"Faster inference with "),As=l(mu,"CODE",{});var Y1=r(As);Jm=n(Y1,"batch_size=1"),Y1.forEach(o),Am=n(mu," :"),mu.forEach(o),Nm=n(O," Since the "),Ns=l(O,"CODE",{});var S1=r(Ns);Um=n(S1,"0.40.0"),S1.forEach(o),Wm=n(O," release of bitsandbytes, for "),Us=l(O,"CODE",{});var H1=r(Us);Zm=n(H1,"batch_size=1"),H1.forEach(o),Pm=n(O," you can benefit from fast inference. Check out "),ca=l(O,"A",{href:!0,rel:!0});var L1=r(ca);Gm=n(L1,"these release notes"),L1.forEach(o),jm=n(O," and make sure to have a version that is greater than "),Ws=l(O,"CODE",{});var O1=r(Ws);Qm=n(O1,"0.40.0"),O1.forEach(o),Bm=n(O," to benefit from this feature out of the box."),O.forEach(o),D1.forEach(o),Fm=u(_o),Zs=l(_o,"LI",{});var K1=r(Zs);de=l(K1,"P",{});var Ka=r(de);Ps=l(Ka,"STRONG",{});var ew=r(Ps);xm=n(ew,"Training:"),ew.forEach(o),Xm=n(Ka," According to "),pa=l(Ka,"A",{href:!0,rel:!0});var tw=r(pa);Im=n(tw,"QLoRA paper"),tw.forEach(o),Rm=n(Ka,", for training 4-bit base models (e.g. using LoRA adapters) one should use "),Gs=l(Ka,"CODE",{});var ow=r(Gs);Vm=n(ow,"bnb_4bit_quant_type='nf4'"),ow.forEach(o),Dm=n(Ka,"."),Ka.forEach(o),K1.forEach(o),Ym=u(_o),js=l(_o,"LI",{});var aw=r(js);I=l(aw,"P",{});var it=r(I);Qs=l(it,"STRONG",{});var lw=r(Qs);Sm=n(lw,"Inference:"),lw.forEach(o),Hm=n(it," For inference, "),Bs=l(it,"CODE",{});var rw=r(Bs);Lm=n(rw,"bnb_4bit_quant_type"),rw.forEach(o),Om=n(it," does not have a huge impact on the performance. However for consistency with the model\u2019s weights, make sure you use the same "),Fs=l(it,"CODE",{});var sw=r(Fs);Km=n(sw,"bnb_4bit_compute_dtype"),sw.forEach(o),eb=n(it," and "),xs=l(it,"CODE",{});var nw=r(xs);tb=n(nw,"torch_dtype"),nw.forEach(o),ob=n(it," arguments."),it.forEach(o),aw.forEach(o),_o.forEach(o),jd=u(e),Ie=l(e,"H4",{class:!0});var bu=r(Ie);Rt=l(bu,"A",{id:!0,class:!0,href:!0});var iw=r(Rt);Xs=l(iw,"SPAN",{});var dw=r(Xs);p(ma.$$.fragment,dw),dw.forEach(o),iw.forEach(o),ab=u(bu),Is=l(bu,"SPAN",{});var hw=r(Is);lb=n(hw,"Load a large model in 4bit"),hw.forEach(o),bu.forEach(o),Qd=u(e),he=l(e,"P",{});var Gl=r(he);rb=n(Gl,"By using "),Rs=l(Gl,"CODE",{});var uw=r(Rs);sb=n(uw,"load_in_4bit=True"),uw.forEach(o),nb=n(Gl," when calling the "),Vs=l(Gl,"CODE",{});var fw=r(Vs);ib=n(fw,".from_pretrained"),fw.forEach(o),db=n(Gl," method, you can divide your memory use by 4 (roughly)."),Gl.forEach(o),Bd=u(e),p(ba.$$.fragment,e),Fd=u(e),p(Vt.$$.fragment,e),xd=u(e),Re=l(e,"H3",{class:!0});var vu=r(Re);Dt=l(vu,"A",{id:!0,class:!0,href:!0});var cw=r(Dt);Ds=l(cw,"SPAN",{});var pw=r(Ds);p(va.$$.fragment,pw),pw.forEach(o),cw.forEach(o),hb=u(vu),Ys=l(vu,"SPAN",{});var mw=r(Ys);ub=n(mw,"Load a large model in 8bit"),mw.forEach(o),vu.forEach(o),Xd=u(e),ue=l(e,"P",{});var jl=r(ue);fb=n(jl,"You can load a model by roughly halving the memory requirements by using "),Ss=l(jl,"CODE",{});var bw=r(Ss);cb=n(bw,"load_in_8bit=True"),bw.forEach(o),pb=n(jl," argument when calling "),Hs=l(jl,"CODE",{});var vw=r(Hs);mb=n(vw,".from_pretrained"),vw.forEach(o),bb=n(jl," method"),jl.forEach(o),Id=u(e),p(ya.$$.fragment,e),Rd=u(e),Yt=l(e,"P",{});var yu=r(Yt);vb=n(yu,"Then, use your model as you would usually use a "),ml=l(yu,"A",{href:!0});var yw=r(ml);yb=n(yw,"PreTrainedModel"),yw.forEach(o),_b=n(yu,"."),yu.forEach(o),Vd=u(e),St=l(e,"P",{});var _u=r(St);gb=n(_u,"You can check the memory footprint of your model with "),Ls=l(_u,"CODE",{});var _w=r(Ls);wb=n(_w,"get_memory_footprint"),_w.forEach(o),Eb=n(_u," method."),_u.forEach(o),Dd=u(e),p(_a.$$.fragment,e),Yd=u(e),bl=l(e,"P",{});var gw=r(bl);$b=n(gw,"With this integration we were able to load large models on smaller devices and run them without any issue."),gw.forEach(o),Sd=u(e),p(Ht.$$.fragment,e),Hd=u(e),Ve=l(e,"H4",{class:!0});var gu=r(Ve);Lt=l(gu,"A",{id:!0,class:!0,href:!0});var ww=r(Lt);Os=l(ww,"SPAN",{});var Ew=r(Os);p(ga.$$.fragment,Ew),Ew.forEach(o),ww.forEach(o),kb=u(gu),Ks=l(gu,"SPAN",{});var $w=r(Ks);Mb=n($w,"Advanced use cases"),$w.forEach(o),gu.forEach(o),Ld=u(e),vl=l(e,"P",{});var kw=r(vl);Tb=n(kw,"Here we will cover some advanced use cases you can perform with FP4 quantization"),kw.forEach(o),Od=u(e),De=l(e,"H5",{class:!0});var wu=r(De);Ot=l(wu,"A",{id:!0,class:!0,href:!0});var Mw=r(Ot);en=l(Mw,"SPAN",{});var Tw=r(en);p(wa.$$.fragment,Tw),Tw.forEach(o),Mw.forEach(o),Cb=u(wu),tn=l(wu,"SPAN",{});var Cw=r(tn);qb=n(Cw,"Change the compute dtype"),Cw.forEach(o),wu.forEach(o),Kd=u(e),fe=l(e,"P",{});var Ql=r(fe);zb=n(Ql,"The compute dtype is used to change the dtype that will be used during computation. For example, hidden states could be in "),on=l(Ql,"CODE",{});var qw=r(on);Jb=n(qw,"float32"),qw.forEach(o),Ab=n(Ql," but computation can be set to bf16 for speedups. By default, the compute dtype is set to "),an=l(Ql,"CODE",{});var zw=r(an);Nb=n(zw,"float32"),zw.forEach(o),Ub=n(Ql,"."),Ql.forEach(o),eh=u(e),p(Ea.$$.fragment,e),th=u(e),Ye=l(e,"H5",{class:!0});var Eu=r(Ye);Kt=l(Eu,"A",{id:!0,class:!0,href:!0});var Jw=r(Kt);ln=l(Jw,"SPAN",{});var Aw=r(ln);p($a.$$.fragment,Aw),Aw.forEach(o),Jw.forEach(o),Wb=u(Eu),rn=l(Eu,"SPAN",{});var Nw=r(rn);Zb=n(Nw,"Using NF4 (Normal Float 4) data type"),Nw.forEach(o),Eu.forEach(o),oh=u(e),yl=l(e,"P",{});var Uw=r(yl);Pb=n(Uw,"You can also use the NF4 data type, which is a new 4bit datatype adapted for weights that have been initialized using a normal distribution. For that run:"),Uw.forEach(o),ah=u(e),p(ka.$$.fragment,e),lh=u(e),Se=l(e,"H5",{class:!0});var $u=r(Se);eo=l($u,"A",{id:!0,class:!0,href:!0});var Ww=r(eo);sn=l(Ww,"SPAN",{});var Zw=r(sn);p(Ma.$$.fragment,Zw),Zw.forEach(o),Ww.forEach(o),Gb=u($u),nn=l($u,"SPAN",{});var Pw=r(nn);jb=n(Pw,"Use nested quantization for more memory efficient inference"),Pw.forEach(o),$u.forEach(o),rh=u(e),_l=l(e,"P",{});var Gw=r(_l);Qb=n(Gw,"We also advise users to use the nested quantization technique. This saves more memory at no additional performance - from our empirical observations, this enables fine-tuning llama-13b model on an NVIDIA-T4 16GB with a sequence length of 1024, batch size of 1 and gradient accumulation steps of 4."),Gw.forEach(o),sh=u(e),p(Ta.$$.fragment,e),nh=u(e),He=l(e,"H3",{class:!0});var ku=r(He);to=l(ku,"A",{id:!0,class:!0,href:!0});var jw=r(to);dn=l(jw,"SPAN",{});var Qw=r(dn);p(Ca.$$.fragment,Qw),Qw.forEach(o),jw.forEach(o),Bb=u(ku),hn=l(ku,"SPAN",{});var Bw=r(hn);Fb=n(Bw,"Push quantized models on the \u{1F917} Hub"),Bw.forEach(o),ku.forEach(o),ih=u(e),R=l(e,"P",{});var go=r(R);xb=n(go,"You can push a quantized model on the Hub by naively using "),un=l(go,"CODE",{});var Fw=r(un);Xb=n(Fw,"push_to_hub"),Fw.forEach(o),Ib=n(go,` method. This will first push the quantization configuration file, then push the quantized model weights.
Make sure to use `),fn=l(go,"CODE",{});var xw=r(fn);Rb=n(xw,"bitsandbytes>0.37.2"),xw.forEach(o),Vb=n(go," (at this time of writing, we tested it on "),cn=l(go,"CODE",{});var Xw=r(cn);Db=n(Xw,"bitsandbytes==0.38.0.post1"),Xw.forEach(o),Yb=n(go,") to be able to use this feature."),go.forEach(o),dh=u(e),p(qa.$$.fragment,e),hh=u(e),p(oo.$$.fragment,e),uh=u(e),Le=l(e,"H3",{class:!0});var Mu=r(Le);ao=l(Mu,"A",{id:!0,class:!0,href:!0});var Iw=r(ao);pn=l(Iw,"SPAN",{});var Rw=r(pn);p(za.$$.fragment,Rw),Rw.forEach(o),Iw.forEach(o),Sb=u(Mu),mn=l(Mu,"SPAN",{});var Vw=r(mn);Hb=n(Vw,"Load a quantized model from the \u{1F917} Hub"),Vw.forEach(o),Mu.forEach(o),fh=u(e),ce=l(e,"P",{});var Bl=r(ce);Lb=n(Bl,"You can load a quantized model from the Hub by using "),bn=l(Bl,"CODE",{});var Dw=r(bn);Ob=n(Dw,"from_pretrained"),Dw.forEach(o),Kb=n(Bl," method. Make sure that the pushed weights are quantized, by checking that the attribute "),vn=l(Bl,"CODE",{});var Yw=r(vn);ev=n(Yw,"quantization_config"),Yw.forEach(o),tv=n(Bl," is present in the model configuration object."),Bl.forEach(o),ch=u(e),p(Ja.$$.fragment,e),ph=u(e),q=l(e,"P",{});var Y=r(q);ov=n(Y,"Note that in this case, you don\u2019t need to specify the arguments "),yn=l(Y,"CODE",{});var Sw=r(yn);av=n(Sw,"load_in_8bit=True"),Sw.forEach(o),lv=n(Y,", but you need to make sure that "),_n=l(Y,"CODE",{});var Hw=r(_n);rv=n(Hw,"bitsandbytes"),Hw.forEach(o),sv=n(Y," and "),gn=l(Y,"CODE",{});var Lw=r(gn);nv=n(Lw,"accelerate"),Lw.forEach(o),iv=n(Y,` are installed.
Note also that `),wn=l(Y,"CODE",{});var Ow=r(wn);dv=n(Ow,"device_map"),Ow.forEach(o),hv=n(Y," is optional but setting "),En=l(Y,"CODE",{});var Kw=r(En);uv=n(Kw,"device_map = 'auto'"),Kw.forEach(o),fv=n(Y," is prefered for inference as it will dispatch efficiently the model on the available ressources."),Y.forEach(o),mh=u(e),Oe=l(e,"H3",{class:!0});var Tu=r(Oe);lo=l(Tu,"A",{id:!0,class:!0,href:!0});var e2=r(lo);$n=l(e2,"SPAN",{});var t2=r($n);p(Aa.$$.fragment,t2),t2.forEach(o),e2.forEach(o),cv=u(Tu),kn=l(Tu,"SPAN",{});var o2=r(kn);pv=n(o2,"Advanced use cases"),o2.forEach(o),Tu.forEach(o),bh=u(e),gl=l(e,"P",{});var a2=r(gl);mv=n(a2,"This section is intended to advanced users, that want to explore what it is possible to do beyond loading and running 8-bit models."),a2.forEach(o),vh=u(e),Ke=l(e,"H4",{class:!0});var Cu=r(Ke);ro=l(Cu,"A",{id:!0,class:!0,href:!0});var l2=r(ro);Mn=l(l2,"SPAN",{});var r2=r(Mn);p(Na.$$.fragment,r2),r2.forEach(o),l2.forEach(o),bv=u(Cu),so=l(Cu,"SPAN",{});var zi=r(so);vv=n(zi,"Offload between "),Tn=l(zi,"CODE",{});var s2=r(Tn);yv=n(s2,"cpu"),s2.forEach(o),_v=n(zi," and "),Cn=l(zi,"CODE",{});var n2=r(Cn);gv=n(n2,"gpu"),n2.forEach(o),zi.forEach(o),Cu.forEach(o),yh=u(e),P=l(e,"P",{});var ge=r(P);wv=n(ge,"One of the advanced use case of this is being able to load a model and dispatch the weights between "),qn=l(ge,"CODE",{});var i2=r(qn);Ev=n(i2,"CPU"),i2.forEach(o),$v=n(ge," and "),zn=l(ge,"CODE",{});var d2=r(zn);kv=n(d2,"GPU"),d2.forEach(o),Mv=n(ge,". Note that the weights that will be dispatched on CPU "),Jn=l(ge,"STRONG",{});var h2=r(Jn);Tv=n(h2,"will not"),h2.forEach(o),Cv=n(ge," be converted in 8-bit, thus kept in "),An=l(ge,"CODE",{});var u2=r(An);qv=n(u2,"float32"),u2.forEach(o),zv=n(ge,". This feature is intended for users that want to fit a very large model and dispatch the model between GPU and CPU."),ge.forEach(o),_h=u(e),G=l(e,"P",{});var we=r(G);Jv=n(we,"First, load a "),wl=l(we,"A",{href:!0});var f2=r(wl);Av=n(f2,"BitsAndBytesConfig"),f2.forEach(o),Nv=n(we," from "),Nn=l(we,"CODE",{});var c2=r(Nn);Uv=n(c2,"transformers"),c2.forEach(o),Wv=n(we," and set the attribute "),Un=l(we,"CODE",{});var p2=r(Un);Zv=n(p2,"llm_int8_enable_fp32_cpu_offload"),p2.forEach(o),Pv=n(we," to "),Wn=l(we,"CODE",{});var m2=r(Wn);Gv=n(m2,"True"),m2.forEach(o),jv=n(we,":"),we.forEach(o),gh=u(e),p(Ua.$$.fragment,e),wh=u(e),pe=l(e,"P",{});var Fl=r(pe);Qv=n(Fl,"Let\u2019s say you want to load "),Zn=l(Fl,"CODE",{});var b2=r(Zn);Bv=n(b2,"bigscience/bloom-1b7"),b2.forEach(o),Fv=n(Fl," model, and you have just enough GPU RAM to fit the entire model except the "),Pn=l(Fl,"CODE",{});var v2=r(Pn);xv=n(v2,"lm_head"),v2.forEach(o),Xv=n(Fl,". Therefore write a custom device_map as follows:"),Fl.forEach(o),Eh=u(e),p(Wa.$$.fragment,e),$h=u(e),El=l(e,"P",{});var y2=r(El);Iv=n(y2,"And load your model as follows:"),y2.forEach(o),kh=u(e),p(Za.$$.fragment,e),Mh=u(e),$l=l(e,"P",{});var _2=r($l);Rv=n(_2,"And that\u2019s it! Enjoy your model!"),_2.forEach(o),Th=u(e),et=l(e,"H4",{class:!0});var qu=r(et);no=l(qu,"A",{id:!0,class:!0,href:!0});var g2=r(no);Gn=l(g2,"SPAN",{});var w2=r(Gn);p(Pa.$$.fragment,w2),w2.forEach(o),g2.forEach(o),Vv=u(qu),kl=l(qu,"SPAN",{});var E_=r(kl);Dv=n(E_,"Play with "),jn=l(E_,"CODE",{});var E2=r(jn);Yv=n(E2,"llm_int8_threshold"),E2.forEach(o),E_.forEach(o),qu.forEach(o),Ch=u(e),me=l(e,"P",{});var xl=r(me);Sv=n(xl,"You can play with the "),Qn=l(xl,"CODE",{});var $2=r(Qn);Hv=n($2,"llm_int8_threshold"),$2.forEach(o),Lv=n(xl,` argument to change the threshold of the outliers. An \u201Coutlier\u201D is a hidden state value that is greater than a certain threshold.
This corresponds to the outlier threshold for outlier detection as described in `),Bn=l(xl,"CODE",{});var k2=r(Bn);Ov=n(k2,"LLM.int8()"),k2.forEach(o),Kv=n(xl,` paper. Any hidden states value that is above this threshold will be considered an outlier and the operation on those values will be done in fp16. Values are usually normally distributed, that is, most values are in the range [-3.5, 3.5], but there are some exceptional systematic outliers that are very differently distributed for large models. These outliers are often in the interval [-60, -6] or [6, 60]. Int8 quantization works well for values of magnitude ~5, but beyond that, there is a significant performance penalty. A good default threshold is 6, but a lower threshold might be needed for more unstable models (small models, fine-tuning).
This argument can impact the inference speed of the model. We suggest to play with this parameter to find which one is the best for your use case.`),xl.forEach(o),qh=u(e),p(Ga.$$.fragment,e),zh=u(e),tt=l(e,"H4",{class:!0});var zu=r(tt);io=l(zu,"A",{id:!0,class:!0,href:!0});var M2=r(io);Fn=l(M2,"SPAN",{});var T2=r(Fn);p(ja.$$.fragment,T2),T2.forEach(o),M2.forEach(o),ey=u(zu),xn=l(zu,"SPAN",{});var C2=r(xn);ty=n(C2,"Skip the conversion of some modules"),C2.forEach(o),zu.forEach(o),Jh=u(e),ot=l(e,"P",{});var Ji=r(ot);oy=n(Ji,"Some models has several modules that needs to be not converted in 8-bit to ensure stability. For example Jukebox model has several "),Xn=l(Ji,"CODE",{});var q2=r(Xn);ay=n(q2,"lm_head"),q2.forEach(o),ly=n(Ji," modules that should be skipped. Play with "),In=l(Ji,"CODE",{});var z2=r(In);ry=n(z2,"llm_int8_skip_modules"),z2.forEach(o),Ji.forEach(o),Ah=u(e),p(Qa.$$.fragment,e),Nh=u(e),at=l(e,"H4",{class:!0});var Ju=r(at);ho=l(Ju,"A",{id:!0,class:!0,href:!0});var J2=r(ho);Rn=l(J2,"SPAN",{});var A2=r(Rn);p(Ba.$$.fragment,A2),A2.forEach(o),J2.forEach(o),sy=u(Ju),Vn=l(Ju,"SPAN",{});var N2=r(Vn);ny=n(N2,"Fine-tune a model that has been loaded in 8-bit"),N2.forEach(o),Ju.forEach(o),Uh=u(e),V=l(e,"P",{});var wo=r(V);iy=n(wo,`With the official support of adapters in the Hugging Face ecosystem, you can fine-tune models that have been loaded in 8-bit.
This enables fine-tuning large models such as `),Dn=l(wo,"CODE",{});var U2=r(Dn);dy=n(U2,"flan-t5-large"),U2.forEach(o),hy=n(wo," or "),Yn=l(wo,"CODE",{});var W2=r(Yn);uy=n(W2,"facebook/opt-6.7b"),W2.forEach(o),fy=n(wo," in a single google Colab. Please have a look at "),Fa=l(wo,"A",{href:!0,rel:!0});var Z2=r(Fa);Sn=l(Z2,"CODE",{});var P2=r(Sn);cy=n(P2,"peft"),P2.forEach(o),Z2.forEach(o),py=n(wo," library for more details."),wo.forEach(o),Wh=u(e),z=l(e,"P",{});var S=r(z);my=n(S,"Note that you don\u2019t need to pass "),Hn=l(S,"CODE",{});var G2=r(Hn);by=n(G2,"device_map"),G2.forEach(o),vy=n(S," when loading the model for training. It will automatically load your model on your GPU. You can also set the device map to a specific device if needed (e.g. "),Ln=l(S,"CODE",{});var j2=r(Ln);yy=n(j2,"cuda:0"),j2.forEach(o),_y=n(S,", "),On=l(S,"CODE",{});var Q2=r(On);gy=n(Q2,"0"),Q2.forEach(o),wy=n(S,", "),Kn=l(S,"CODE",{});var B2=r(Kn);Ey=n(B2,"torch.device('cuda:0')"),B2.forEach(o),$y=n(S,"). Please note that "),ei=l(S,"CODE",{});var F2=r(ei);ky=n(F2,"device_map=auto"),F2.forEach(o),My=n(S," should be used for inference only."),S.forEach(o),Zh=u(e),lt=l(e,"H3",{class:!0});var Au=r(lt);uo=l(Au,"A",{id:!0,class:!0,href:!0});var x2=r(uo);ti=l(x2,"SPAN",{});var X2=r(ti);p(xa.$$.fragment,X2),X2.forEach(o),x2.forEach(o),Ty=u(Au),oi=l(Au,"SPAN",{});var I2=r(oi);Cy=n(I2,"BitsAndBytesConfig"),I2.forEach(o),Au.forEach(o),Ph=u(e),M=l(e,"DIV",{class:!0});var J=r(M);p(Xa.$$.fragment,J),qy=u(J),Ia=l(J,"P",{});var Nu=r(Ia);zy=n(Nu,`This is a wrapper class about all possible attributes and features that you can play with a model that has been
loaded using `),ai=l(Nu,"CODE",{});var R2=r(ai);Jy=n(R2,"bitsandbytes"),R2.forEach(o),Ay=n(Nu,"."),Nu.forEach(o),Ny=u(J),rt=l(J,"P",{});var Xl=r(rt);Uy=n(Xl,"This replaces "),li=l(Xl,"CODE",{});var V2=r(li);Wy=n(V2,"load_in_8bit"),V2.forEach(o),Zy=n(Xl," or "),ri=l(Xl,"CODE",{});var D2=r(ri);Py=n(D2,"load_in_4bit"),D2.forEach(o),Gy=n(Xl,"therefore both options are mutually exclusive."),Xl.forEach(o),jy=u(J),B=l(J,"P",{});var Ee=r(B);Qy=n(Ee,"Currently only supports "),si=l(Ee,"CODE",{});var Y2=r(si);By=n(Y2,"LLM.int8()"),Y2.forEach(o),Fy=n(Ee,", "),ni=l(Ee,"CODE",{});var S2=r(ni);xy=n(S2,"FP4"),S2.forEach(o),Xy=n(Ee,", and "),ii=l(Ee,"CODE",{});var H2=r(ii);Iy=n(H2,"NF4"),H2.forEach(o),Ry=n(Ee," quantization. If more methods are added to "),di=l(Ee,"CODE",{});var L2=r(di);Vy=n(L2,"bitsandbytes"),L2.forEach(o),Dy=n(Ee,`,
then more arguments will be added to this class.`),Ee.forEach(o),Yy=u(J),fo=l(J,"DIV",{class:!0});var Uu=r(fo);p(Ra.$$.fragment,Uu),Sy=u(Uu),st=l(Uu,"P",{});var Il=r(st);Hy=n(Il,"Returns "),hi=l(Il,"CODE",{});var O2=r(hi);Ly=n(O2,"True"),O2.forEach(o),Oy=n(Il," if the model is quantizable, "),ui=l(Il,"CODE",{});var K2=r(ui);Ky=n(K2,"False"),K2.forEach(o),e_=n(Il," otherwise."),Il.forEach(o),Uu.forEach(o),t_=u(J),co=l(J,"DIV",{class:!0});var Wu=r(co);p(Va.$$.fragment,Wu),o_=u(Wu),fi=l(Wu,"P",{});var e0=r(fi);a_=n(e0,"Safety checker that arguments are correct - also replaces some NoneType arguments with their default values."),e0.forEach(o),Wu.forEach(o),l_=u(J),po=l(J,"DIV",{class:!0});var Zu=r(po);p(Da.$$.fragment,Zu),r_=u(Zu),Ya=l(Zu,"P",{});var Pu=r(Ya);s_=n(Pu,`This method returns the quantization method used for the model. If the model is not quantizable, it returns
`),ci=l(Pu,"CODE",{});var t0=r(ci);n_=n(t0,"None"),t0.forEach(o),i_=n(Pu,"."),Pu.forEach(o),Zu.forEach(o),d_=u(J),mo=l(J,"DIV",{class:!0});var Gu=r(mo);p(Sa.$$.fragment,Gu),h_=u(Gu),pi=l(Gu,"P",{});var o0=r(pi);u_=n(o0,`Removes all attributes from config which correspond to the default config attributes for better readability and
serializes to a Python dictionary.`),o0.forEach(o),Gu.forEach(o),J.forEach(o),Gh=u(e),nt=l(e,"H2",{class:!0});var ju=r(nt);bo=l(ju,"A",{id:!0,class:!0,href:!0});var a0=r(bo);mi=l(a0,"SPAN",{});var l0=r(mi);p(Ha.$$.fragment,l0),l0.forEach(o),a0.forEach(o),f_=u(ju),Ml=l(ju,"SPAN",{});var $_=r(Ml);c_=n($_,"Quantization with \u{1F917} "),bi=l($_,"CODE",{});var r0=r(bi);p_=n(r0,"optimum"),r0.forEach(o),$_.forEach(o),ju.forEach(o),jh=u(e),be=l(e,"P",{});var Rl=r(be);m_=n(Rl,"Please have a look at "),La=l(Rl,"A",{href:!0,rel:!0});var s0=r(La);b_=n(s0,"Optimum documentation"),s0.forEach(o),v_=n(Rl," to learn more about quantization methods that are supported by "),vi=l(Rl,"CODE",{});var n0=r(vi);y_=n(n0,"optimum"),n0.forEach(o),__=n(Rl," and see if these are applicable for your use case."),Rl.forEach(o),this.h()},h(){f(_,"name","hf:doc:metadata"),f(_,"content",JSON.stringify(y0)),f(k,"id","quantize-transformers-models"),f(k,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(k,"href","#quantize-transformers-models"),f(w,"class","relative group"),f(A,"id","autogptq-integration"),f(A,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(A,"href","#autogptq-integration"),f(Q,"class","relative group"),f(Mo,"href","https://arxiv.org/pdf/2210.17323.pdf"),f(Mo,"rel","nofollow"),f(To,"href","https://huggingface.co/docs/optimum/llm_quantization/usage_guides/quantization"),f(To,"rel","nofollow"),f(qo,"href","https://github.com/PanQiWei/AutoGPTQ"),f(qo,"rel","nofollow"),f(ft,"id","requirements"),f(ft,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ft,"href","#requirements"),f(ze,"class","relative group"),f(vt,"id","load-and-quantize-a-model"),f(vt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(vt,"href","#load-and-quantize-a-model"),f(Je,"class","relative group"),f(_t,"id","gptq-configuration"),f(_t,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(_t,"href","#gptq-configuration"),f(Ae,"class","relative group"),f(nl,"href","/docs/transformers/v4.32.0/en/main_classes/quantization#transformers.GPTQConfig"),f(gt,"id","quantization"),f(gt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(gt,"href","#quantization"),f(Ne,"class","relative group"),f(Go,"href","https://huggingface.co/docs/accelerate/usage_guides/big_modeling#designing-a-device-map"),f(Go,"rel","nofollow"),f($t,"id","push-quantized-model-to-hub"),f($t,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f($t,"href","#push-quantized-model-to-hub"),f(Ue,"class","relative group"),f(Tt,"id","load-a-quantized-model-from-the-hub"),f(Tt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Tt,"href","#load-a-quantized-model-from-the-hub"),f(We,"class","relative group"),f(Ct,"id","exllama-kernels-for-faster-inference"),f(Ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ct,"href","#exllama-kernels-for-faster-inference"),f(Ze,"class","relative group"),f(hl,"href","/docs/transformers/v4.32.0/en/main_classes/quantization#transformers.GPTQConfig"),f(qt,"id","finetune-a-quantized-model"),f(qt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(qt,"href","#finetune-a-quantized-model"),f(Pe,"class","relative group"),f(Yo,"href","https://github.com/huggingface/peft"),f(Yo,"rel","nofollow"),f(Jt,"id","example-demo"),f(Jt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Jt,"href","#example-demo"),f(Ge,"class","relative group"),f(Ho,"href","https://colab.research.google.com/drive/1_TIrmuKOFhuRRiTWN94iLKUFu6ZX4ceb?usp=sharing"),f(Ho,"rel","nofollow"),f(Nt,"id","transformers.GPTQConfig"),f(Nt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Nt,"href","#transformers.GPTQConfig"),f(je,"class","relative group"),f(Ut,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(Wt,"id","bitsandbytes-integration"),f(Wt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Wt,"href","#bitsandbytes-integration"),f(Qe,"class","relative group"),f(oa,"href","https://arxiv.org/abs/2208.07339"),f(oa,"rel","nofollow"),f(aa,"href","https://huggingface.co/blog/hf-bitsandbytes-integration"),f(aa,"rel","nofollow"),f(la,"href","https://huggingface.co/docs/accelerate/main/en/usage_guides/quantization"),f(la,"rel","nofollow"),f(Gt,"id","general-usage"),f(Gt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Gt,"href","#general-usage"),f(Be,"class","relative group"),f(cl,"href","/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained"),f(jt,"id","fp4-quantization"),f(jt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(jt,"href","#fp4-quantization"),f(Fe,"class","relative group"),f(Qt,"id","requirements"),f(Qt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Qt,"href","#requirements"),f(xe,"class","relative group"),f(Xt,"id","tips-and-best-practices"),f(Xt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Xt,"href","#tips-and-best-practices"),f(Xe,"class","relative group"),f(ua,"href","https://colab.research.google.com/drive/1ge2F1QSK8Q7h0hn3YKuBCOAS0bK8E0wf"),f(ua,"rel","nofollow"),f(ca,"href","https://github.com/TimDettmers/bitsandbytes/releases/tag/0.40.0"),f(ca,"rel","nofollow"),f(pa,"href","https://arxiv.org/abs/2305.14314"),f(pa,"rel","nofollow"),f(Rt,"id","load-a-large-model-in-4bit"),f(Rt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Rt,"href","#load-a-large-model-in-4bit"),f(Ie,"class","relative group"),f(Dt,"id","load-a-large-model-in-8bit"),f(Dt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Dt,"href","#load-a-large-model-in-8bit"),f(Re,"class","relative group"),f(ml,"href","/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel"),f(Lt,"id","advanced-use-cases"),f(Lt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Lt,"href","#advanced-use-cases"),f(Ve,"class","relative group"),f(Ot,"id","change-the-compute-dtype"),f(Ot,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Ot,"href","#change-the-compute-dtype"),f(De,"class","relative group"),f(Kt,"id","using-nf4-normal-float-4-data-type"),f(Kt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(Kt,"href","#using-nf4-normal-float-4-data-type"),f(Ye,"class","relative group"),f(eo,"id","use-nested-quantization-for-more-memory-efficient-inference"),f(eo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(eo,"href","#use-nested-quantization-for-more-memory-efficient-inference"),f(Se,"class","relative group"),f(to,"id","push-quantized-models-on-the-hub"),f(to,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(to,"href","#push-quantized-models-on-the-hub"),f(He,"class","relative group"),f(ao,"id","load-a-quantized-model-from-the-hub"),f(ao,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ao,"href","#load-a-quantized-model-from-the-hub"),f(Le,"class","relative group"),f(lo,"id","advanced-use-cases"),f(lo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(lo,"href","#advanced-use-cases"),f(Oe,"class","relative group"),f(ro,"id","offload-between-cpu-and-gpu"),f(ro,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ro,"href","#offload-between-cpu-and-gpu"),f(Ke,"class","relative group"),f(wl,"href","/docs/transformers/v4.32.0/en/main_classes/quantization#transformers.BitsAndBytesConfig"),f(no,"id","play-with-llmint8threshold"),f(no,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(no,"href","#play-with-llmint8threshold"),f(et,"class","relative group"),f(io,"id","skip-the-conversion-of-some-modules"),f(io,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(io,"href","#skip-the-conversion-of-some-modules"),f(tt,"class","relative group"),f(ho,"id","finetune-a-model-that-has-been-loaded-in-8bit"),f(ho,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ho,"href","#finetune-a-model-that-has-been-loaded-in-8bit"),f(at,"class","relative group"),f(Fa,"href","https://github.com/huggingface/peft"),f(Fa,"rel","nofollow"),f(uo,"id","transformers.BitsAndBytesConfig"),f(uo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(uo,"href","#transformers.BitsAndBytesConfig"),f(lt,"class","relative group"),f(fo,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(co,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(po,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(mo,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(M,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),f(bo,"id","quantization-with-optimum"),f(bo,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(bo,"href","#quantization-with-optimum"),f(nt,"class","relative group"),f(La,"href","https://huggingface.co/docs/optimum/index"),f(La,"rel","nofollow")},m(e,i){t(document.head,_),d(e,$,i),d(e,w,i),t(w,k),t(k,$e),m(j,$e,null),t(w,Eo),t(w,ke),t(ke,Me),d(e,dt,i),d(e,Q,i),t(Q,A),t(A,Te),m(H,Te,null),t(Q,Ce),t(Q,T),t(T,ht),t(ht,tl),t(T,ol),d(e,$o,i),d(e,ut,i),t(ut,Bu),t(ut,Vl),t(Vl,Fu),t(ut,xu),d(e,Ai,i),d(e,al,i),t(al,Xu),d(e,Ni,i),d(e,ee,i),t(ee,ko),t(ko,Iu),t(ko,Mo),t(Mo,Ru),t(ko,Vu),t(ee,Du),t(ee,qe),t(qe,Yu),t(qe,Dl),t(Dl,Su),t(qe,Hu),t(qe,To),t(To,Lu),t(qe,Ou),t(ee,Ku),t(ee,Co),t(Co,ef),t(Co,qo),t(qo,Yl),t(Yl,tf),t(Co,of),d(e,Ui,i),d(e,ze,i),t(ze,ft),t(ft,Sl),m(zo,Sl,null),t(ze,af),t(ze,Hl),t(Hl,lf),d(e,Wi,i),d(e,ll,i),t(ll,rf),d(e,Zi,i),d(e,F,i),t(F,Ll),t(Ll,ct),t(ct,sf),t(ct,Ol),t(Ol,nf),t(ct,df),t(ct,Kl),t(Kl,hf),t(F,uf),t(F,er),t(er,pt),t(pt,ff),t(pt,tr),t(tr,cf),t(pt,pf),t(pt,or),t(or,mf),t(F,bf),t(F,ar),t(ar,mt),t(mt,vf),t(mt,lr),t(lr,yf),t(mt,_f),t(mt,rr),t(rr,gf),t(F,wf),t(F,sr),t(sr,bt),t(bt,Ef),t(bt,nr),t(nr,$f),t(bt,kf),t(bt,ir),t(ir,Mf),d(e,Pi,i),d(e,rl,i),t(rl,Tf),d(e,Gi,i),d(e,Je,i),t(Je,vt),t(vt,dr),m(Jo,dr,null),t(Je,Cf),t(Je,hr),t(hr,qf),d(e,ji,i),d(e,yt,i),t(yt,zf),t(yt,ur),t(ur,Jf),t(yt,Af),d(e,Qi,i),d(e,sl,i),t(sl,Nf),d(e,Bi,i),d(e,Ae,i),t(Ae,_t),t(_t,fr),m(Ao,fr,null),t(Ae,Uf),t(Ae,cr),t(cr,Wf),d(e,Fi,i),d(e,N,i),t(N,Zf),t(N,nl),t(nl,Pf),t(N,Gf),t(N,pr),t(pr,jf),t(N,Qf),t(N,mr),t(mr,Bf),t(N,Ff),t(N,br),t(br,xf),t(N,Xf),d(e,xi,i),m(No,e,i),d(e,Xi,i),d(e,il,i),t(il,If),d(e,Ii,i),m(Uo,e,i),d(e,Ri,i),d(e,Ne,i),t(Ne,gt),t(gt,vr),m(Wo,vr,null),t(Ne,Rf),t(Ne,yr),t(yr,Vf),d(e,Vi,i),d(e,te,i),t(te,Df),t(te,_r),t(_r,Yf),t(te,Sf),t(te,gr),t(gr,Hf),t(te,Lf),d(e,Di,i),m(Zo,e,i),d(e,Yi,i),d(e,dl,i),t(dl,Of),d(e,Si,i),d(e,wt,i),t(wt,Kf),t(wt,wr),t(wr,ec),t(wt,tc),d(e,Hi,i),m(Po,e,i),d(e,Li,i),d(e,C,i),t(C,oc),t(C,Er),t(Er,ac),t(C,lc),t(C,$r),t($r,rc),t(C,sc),t(C,Go),t(Go,nc),t(C,ic),t(C,kr),t(kr,dc),t(C,hc),t(C,Mr),t(Mr,uc),t(C,fc),d(e,Oi,i),m(Et,e,i),d(e,Ki,i),d(e,Ue,i),t(Ue,$t),t($t,Tr),m(jo,Tr,null),t(Ue,cc),t(Ue,Cr),t(Cr,pc),d(e,ed,i),d(e,kt,i),t(kt,mc),t(kt,qr),t(qr,bc),t(kt,vc),d(e,td,i),m(Qo,e,i),d(e,od,i),d(e,Mt,i),t(Mt,yc),t(Mt,zr),t(zr,_c),t(Mt,gc),d(e,ad,i),m(Bo,e,i),d(e,ld,i),d(e,oe,i),t(oe,wc),t(oe,Jr),t(Jr,Ec),t(oe,$c),t(oe,Ar),t(Ar,kc),t(oe,Mc),d(e,rd,i),m(Fo,e,i),d(e,sd,i),d(e,We,i),t(We,Tt),t(Tt,Nr),m(xo,Nr,null),t(We,Tc),t(We,Ur),t(Ur,Cc),d(e,nd,i),d(e,ae,i),t(ae,qc),t(ae,Wr),t(Wr,zc),t(ae,Jc),t(ae,Zr),t(Zr,Ac),t(ae,Nc),d(e,id,i),m(Xo,e,i),d(e,dd,i),d(e,le,i),t(le,Uc),t(le,Pr),t(Pr,Wc),t(le,Zc),t(le,Gr),t(Gr,Pc),t(le,Gc),d(e,hd,i),m(Io,e,i),d(e,ud,i),d(e,Ze,i),t(Ze,Ct),t(Ct,jr),m(Ro,jr,null),t(Ze,jc),t(Ze,Qr),t(Qr,Qc),d(e,fd,i),d(e,re,i),t(re,Bc),t(re,Br),t(Br,Fc),t(re,xc),t(re,hl),t(hl,Xc),t(re,Ic),d(e,cd,i),m(Vo,e,i),d(e,pd,i),d(e,ul,i),t(ul,Rc),d(e,md,i),d(e,Pe,i),t(Pe,qt),t(qt,Fr),m(Do,Fr,null),t(Pe,Vc),t(Pe,xr),t(xr,Dc),d(e,bd,i),d(e,zt,i),t(zt,Yc),t(zt,Yo),t(Yo,Xr),t(Xr,Sc),t(zt,Hc),d(e,vd,i),d(e,Ge,i),t(Ge,Jt),t(Jt,Ir),m(So,Ir,null),t(Ge,Lc),t(Ge,Rr),t(Rr,Oc),d(e,yd,i),d(e,At,i),t(At,Kc),t(At,Ho),t(Ho,ep),t(At,tp),d(e,_d,i),d(e,je,i),t(je,Nt),t(Nt,Vr),m(Lo,Vr,null),t(je,op),t(je,Dr),t(Dr,ap),d(e,gd,i),d(e,L,i),m(Oo,L,null),t(L,lp),t(L,Ko),t(Ko,rp),t(Ko,Yr),t(Yr,sp),t(Ko,np),t(L,ip),t(L,Ut),m(ea,Ut,null),t(Ut,dp),t(Ut,Sr),t(Sr,hp),d(e,wd,i),d(e,Qe,i),t(Qe,Wt),t(Wt,Hr),m(ta,Hr,null),t(Qe,up),t(Qe,fl),t(fl,Lr),t(Lr,fp),t(fl,cp),d(e,Ed,i),d(e,x,i),t(x,pp),t(x,Or),t(Or,mp),t(x,bp),t(x,Kr),t(Kr,vp),t(x,yp),t(x,es),t(es,_p),t(x,gp),d(e,$d,i),d(e,se,i),t(se,wp),t(se,oa),t(oa,Ep),t(se,$p),t(se,aa),t(aa,kp),t(se,Mp),d(e,kd,i),d(e,ne,i),t(ne,Tp),t(ne,ts),t(ts,Cp),t(ne,qp),t(ne,os),t(os,zp),t(ne,Jp),d(e,Md,i),d(e,Zt,i),t(Zt,Ap),t(Zt,la),t(la,Np),t(Zt,Up),d(e,Td,i),d(e,Pt,i),t(Pt,Wp),t(Pt,as),t(as,Zp),t(Pt,Pp),d(e,Cd,i),d(e,Be,i),t(Be,Gt),t(Gt,ls),m(ra,ls,null),t(Be,Gp),t(Be,rs),t(rs,jp),d(e,qd,i),d(e,U,i),t(U,Qp),t(U,ss),t(ss,Bp),t(U,Fp),t(U,ns),t(ns,xp),t(U,Xp),t(U,cl),t(cl,Ip),t(U,Rp),t(U,is),t(is,Vp),t(U,Dp),d(e,zd,i),m(sa,e,i),d(e,Jd,i),d(e,W,i),t(W,Yp),t(W,ds),t(ds,Sp),t(W,Hp),t(W,hs),t(hs,Lp),t(W,Op),t(W,us),t(us,Kp),t(W,em),t(W,fs),t(fs,tm),t(W,om),d(e,Ad,i),m(na,e,i),d(e,Nd,i),d(e,Fe,i),t(Fe,jt),t(jt,cs),m(ia,cs,null),t(Fe,am),t(Fe,ps),t(ps,lm),d(e,Ud,i),d(e,xe,i),t(xe,Qt),t(Qt,ms),m(da,ms,null),t(xe,rm),t(xe,bs),t(bs,sm),d(e,Wd,i),d(e,pl,i),t(pl,nm),d(e,Zd,i),d(e,ie,i),t(ie,vs),t(vs,Bt),t(Bt,im),t(Bt,ys),t(ys,dm),t(Bt,hm),t(Bt,_s),t(_s,um),t(ie,fm),t(ie,gs),t(gs,Ft),t(Ft,cm),t(Ft,ws),t(ws,pm),t(Ft,mm),t(Ft,Es),t(Es,bm),t(ie,vm),t(ie,$s),t($s,xt),t(xt,ym),t(xt,ks),t(ks,_m),t(xt,gm),t(xt,Ms),t(Ms,wm),d(e,Pd,i),d(e,Xe,i),t(Xe,Xt),t(Xt,Ts),m(ha,Ts,null),t(Xe,Em),t(Xe,Cs),t(Cs,$m),d(e,Gd,i),d(e,X,i),t(X,qs),t(qs,It),t(It,zs),t(zs,km),t(It,Mm),t(It,ua),t(ua,Tm),t(It,Cm),t(X,qm),t(X,Js),t(Js,Z),t(Z,fa),t(fa,zm),t(fa,As),t(As,Jm),t(fa,Am),t(Z,Nm),t(Z,Ns),t(Ns,Um),t(Z,Wm),t(Z,Us),t(Us,Zm),t(Z,Pm),t(Z,ca),t(ca,Gm),t(Z,jm),t(Z,Ws),t(Ws,Qm),t(Z,Bm),t(X,Fm),t(X,Zs),t(Zs,de),t(de,Ps),t(Ps,xm),t(de,Xm),t(de,pa),t(pa,Im),t(de,Rm),t(de,Gs),t(Gs,Vm),t(de,Dm),t(X,Ym),t(X,js),t(js,I),t(I,Qs),t(Qs,Sm),t(I,Hm),t(I,Bs),t(Bs,Lm),t(I,Om),t(I,Fs),t(Fs,Km),t(I,eb),t(I,xs),t(xs,tb),t(I,ob),d(e,jd,i),d(e,Ie,i),t(Ie,Rt),t(Rt,Xs),m(ma,Xs,null),t(Ie,ab),t(Ie,Is),t(Is,lb),d(e,Qd,i),d(e,he,i),t(he,rb),t(he,Rs),t(Rs,sb),t(he,nb),t(he,Vs),t(Vs,ib),t(he,db),d(e,Bd,i),m(ba,e,i),d(e,Fd,i),m(Vt,e,i),d(e,xd,i),d(e,Re,i),t(Re,Dt),t(Dt,Ds),m(va,Ds,null),t(Re,hb),t(Re,Ys),t(Ys,ub),d(e,Xd,i),d(e,ue,i),t(ue,fb),t(ue,Ss),t(Ss,cb),t(ue,pb),t(ue,Hs),t(Hs,mb),t(ue,bb),d(e,Id,i),m(ya,e,i),d(e,Rd,i),d(e,Yt,i),t(Yt,vb),t(Yt,ml),t(ml,yb),t(Yt,_b),d(e,Vd,i),d(e,St,i),t(St,gb),t(St,Ls),t(Ls,wb),t(St,Eb),d(e,Dd,i),m(_a,e,i),d(e,Yd,i),d(e,bl,i),t(bl,$b),d(e,Sd,i),m(Ht,e,i),d(e,Hd,i),d(e,Ve,i),t(Ve,Lt),t(Lt,Os),m(ga,Os,null),t(Ve,kb),t(Ve,Ks),t(Ks,Mb),d(e,Ld,i),d(e,vl,i),t(vl,Tb),d(e,Od,i),d(e,De,i),t(De,Ot),t(Ot,en),m(wa,en,null),t(De,Cb),t(De,tn),t(tn,qb),d(e,Kd,i),d(e,fe,i),t(fe,zb),t(fe,on),t(on,Jb),t(fe,Ab),t(fe,an),t(an,Nb),t(fe,Ub),d(e,eh,i),m(Ea,e,i),d(e,th,i),d(e,Ye,i),t(Ye,Kt),t(Kt,ln),m($a,ln,null),t(Ye,Wb),t(Ye,rn),t(rn,Zb),d(e,oh,i),d(e,yl,i),t(yl,Pb),d(e,ah,i),m(ka,e,i),d(e,lh,i),d(e,Se,i),t(Se,eo),t(eo,sn),m(Ma,sn,null),t(Se,Gb),t(Se,nn),t(nn,jb),d(e,rh,i),d(e,_l,i),t(_l,Qb),d(e,sh,i),m(Ta,e,i),d(e,nh,i),d(e,He,i),t(He,to),t(to,dn),m(Ca,dn,null),t(He,Bb),t(He,hn),t(hn,Fb),d(e,ih,i),d(e,R,i),t(R,xb),t(R,un),t(un,Xb),t(R,Ib),t(R,fn),t(fn,Rb),t(R,Vb),t(R,cn),t(cn,Db),t(R,Yb),d(e,dh,i),m(qa,e,i),d(e,hh,i),m(oo,e,i),d(e,uh,i),d(e,Le,i),t(Le,ao),t(ao,pn),m(za,pn,null),t(Le,Sb),t(Le,mn),t(mn,Hb),d(e,fh,i),d(e,ce,i),t(ce,Lb),t(ce,bn),t(bn,Ob),t(ce,Kb),t(ce,vn),t(vn,ev),t(ce,tv),d(e,ch,i),m(Ja,e,i),d(e,ph,i),d(e,q,i),t(q,ov),t(q,yn),t(yn,av),t(q,lv),t(q,_n),t(_n,rv),t(q,sv),t(q,gn),t(gn,nv),t(q,iv),t(q,wn),t(wn,dv),t(q,hv),t(q,En),t(En,uv),t(q,fv),d(e,mh,i),d(e,Oe,i),t(Oe,lo),t(lo,$n),m(Aa,$n,null),t(Oe,cv),t(Oe,kn),t(kn,pv),d(e,bh,i),d(e,gl,i),t(gl,mv),d(e,vh,i),d(e,Ke,i),t(Ke,ro),t(ro,Mn),m(Na,Mn,null),t(Ke,bv),t(Ke,so),t(so,vv),t(so,Tn),t(Tn,yv),t(so,_v),t(so,Cn),t(Cn,gv),d(e,yh,i),d(e,P,i),t(P,wv),t(P,qn),t(qn,Ev),t(P,$v),t(P,zn),t(zn,kv),t(P,Mv),t(P,Jn),t(Jn,Tv),t(P,Cv),t(P,An),t(An,qv),t(P,zv),d(e,_h,i),d(e,G,i),t(G,Jv),t(G,wl),t(wl,Av),t(G,Nv),t(G,Nn),t(Nn,Uv),t(G,Wv),t(G,Un),t(Un,Zv),t(G,Pv),t(G,Wn),t(Wn,Gv),t(G,jv),d(e,gh,i),m(Ua,e,i),d(e,wh,i),d(e,pe,i),t(pe,Qv),t(pe,Zn),t(Zn,Bv),t(pe,Fv),t(pe,Pn),t(Pn,xv),t(pe,Xv),d(e,Eh,i),m(Wa,e,i),d(e,$h,i),d(e,El,i),t(El,Iv),d(e,kh,i),m(Za,e,i),d(e,Mh,i),d(e,$l,i),t($l,Rv),d(e,Th,i),d(e,et,i),t(et,no),t(no,Gn),m(Pa,Gn,null),t(et,Vv),t(et,kl),t(kl,Dv),t(kl,jn),t(jn,Yv),d(e,Ch,i),d(e,me,i),t(me,Sv),t(me,Qn),t(Qn,Hv),t(me,Lv),t(me,Bn),t(Bn,Ov),t(me,Kv),d(e,qh,i),m(Ga,e,i),d(e,zh,i),d(e,tt,i),t(tt,io),t(io,Fn),m(ja,Fn,null),t(tt,ey),t(tt,xn),t(xn,ty),d(e,Jh,i),d(e,ot,i),t(ot,oy),t(ot,Xn),t(Xn,ay),t(ot,ly),t(ot,In),t(In,ry),d(e,Ah,i),m(Qa,e,i),d(e,Nh,i),d(e,at,i),t(at,ho),t(ho,Rn),m(Ba,Rn,null),t(at,sy),t(at,Vn),t(Vn,ny),d(e,Uh,i),d(e,V,i),t(V,iy),t(V,Dn),t(Dn,dy),t(V,hy),t(V,Yn),t(Yn,uy),t(V,fy),t(V,Fa),t(Fa,Sn),t(Sn,cy),t(V,py),d(e,Wh,i),d(e,z,i),t(z,my),t(z,Hn),t(Hn,by),t(z,vy),t(z,Ln),t(Ln,yy),t(z,_y),t(z,On),t(On,gy),t(z,wy),t(z,Kn),t(Kn,Ey),t(z,$y),t(z,ei),t(ei,ky),t(z,My),d(e,Zh,i),d(e,lt,i),t(lt,uo),t(uo,ti),m(xa,ti,null),t(lt,Ty),t(lt,oi),t(oi,Cy),d(e,Ph,i),d(e,M,i),m(Xa,M,null),t(M,qy),t(M,Ia),t(Ia,zy),t(Ia,ai),t(ai,Jy),t(Ia,Ay),t(M,Ny),t(M,rt),t(rt,Uy),t(rt,li),t(li,Wy),t(rt,Zy),t(rt,ri),t(ri,Py),t(rt,Gy),t(M,jy),t(M,B),t(B,Qy),t(B,si),t(si,By),t(B,Fy),t(B,ni),t(ni,xy),t(B,Xy),t(B,ii),t(ii,Iy),t(B,Ry),t(B,di),t(di,Vy),t(B,Dy),t(M,Yy),t(M,fo),m(Ra,fo,null),t(fo,Sy),t(fo,st),t(st,Hy),t(st,hi),t(hi,Ly),t(st,Oy),t(st,ui),t(ui,Ky),t(st,e_),t(M,t_),t(M,co),m(Va,co,null),t(co,o_),t(co,fi),t(fi,a_),t(M,l_),t(M,po),m(Da,po,null),t(po,r_),t(po,Ya),t(Ya,s_),t(Ya,ci),t(ci,n_),t(Ya,i_),t(M,d_),t(M,mo),m(Sa,mo,null),t(mo,h_),t(mo,pi),t(pi,u_),d(e,Gh,i),d(e,nt,i),t(nt,bo),t(bo,mi),m(Ha,mi,null),t(nt,f_),t(nt,Ml),t(Ml,c_),t(Ml,bi),t(bi,p_),d(e,jh,i),d(e,be,i),t(be,m_),t(be,La),t(La,b_),t(be,v_),t(be,vi),t(vi,y_),t(be,__),Qh=!0},p(e,[i]){const Oa={};i&2&&(Oa.$$scope={dirty:i,ctx:e}),Et.$set(Oa);const yi={};i&2&&(yi.$$scope={dirty:i,ctx:e}),Vt.$set(yi);const _i={};i&2&&(_i.$$scope={dirty:i,ctx:e}),Ht.$set(_i);const gi={};i&2&&(gi.$$scope={dirty:i,ctx:e}),oo.$set(gi)},i(e){Qh||(b(j.$$.fragment,e),b(H.$$.fragment,e),b(zo.$$.fragment,e),b(Jo.$$.fragment,e),b(Ao.$$.fragment,e),b(No.$$.fragment,e),b(Uo.$$.fragment,e),b(Wo.$$.fragment,e),b(Zo.$$.fragment,e),b(Po.$$.fragment,e),b(Et.$$.fragment,e),b(jo.$$.fragment,e),b(Qo.$$.fragment,e),b(Bo.$$.fragment,e),b(Fo.$$.fragment,e),b(xo.$$.fragment,e),b(Xo.$$.fragment,e),b(Io.$$.fragment,e),b(Ro.$$.fragment,e),b(Vo.$$.fragment,e),b(Do.$$.fragment,e),b(So.$$.fragment,e),b(Lo.$$.fragment,e),b(Oo.$$.fragment,e),b(ea.$$.fragment,e),b(ta.$$.fragment,e),b(ra.$$.fragment,e),b(sa.$$.fragment,e),b(na.$$.fragment,e),b(ia.$$.fragment,e),b(da.$$.fragment,e),b(ha.$$.fragment,e),b(ma.$$.fragment,e),b(ba.$$.fragment,e),b(Vt.$$.fragment,e),b(va.$$.fragment,e),b(ya.$$.fragment,e),b(_a.$$.fragment,e),b(Ht.$$.fragment,e),b(ga.$$.fragment,e),b(wa.$$.fragment,e),b(Ea.$$.fragment,e),b($a.$$.fragment,e),b(ka.$$.fragment,e),b(Ma.$$.fragment,e),b(Ta.$$.fragment,e),b(Ca.$$.fragment,e),b(qa.$$.fragment,e),b(oo.$$.fragment,e),b(za.$$.fragment,e),b(Ja.$$.fragment,e),b(Aa.$$.fragment,e),b(Na.$$.fragment,e),b(Ua.$$.fragment,e),b(Wa.$$.fragment,e),b(Za.$$.fragment,e),b(Pa.$$.fragment,e),b(Ga.$$.fragment,e),b(ja.$$.fragment,e),b(Qa.$$.fragment,e),b(Ba.$$.fragment,e),b(xa.$$.fragment,e),b(Xa.$$.fragment,e),b(Ra.$$.fragment,e),b(Va.$$.fragment,e),b(Da.$$.fragment,e),b(Sa.$$.fragment,e),b(Ha.$$.fragment,e),Qh=!0)},o(e){v(j.$$.fragment,e),v(H.$$.fragment,e),v(zo.$$.fragment,e),v(Jo.$$.fragment,e),v(Ao.$$.fragment,e),v(No.$$.fragment,e),v(Uo.$$.fragment,e),v(Wo.$$.fragment,e),v(Zo.$$.fragment,e),v(Po.$$.fragment,e),v(Et.$$.fragment,e),v(jo.$$.fragment,e),v(Qo.$$.fragment,e),v(Bo.$$.fragment,e),v(Fo.$$.fragment,e),v(xo.$$.fragment,e),v(Xo.$$.fragment,e),v(Io.$$.fragment,e),v(Ro.$$.fragment,e),v(Vo.$$.fragment,e),v(Do.$$.fragment,e),v(So.$$.fragment,e),v(Lo.$$.fragment,e),v(Oo.$$.fragment,e),v(ea.$$.fragment,e),v(ta.$$.fragment,e),v(ra.$$.fragment,e),v(sa.$$.fragment,e),v(na.$$.fragment,e),v(ia.$$.fragment,e),v(da.$$.fragment,e),v(ha.$$.fragment,e),v(ma.$$.fragment,e),v(ba.$$.fragment,e),v(Vt.$$.fragment,e),v(va.$$.fragment,e),v(ya.$$.fragment,e),v(_a.$$.fragment,e),v(Ht.$$.fragment,e),v(ga.$$.fragment,e),v(wa.$$.fragment,e),v(Ea.$$.fragment,e),v($a.$$.fragment,e),v(ka.$$.fragment,e),v(Ma.$$.fragment,e),v(Ta.$$.fragment,e),v(Ca.$$.fragment,e),v(qa.$$.fragment,e),v(oo.$$.fragment,e),v(za.$$.fragment,e),v(Ja.$$.fragment,e),v(Aa.$$.fragment,e),v(Na.$$.fragment,e),v(Ua.$$.fragment,e),v(Wa.$$.fragment,e),v(Za.$$.fragment,e),v(Pa.$$.fragment,e),v(Ga.$$.fragment,e),v(ja.$$.fragment,e),v(Qa.$$.fragment,e),v(Ba.$$.fragment,e),v(xa.$$.fragment,e),v(Xa.$$.fragment,e),v(Ra.$$.fragment,e),v(Va.$$.fragment,e),v(Da.$$.fragment,e),v(Sa.$$.fragment,e),v(Ha.$$.fragment,e),Qh=!1},d(e){o(_),e&&o($),e&&o(w),y(j),e&&o(dt),e&&o(Q),y(H),e&&o($o),e&&o(ut),e&&o(Ai),e&&o(al),e&&o(Ni),e&&o(ee),e&&o(Ui),e&&o(ze),y(zo),e&&o(Wi),e&&o(ll),e&&o(Zi),e&&o(F),e&&o(Pi),e&&o(rl),e&&o(Gi),e&&o(Je),y(Jo),e&&o(ji),e&&o(yt),e&&o(Qi),e&&o(sl),e&&o(Bi),e&&o(Ae),y(Ao),e&&o(Fi),e&&o(N),e&&o(xi),y(No,e),e&&o(Xi),e&&o(il),e&&o(Ii),y(Uo,e),e&&o(Ri),e&&o(Ne),y(Wo),e&&o(Vi),e&&o(te),e&&o(Di),y(Zo,e),e&&o(Yi),e&&o(dl),e&&o(Si),e&&o(wt),e&&o(Hi),y(Po,e),e&&o(Li),e&&o(C),e&&o(Oi),y(Et,e),e&&o(Ki),e&&o(Ue),y(jo),e&&o(ed),e&&o(kt),e&&o(td),y(Qo,e),e&&o(od),e&&o(Mt),e&&o(ad),y(Bo,e),e&&o(ld),e&&o(oe),e&&o(rd),y(Fo,e),e&&o(sd),e&&o(We),y(xo),e&&o(nd),e&&o(ae),e&&o(id),y(Xo,e),e&&o(dd),e&&o(le),e&&o(hd),y(Io,e),e&&o(ud),e&&o(Ze),y(Ro),e&&o(fd),e&&o(re),e&&o(cd),y(Vo,e),e&&o(pd),e&&o(ul),e&&o(md),e&&o(Pe),y(Do),e&&o(bd),e&&o(zt),e&&o(vd),e&&o(Ge),y(So),e&&o(yd),e&&o(At),e&&o(_d),e&&o(je),y(Lo),e&&o(gd),e&&o(L),y(Oo),y(ea),e&&o(wd),e&&o(Qe),y(ta),e&&o(Ed),e&&o(x),e&&o($d),e&&o(se),e&&o(kd),e&&o(ne),e&&o(Md),e&&o(Zt),e&&o(Td),e&&o(Pt),e&&o(Cd),e&&o(Be),y(ra),e&&o(qd),e&&o(U),e&&o(zd),y(sa,e),e&&o(Jd),e&&o(W),e&&o(Ad),y(na,e),e&&o(Nd),e&&o(Fe),y(ia),e&&o(Ud),e&&o(xe),y(da),e&&o(Wd),e&&o(pl),e&&o(Zd),e&&o(ie),e&&o(Pd),e&&o(Xe),y(ha),e&&o(Gd),e&&o(X),e&&o(jd),e&&o(Ie),y(ma),e&&o(Qd),e&&o(he),e&&o(Bd),y(ba,e),e&&o(Fd),y(Vt,e),e&&o(xd),e&&o(Re),y(va),e&&o(Xd),e&&o(ue),e&&o(Id),y(ya,e),e&&o(Rd),e&&o(Yt),e&&o(Vd),e&&o(St),e&&o(Dd),y(_a,e),e&&o(Yd),e&&o(bl),e&&o(Sd),y(Ht,e),e&&o(Hd),e&&o(Ve),y(ga),e&&o(Ld),e&&o(vl),e&&o(Od),e&&o(De),y(wa),e&&o(Kd),e&&o(fe),e&&o(eh),y(Ea,e),e&&o(th),e&&o(Ye),y($a),e&&o(oh),e&&o(yl),e&&o(ah),y(ka,e),e&&o(lh),e&&o(Se),y(Ma),e&&o(rh),e&&o(_l),e&&o(sh),y(Ta,e),e&&o(nh),e&&o(He),y(Ca),e&&o(ih),e&&o(R),e&&o(dh),y(qa,e),e&&o(hh),y(oo,e),e&&o(uh),e&&o(Le),y(za),e&&o(fh),e&&o(ce),e&&o(ch),y(Ja,e),e&&o(ph),e&&o(q),e&&o(mh),e&&o(Oe),y(Aa),e&&o(bh),e&&o(gl),e&&o(vh),e&&o(Ke),y(Na),e&&o(yh),e&&o(P),e&&o(_h),e&&o(G),e&&o(gh),y(Ua,e),e&&o(wh),e&&o(pe),e&&o(Eh),y(Wa,e),e&&o($h),e&&o(El),e&&o(kh),y(Za,e),e&&o(Mh),e&&o($l),e&&o(Th),e&&o(et),y(Pa),e&&o(Ch),e&&o(me),e&&o(qh),y(Ga,e),e&&o(zh),e&&o(tt),y(ja),e&&o(Jh),e&&o(ot),e&&o(Ah),y(Qa,e),e&&o(Nh),e&&o(at),y(Ba),e&&o(Uh),e&&o(V),e&&o(Wh),e&&o(z),e&&o(Zh),e&&o(lt),y(xa),e&&o(Ph),e&&o(M),y(Xa),y(Ra),y(Va),y(Da),y(Sa),e&&o(Gh),e&&o(nt),y(Ha),e&&o(jh),e&&o(be)}}}const y0={local:"quantize-transformers-models",sections:[{local:"autogptq-integration",sections:[{local:"requirements",title:"Requirements"},{local:"load-and-quantize-a-model",sections:[{local:"gptq-configuration",title:"GPTQ Configuration"},{local:"quantization",title:"Quantization"}],title:"Load and quantize a model"},{local:"push-quantized-model-to-hub",title:"Push quantized model to \u{1F917} Hub"},{local:"load-a-quantized-model-from-the-hub",title:"Load a quantized model from the \u{1F917} Hub"},{local:"exllama-kernels-for-faster-inference",sections:[{local:"finetune-a-quantized-model",title:"Fine-tune a quantized model "}],title:"Exllama kernels for faster inference"},{local:"example-demo",title:"Example demo"},{local:"transformers.GPTQConfig",title:"GPTQConfig"}],title:"`AutoGPTQ` Integration"},{local:"bitsandbytes-integration",sections:[{local:"general-usage",title:"General usage"},{local:"fp4-quantization",sections:[{local:"requirements",title:"Requirements"},{local:"tips-and-best-practices",title:"Tips and best practices"},{local:"load-a-large-model-in-4bit",title:"Load a large model in 4bit"}],title:"FP4 quantization "},{local:"load-a-large-model-in-8bit",sections:[{local:"advanced-use-cases",sections:[{local:"change-the-compute-dtype",title:"Change the compute dtype"},{local:"using-nf4-normal-float-4-data-type",title:"Using NF4 (Normal Float 4) data type "},{local:"use-nested-quantization-for-more-memory-efficient-inference",title:"Use nested quantization for more memory efficient inference"}],title:"Advanced use cases"}],title:"Load a large model in 8bit"},{local:"push-quantized-models-on-the-hub",title:"Push quantized models on the \u{1F917} Hub"},{local:"load-a-quantized-model-from-the-hub",title:"Load a quantized model from the \u{1F917} Hub"},{local:"advanced-use-cases",sections:[{local:"offload-between-cpu-and-gpu",title:"Offload between `cpu` and `gpu`"},{local:"play-with-llmint8threshold",title:"Play with `llm_int8_threshold`"},{local:"skip-the-conversion-of-some-modules",title:"Skip the conversion of some modules"},{local:"finetune-a-model-that-has-been-loaded-in-8bit",title:"Fine-tune a model that has been loaded in 8-bit"}],title:"Advanced use cases"},{local:"transformers.BitsAndBytesConfig",title:"BitsAndBytesConfig"}],title:"`bitsandbytes` Integration"},{local:"quantization-with-optimum",title:"Quantization with \u{1F917} `optimum` "}],title:"Quantize \u{1F917} Transformers models"};function _0(K){return f0(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class M0 extends i0{constructor(_){super();d0(this,_,_0,v0,h0,{})}}export{M0 as default,y0 as metadata};
