import{s as st,o as it,n as ot}from"../chunks/scheduler.25b97de1.js";import{S as pt,i as rt,g as p,s,r as F,A as mt,h as r,f as a,c as i,j as lt,u as C,x as u,k as nt,y as ut,a as l,v as V,d as G,t as T,w as k}from"../chunks/index.d9030fc9.js";import{T as dt}from"../chunks/Tip.baa67368.js";import{C as S}from"../chunks/CodeBlock.e6cd0d95.js";import{H as ct,E as ft}from"../chunks/EditOnGithub.91d95064.js";function ht(J){let n,d="You need a GPU with compute capability>=9 (e.g. H100)";return{c(){n=p("p"),n.textContent=d},l(o){n=r(o,"P",{"data-svelte-h":!0}),u(n)!=="svelte-uga55r"&&(n.textContent=d)},m(o,Z){l(o,n,Z)},p:ot,d(o){o&&a(n)}}}function Mt(J){let n,d,o,Z,c,q,f,N="With FBGEMM FP8 quantization method, you can quantize your model in FP8 (W8A8):",R,h,A="<li>the weights will be quantized in 8bit (FP8) per channel</li> <li>the activation will be quantized in 8bit (FP8) per token</li>",W,M,K='It relies on the <a href="https://github.com/pytorch/FBGEMM" rel="nofollow">FBGEMM</a> library which provides efficient low-precision general matrix multiplication for small batch sizes and support for accuracy-loss minimizing techniques such as row-wise quantization and outlier-aware quantization.',z,m,B,b,D="Before you begin, make sure the following libraries are installed with their latest version:",j,y,x,w,O='If you are having issues with fbgemm-gpu and torch library, you might need to install the nightly release. You can follow the instruction <a href="https://pytorch.org/FBGEMM/fbgemm_gpu-development/InstallationInstructions.html#fbgemm-gpu-install-libraries:~:text=found%20here.-,Install%20the%20FBGEMM_GPU%20Package,-Install%20through%20PyTorch" rel="nofollow">here</a>',E,U,tt="By default, the weights are loaded in full precision (torch.float32) regardless of the actual data type the weights are stored in such as torch.float16. Set <code>torch_dtype=&quot;auto&quot;</code> to load the weights in the data type defined in a model’s <code>config.json</code> file to automatically load the most memory-optimal data type.",H,g,P,v,et="A quantized model can be saved via “saved_pretrained” and be reused again via the “from_pretrained”.",Y,_,Q,$,I,X,L;return c=new ct({props:{title:"FBGEMM FP8",local:"fbgemm-fp8",headingTag:"h1"}}),m=new dt({props:{warning:!1,$$slots:{default:[ht]},$$scope:{ctx:J}}}),y=new S({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMGFjY2VsZXJhdGUlMjBmYmdlbW0tZ3B1JTIwdG9yY2g=",highlighted:"pip install --upgrade accelerate fbgemm-gpu torch",wrap:!1}}),g=new S({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEZiZ2VtbUZwOENvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX25hbWUlMjAlM0QlMjAlMjJtZXRhLWxsYW1hJTJGTWV0YS1MbGFtYS0zLThCJTIyJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEZiZ2VtbUZwOENvbmZpZygpJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUlMkMlMjB0b3JjaF9kdHlwZSUzRCUyMmF1dG8lMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX25hbWUpJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMldoYXQlMjBhcmUlMjB3ZSUyMGhhdmluZyUyMGZvciUyMGRpbm5lciUzRiUyMiUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplcihpbnB1dF90ZXh0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oJTIyY3VkYSUyMiklMEElMEFvdXRwdXQlMjAlM0QlMjBxdWFudGl6ZWRfbW9kZWwuZ2VuZXJhdGUoKippbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0JTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> FbgemmFp8Config, AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B&quot;</span>
quantization_config = FbgemmFp8Config()
quantized_model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=quantization_config)

tokenizer = AutoTokenizer.from_pretrained(model_name)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),_=new S({props:{code:"cXVhbnRfcGF0aCUyMCUzRCUyMCUyMiUyRnBhdGglMkZ0byUyRnNhdmUlMkZxdWFudGl6ZWQlMkZtb2RlbCUyMiUwQW1vZGVsLnNhdmVfcHJldHJhaW5lZChxdWFudF9wYXRoKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKHF1YW50X3BhdGglMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`quant_path = <span class="hljs-string">&quot;/path/to/save/quantized/model&quot;</span>
model.save_pretrained(quant_path)
model = AutoModelForCausalLM.from_pretrained(quant_path, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),$=new ft({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/fbgemm_fp8.md"}}),{c(){n=p("meta"),d=s(),o=p("p"),Z=s(),F(c.$$.fragment),q=s(),f=p("p"),f.textContent=N,R=s(),h=p("ul"),h.innerHTML=A,W=s(),M=p("p"),M.innerHTML=K,z=s(),F(m.$$.fragment),B=s(),b=p("p"),b.textContent=D,j=s(),F(y.$$.fragment),x=s(),w=p("p"),w.innerHTML=O,E=s(),U=p("p"),U.innerHTML=tt,H=s(),F(g.$$.fragment),P=s(),v=p("p"),v.textContent=et,Y=s(),F(_.$$.fragment),Q=s(),F($.$$.fragment),I=s(),X=p("p"),this.h()},l(t){const e=mt("svelte-u9bgzb",document.head);n=r(e,"META",{name:!0,content:!0}),e.forEach(a),d=i(t),o=r(t,"P",{}),lt(o).forEach(a),Z=i(t),C(c.$$.fragment,t),q=i(t),f=r(t,"P",{"data-svelte-h":!0}),u(f)!=="svelte-1ippvwg"&&(f.textContent=N),R=i(t),h=r(t,"UL",{"data-svelte-h":!0}),u(h)!=="svelte-c7lsip"&&(h.innerHTML=A),W=i(t),M=r(t,"P",{"data-svelte-h":!0}),u(M)!=="svelte-p8lslh"&&(M.innerHTML=K),z=i(t),C(m.$$.fragment,t),B=i(t),b=r(t,"P",{"data-svelte-h":!0}),u(b)!=="svelte-1wrsrbm"&&(b.textContent=D),j=i(t),C(y.$$.fragment,t),x=i(t),w=r(t,"P",{"data-svelte-h":!0}),u(w)!=="svelte-ygzl1b"&&(w.innerHTML=O),E=i(t),U=r(t,"P",{"data-svelte-h":!0}),u(U)!=="svelte-9fdmcv"&&(U.innerHTML=tt),H=i(t),C(g.$$.fragment,t),P=i(t),v=r(t,"P",{"data-svelte-h":!0}),u(v)!=="svelte-b8ansb"&&(v.textContent=et),Y=i(t),C(_.$$.fragment,t),Q=i(t),C($.$$.fragment,t),I=i(t),X=r(t,"P",{}),lt(X).forEach(a),this.h()},h(){nt(n,"name","hf:doc:metadata"),nt(n,"content",bt)},m(t,e){ut(document.head,n),l(t,d,e),l(t,o,e),l(t,Z,e),V(c,t,e),l(t,q,e),l(t,f,e),l(t,R,e),l(t,h,e),l(t,W,e),l(t,M,e),l(t,z,e),V(m,t,e),l(t,B,e),l(t,b,e),l(t,j,e),V(y,t,e),l(t,x,e),l(t,w,e),l(t,E,e),l(t,U,e),l(t,H,e),V(g,t,e),l(t,P,e),l(t,v,e),l(t,Y,e),V(_,t,e),l(t,Q,e),V($,t,e),l(t,I,e),l(t,X,e),L=!0},p(t,[e]){const at={};e&2&&(at.$$scope={dirty:e,ctx:t}),m.$set(at)},i(t){L||(G(c.$$.fragment,t),G(m.$$.fragment,t),G(y.$$.fragment,t),G(g.$$.fragment,t),G(_.$$.fragment,t),G($.$$.fragment,t),L=!0)},o(t){T(c.$$.fragment,t),T(m.$$.fragment,t),T(y.$$.fragment,t),T(g.$$.fragment,t),T(_.$$.fragment,t),T($.$$.fragment,t),L=!1},d(t){t&&(a(d),a(o),a(Z),a(q),a(f),a(R),a(h),a(W),a(M),a(z),a(B),a(b),a(j),a(x),a(w),a(E),a(U),a(H),a(P),a(v),a(Y),a(Q),a(I),a(X)),a(n),k(c,t),k(m,t),k(y,t),k(g,t),k(_,t),k($,t)}}}const bt='{"title":"FBGEMM FP8","local":"fbgemm-fp8","sections":[],"depth":1}';function yt(J){return it(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $t extends pt{constructor(n){super(),rt(this,n,yt,Mt,st,{})}}export{$t as component};
