import{s as ct,o as ft,n as dt}from"../chunks/scheduler.25b97de1.js";import{S as ht,i as gt,g as s,s as o,r as J,A as yt,h as r,f as a,c as l,j as ut,u as U,x as u,k as O,y as vt,a as i,v as k,d as V,t as Z,w as H}from"../chunks/index.d9030fc9.js";import{T as bt}from"../chunks/Tip.baa67368.js";import{C as pt}from"../chunks/CodeBlock.e6cd0d95.js";import{H as wt,E as Tt}from"../chunks/EditOnGithub.91d95064.js";function qt(L){let n,f='Try optimum-quanto + transformers with this <a href="https://colab.research.google.com/drive/16CXfVmtdQvciSh9BopZUDYcmXCDpvgrT?usp=sharing" rel="nofollow">notebook</a>!';return{c(){n=s("p"),n.innerHTML=f},l(m){n=r(m,"P",{"data-svelte-h":!0}),u(n)!=="svelte-12h31kl"&&(n.innerHTML=f)},m(m,z){i(m,n,z)},p:dt,d(m){m&&a(n)}}}function $t(L){let n,f,m,z,d,G,p,X,h,K='<a href="https://github.com/huggingface/optimum-quanto" rel="nofollow">🤗 optimum-quanto</a> library is a versatile pytorch quantization toolkit. The quantization method used is the linear quantization. Quanto provides several unique features such as:',P,g,tt="<li>weights quantization (<code>float8</code>,<code>int8</code>,<code>int4</code>,<code>int2</code>)</li> <li>activation quantization (<code>float8</code>,<code>int8</code>)</li> <li>modality agnostic (e.g CV,LLM)</li> <li>device agnostic (e.g CUDA,XPU,MPS,CPU)</li> <li>compatibility with <code>torch.compile</code></li> <li>easy to add custom kernel for specific device</li> <li>supports quantization aware training</li>",W,y,et="Before you begin, make sure the following libraries are installed:",Q,v,j,b,at='Now you can quantize a model by passing <a href="/docs/transformers/main/en/main_classes/quantization#transformers.QuantoConfig">QuantoConfig</a> object in the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method. This works for any model in any modality, as long as it contains <code>torch.nn.Linear</code> layers.',F,w,it='The integration with transformers only supports weights quantization. For the more complex use case such as activation quantization, calibration and quantization aware training, you should use <a href="https://github.com/huggingface/optimum-quanto" rel="nofollow">optimum-quanto</a> library instead.',I,T,nt="By default, the weights are loaded in full precision (torch.float32) regardless of the actual data type the weights are stored in such as torch.float16. Set <code>torch_dtype=&quot;auto&quot;</code> to load the weights in the data type defined in a model’s <code>config.json</code> file to automatically load the most memory-optimal data type.",R,q,E,$,ot="Note that serialization is not supported yet with transformers but it is coming soon! If you want to save the model, you can use quanto library instead.",N,_,lt='Optimum-quanto library uses linear quantization algorithm for quantization. Even though this is a basic quantization technique, we get very good results! Have a look at the following benchmark (llama-2-7b on perplexity metric). You can find more benchmarks <a href="https://github.com/huggingface/optimum-quanto/tree/main/bench/generation" rel="nofollow">here</a>',S,c,st='<div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/NousResearch-Llama-2-7b-hf_Perplexity.png" alt="llama-2-7b-quanto-perplexity"/></div>',D,M,rt="The library is versatile enough to be compatible with most PTQ optimization algorithms. The plan in the future is to integrate the most popular algorithms in the most seamless possible way (AWQ, Smoothquant).",A,C,Y,x,B;return d=new wt({props:{title:"Optimum-quanto",local:"optimum-quanto",headingTag:"h1"}}),p=new bt({props:{$$slots:{default:[qt]},$$scope:{ctx:L}}}),v=new pt({props:{code:"cGlwJTIwaW5zdGFsbCUyMG9wdGltdW0tcXVhbnRvJTIwYWNjZWxlcmF0ZSUyMHRyYW5zZm9ybWVycw==",highlighted:"pip install optimum-quanto accelerate transformers",wrap:!1}}),q=new pt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMFF1YW50b0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyZmFjZWJvb2slMkZvcHQtMTI1bSUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBRdWFudG9Db25maWcod2VpZ2h0cyUzRCUyMmludDglMjIpJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmN1ZGElM0EwJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, QuantoConfig

model_id = <span class="hljs-string">&quot;facebook/opt-125m&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
quantization_config = QuantoConfig(weights=<span class="hljs-string">&quot;int8&quot;</span>)
quantized_model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, device_map=<span class="hljs-string">&quot;cuda:0&quot;</span>, quantization_config=quantization_config)`,wrap:!1}}),C=new Tt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/quanto.md"}}),{c(){n=s("meta"),f=o(),m=s("p"),z=o(),J(d.$$.fragment),G=o(),J(p.$$.fragment),X=o(),h=s("p"),h.innerHTML=K,P=o(),g=s("ul"),g.innerHTML=tt,W=o(),y=s("p"),y.textContent=et,Q=o(),J(v.$$.fragment),j=o(),b=s("p"),b.innerHTML=at,F=o(),w=s("p"),w.innerHTML=it,I=o(),T=s("p"),T.innerHTML=nt,R=o(),J(q.$$.fragment),E=o(),$=s("p"),$.textContent=ot,N=o(),_=s("p"),_.innerHTML=lt,S=o(),c=s("div"),c.innerHTML=st,D=o(),M=s("p"),M.textContent=rt,A=o(),J(C.$$.fragment),Y=o(),x=s("p"),this.h()},l(t){const e=yt("svelte-u9bgzb",document.head);n=r(e,"META",{name:!0,content:!0}),e.forEach(a),f=l(t),m=r(t,"P",{}),ut(m).forEach(a),z=l(t),U(d.$$.fragment,t),G=l(t),U(p.$$.fragment,t),X=l(t),h=r(t,"P",{"data-svelte-h":!0}),u(h)!=="svelte-qcmv6u"&&(h.innerHTML=K),P=l(t),g=r(t,"UL",{"data-svelte-h":!0}),u(g)!=="svelte-10h0pwp"&&(g.innerHTML=tt),W=l(t),y=r(t,"P",{"data-svelte-h":!0}),u(y)!=="svelte-13c61n1"&&(y.textContent=et),Q=l(t),U(v.$$.fragment,t),j=l(t),b=r(t,"P",{"data-svelte-h":!0}),u(b)!=="svelte-a77ps0"&&(b.innerHTML=at),F=l(t),w=r(t,"P",{"data-svelte-h":!0}),u(w)!=="svelte-1u1219t"&&(w.innerHTML=it),I=l(t),T=r(t,"P",{"data-svelte-h":!0}),u(T)!=="svelte-9fdmcv"&&(T.innerHTML=nt),R=l(t),U(q.$$.fragment,t),E=l(t),$=r(t,"P",{"data-svelte-h":!0}),u($)!=="svelte-1sscg20"&&($.textContent=ot),N=l(t),_=r(t,"P",{"data-svelte-h":!0}),u(_)!=="svelte-ui8rjh"&&(_.innerHTML=lt),S=l(t),c=r(t,"DIV",{class:!0,"data-svelte-h":!0}),u(c)!=="svelte-196rp13"&&(c.innerHTML=st),D=l(t),M=r(t,"P",{"data-svelte-h":!0}),u(M)!=="svelte-1fyeg9"&&(M.textContent=rt),A=l(t),U(C.$$.fragment,t),Y=l(t),x=r(t,"P",{}),ut(x).forEach(a),this.h()},h(){O(n,"name","hf:doc:metadata"),O(n,"content",_t),O(c,"class","flex gap-4")},m(t,e){vt(document.head,n),i(t,f,e),i(t,m,e),i(t,z,e),k(d,t,e),i(t,G,e),k(p,t,e),i(t,X,e),i(t,h,e),i(t,P,e),i(t,g,e),i(t,W,e),i(t,y,e),i(t,Q,e),k(v,t,e),i(t,j,e),i(t,b,e),i(t,F,e),i(t,w,e),i(t,I,e),i(t,T,e),i(t,R,e),k(q,t,e),i(t,E,e),i(t,$,e),i(t,N,e),i(t,_,e),i(t,S,e),i(t,c,e),i(t,D,e),i(t,M,e),i(t,A,e),k(C,t,e),i(t,Y,e),i(t,x,e),B=!0},p(t,[e]){const mt={};e&2&&(mt.$$scope={dirty:e,ctx:t}),p.$set(mt)},i(t){B||(V(d.$$.fragment,t),V(p.$$.fragment,t),V(v.$$.fragment,t),V(q.$$.fragment,t),V(C.$$.fragment,t),B=!0)},o(t){Z(d.$$.fragment,t),Z(p.$$.fragment,t),Z(v.$$.fragment,t),Z(q.$$.fragment,t),Z(C.$$.fragment,t),B=!1},d(t){t&&(a(f),a(m),a(z),a(G),a(X),a(h),a(P),a(g),a(W),a(y),a(Q),a(j),a(b),a(F),a(w),a(I),a(T),a(R),a(E),a($),a(N),a(_),a(S),a(c),a(D),a(M),a(A),a(Y),a(x)),a(n),H(d,t),H(p,t),H(v,t),H(q,t),H(C,t)}}}const _t='{"title":"Optimum-quanto","local":"optimum-quanto","sections":[],"depth":1}';function Mt(L){return ft(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class kt extends ht{constructor(n){super(),gt(this,n,Mt,$t,ct,{})}}export{kt as component};
