import{s as mt,o as ut,n as pt}from"../chunks/scheduler.25b97de1.js";import{S as ct,i as ft,g as l,s as o,r as x,A as dt,h as r,f as a,c as s,j as lt,u as k,x as u,k as Y,y as ht,a as n,v as U,d as V,t as J,w as Z}from"../chunks/index.d9030fc9.js";import{T as gt}from"../chunks/Tip.baa67368.js";import{C as rt}from"../chunks/CodeBlock.e6cd0d95.js";import{H as vt,E as bt}from"../chunks/EditOnGithub.91d95064.js";function yt(Q){let i,f='Try Quanto + transformers with this <a href="https://colab.research.google.com/drive/16CXfVmtdQvciSh9BopZUDYcmXCDpvgrT?usp=sharing" rel="nofollow">notebook</a>!';return{c(){i=l("p"),i.innerHTML=f},l(m){i=r(m,"P",{"data-svelte-h":!0}),u(i)!=="svelte-1m6vmvl"&&(i.innerHTML=f)},m(m,C){n(m,i,C)},p:pt,d(m){m&&a(i)}}}function wt(Q){let i,f,m,C,d,L,p,G,h,B='<a href="https://github.com/huggingface/quanto" rel="nofollow">🤗 Quanto</a> library is a versatile pytorch quantization toolkit. The quantization method used is the linear quantization. Quanto provides several unique features such as:',H,g,K="<li>weights quantization (<code>float8</code>,<code>int8</code>,<code>int4</code>,<code>int2</code>)</li> <li>activation quantization (<code>float8</code>,<code>int8</code>)</li> <li>modality agnostic (e.g CV,LLM)</li> <li>device agnostic (e.g CUDA,MPS,CPU)</li> <li>compatibility with <code>torch.compile</code></li> <li>easy to add custom kernel for specific device</li> <li>supports quantization aware training</li>",X,v,O="Before you begin, make sure the following libraries are installed:",F,b,P,y,tt='Now you can quantize a model by passing <a href="/docs/transformers/main/en/main_classes/quantization#transformers.QuantoConfig">QuantoConfig</a> object in the <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method. This works for any model in any modality, as long as it contains <code>torch.nn.Linear</code> layers.',E,w,et='The integration with transformers only supports weights quantization. For the more complex use case such as activation quantization, calibration and quantization aware training, you should use <a href="https://github.com/huggingface/quanto" rel="nofollow">quanto</a> library instead.',W,$,j,T,at="Note that serialization is not supported yet with transformers but it is coming soon! If you want to save the model, you can use quanto library instead.",I,M,nt='Quanto library uses linear quantization algorithm for quantization. Even though this is a basic quantization technique, we get very good results! Have a look at the following benchmark (llama-2-7b on perplexity metric). You can find more benchmarks <a href="https://github.com/huggingface/quanto/tree/main/bench/generation" rel="nofollow">here</a>',R,c,it='<div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/NousResearch-Llama-2-7b-hf_Perplexity.png" alt="llama-2-7b-quanto-perplexity"/></div>',N,_,ot="The library is versatile enough to be compatible with most PTQ optimization algorithms. The plan in the future is to integrate the most popular algorithms in the most seamless possible way (AWQ, Smoothquant).",S,q,A,z,D;return d=new vt({props:{title:"Quanto",local:"quanto",headingTag:"h1"}}),p=new gt({props:{$$slots:{default:[yt]},$$scope:{ctx:Q}}}),b=new rt({props:{code:"cGlwJTIwaW5zdGFsbCUyMHF1YW50byUyMGFjY2VsZXJhdGUlMjB0cmFuc2Zvcm1lcnM=",highlighted:"pip install quanto accelerate transformers",wrap:!1}}),$=new rt({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMFF1YW50b0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyZmFjZWJvb2slMkZvcHQtMTI1bSUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBRdWFudG9Db25maWcod2VpZ2h0cyUzRCUyMmludDglMjIpJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZGV2aWNlX21hcCUzRCUyMmN1ZGElM0EwJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, QuantoConfig

model_id = <span class="hljs-string">&quot;facebook/opt-125m&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
quantization_config = QuantoConfig(weights=<span class="hljs-string">&quot;int8&quot;</span>)
quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;cuda:0&quot;</span>, quantization_config=quantization_config)`,wrap:!1}}),q=new bt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/quanto.md"}}),{c(){i=l("meta"),f=o(),m=l("p"),C=o(),x(d.$$.fragment),L=o(),x(p.$$.fragment),G=o(),h=l("p"),h.innerHTML=B,H=o(),g=l("ul"),g.innerHTML=K,X=o(),v=l("p"),v.textContent=O,F=o(),x(b.$$.fragment),P=o(),y=l("p"),y.innerHTML=tt,E=o(),w=l("p"),w.innerHTML=et,W=o(),x($.$$.fragment),j=o(),T=l("p"),T.textContent=at,I=o(),M=l("p"),M.innerHTML=nt,R=o(),c=l("div"),c.innerHTML=it,N=o(),_=l("p"),_.textContent=ot,S=o(),x(q.$$.fragment),A=o(),z=l("p"),this.h()},l(t){const e=dt("svelte-u9bgzb",document.head);i=r(e,"META",{name:!0,content:!0}),e.forEach(a),f=s(t),m=r(t,"P",{}),lt(m).forEach(a),C=s(t),k(d.$$.fragment,t),L=s(t),k(p.$$.fragment,t),G=s(t),h=r(t,"P",{"data-svelte-h":!0}),u(h)!=="svelte-1uxysak"&&(h.innerHTML=B),H=s(t),g=r(t,"UL",{"data-svelte-h":!0}),u(g)!=="svelte-1rrl1p8"&&(g.innerHTML=K),X=s(t),v=r(t,"P",{"data-svelte-h":!0}),u(v)!=="svelte-13c61n1"&&(v.textContent=O),F=s(t),k(b.$$.fragment,t),P=s(t),y=r(t,"P",{"data-svelte-h":!0}),u(y)!=="svelte-a77ps0"&&(y.innerHTML=tt),E=s(t),w=r(t,"P",{"data-svelte-h":!0}),u(w)!=="svelte-acy0np"&&(w.innerHTML=et),W=s(t),k($.$$.fragment,t),j=s(t),T=r(t,"P",{"data-svelte-h":!0}),u(T)!=="svelte-1sscg20"&&(T.textContent=at),I=s(t),M=r(t,"P",{"data-svelte-h":!0}),u(M)!=="svelte-6cetal"&&(M.innerHTML=nt),R=s(t),c=r(t,"DIV",{class:!0,"data-svelte-h":!0}),u(c)!=="svelte-196rp13"&&(c.innerHTML=it),N=s(t),_=r(t,"P",{"data-svelte-h":!0}),u(_)!=="svelte-1fyeg9"&&(_.textContent=ot),S=s(t),k(q.$$.fragment,t),A=s(t),z=r(t,"P",{}),lt(z).forEach(a),this.h()},h(){Y(i,"name","hf:doc:metadata"),Y(i,"content",$t),Y(c,"class","flex gap-4")},m(t,e){ht(document.head,i),n(t,f,e),n(t,m,e),n(t,C,e),U(d,t,e),n(t,L,e),U(p,t,e),n(t,G,e),n(t,h,e),n(t,H,e),n(t,g,e),n(t,X,e),n(t,v,e),n(t,F,e),U(b,t,e),n(t,P,e),n(t,y,e),n(t,E,e),n(t,w,e),n(t,W,e),U($,t,e),n(t,j,e),n(t,T,e),n(t,I,e),n(t,M,e),n(t,R,e),n(t,c,e),n(t,N,e),n(t,_,e),n(t,S,e),U(q,t,e),n(t,A,e),n(t,z,e),D=!0},p(t,[e]){const st={};e&2&&(st.$$scope={dirty:e,ctx:t}),p.$set(st)},i(t){D||(V(d.$$.fragment,t),V(p.$$.fragment,t),V(b.$$.fragment,t),V($.$$.fragment,t),V(q.$$.fragment,t),D=!0)},o(t){J(d.$$.fragment,t),J(p.$$.fragment,t),J(b.$$.fragment,t),J($.$$.fragment,t),J(q.$$.fragment,t),D=!1},d(t){t&&(a(f),a(m),a(C),a(L),a(G),a(h),a(H),a(g),a(X),a(v),a(F),a(P),a(y),a(E),a(w),a(W),a(j),a(T),a(I),a(M),a(R),a(c),a(N),a(_),a(S),a(A),a(z)),a(i),Z(d,t),Z(p,t),Z(b,t),Z($,t),Z(q,t)}}}const $t='{"title":"Quanto","local":"quanto","sections":[],"depth":1}';function Tt(Q){return ut(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class xt extends ct{constructor(i){super(),ft(this,i,Tt,wt,mt,{})}}export{xt as component};
