import{s as yt,n as xt,o as Pt}from"../chunks/scheduler.01eeda35.js";import{S as Lt,i as _t,g as o,s as i,r as Q,A as Ct,h as d,f as l,c as a,j as gt,u as k,x as r,k as wt,y as Qt,a as n,v as q,d as V,t as j,w as H}from"../chunks/index.6dd51b66.js";import{C as $t}from"../chunks/CodeBlock.864da1b0.js";import{H as at,E as kt}from"../chunks/EditOnGithub.7faefd25.js";function qt(ot){let s,E,A,B,m,U,u,dt='<a href="https://github.com/microsoft/VPTQ" rel="nofollow">Vector Post-Training Quantization (VPTQ)</a> is a Post-Training Quantization (PTQ) method that leverages vector quantization to quantize LLMs at an extremely low bit-width (&lt;2-bit). VPTQ can compress a 70B, even a 405B model, to 1-2 bits without retraining and still maintain a high-degree of accuracy. It is a lightweight quantization algorithm that takes ~17 hours to quantize a 405B model. VPTQ features agile quantization inference with low decoding overhead and high throughput and Time To First Token (TTFT).',F,f,rt="Run the command below to install VPTQ which provides efficient kernels for inference on NVIDIA and AMD GPUs.",G,h,W,p,st='The <a href="https://huggingface.co/VPTQ-community" rel="nofollow">VPTQ-community</a> provides a collection of VPTQ-quantized models. The model name contains information about its bitwidth (excluding cookbook, parameter, and padding overhead). Consider the [Meta-Llama-3.1-70B-Instruct-v8-k65536-256-woft] model as an example.',R,c,mt="<li>The model name is Meta-Llama-3.1-70B-Instruct.</li> <li>The number of centroids is given by 65536 (2^16).</li> <li>The number of residual centroids is given by 256 (2^8).</li>",Z,T,ut="The equivalent bit-width calculation is given by the following.",I,M,ft="<li>index: log2(65536) = 16 / 8 = 2-bits</li> <li>residual index: log2(256) = 8 / 8 = 1-bit</li> <li>total bit-width: 2 + 1 = 3-bits</li>",S,b,ht="From here, estimate the model size by multiplying 70B * 3-bits / 8-bits/byte for a total of 26.25GB.",J,v,pt='Load a VPTQ quantized model with <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a>.',N,g,X,w,ct='To quantize your own model, refer to the <a href="https://github.com/microsoft/VPTQ/blob/algorithm/algorithm.md" rel="nofollow">VPTQ Quantization Algorithm Tutorial</a> tutorial.',Y,$,D,y,Tt="VPTQ achieves better accuracy and higher throughput with lower quantization overhead across models of different sizes. The following experimental results are for reference only; VPTQ can achieve better outcomes under reasonable parameters, especially in terms of model accuracy and inference speed.",O,x,Mt="<thead><tr><th>Model</th> <th>bitwidth</th> <th>W2↓</th> <th>C4↓</th> <th>AvgQA↑</th> <th>tok/s↑</th> <th>mem(GB)</th> <th>cost/h↓</th></tr></thead> <tbody><tr><td>LLaMA-2 7B</td> <td>2.02</td> <td>6.13</td> <td>8.07</td> <td>58.2</td> <td>39.9</td> <td>2.28</td> <td>2</td></tr> <tr><td></td> <td>2.26</td> <td>5.95</td> <td>7.87</td> <td>59.4</td> <td>35.7</td> <td>2.48</td> <td>3.1</td></tr> <tr><td>LLaMA-2 13B</td> <td>2.02</td> <td>5.32</td> <td>7.15</td> <td>62.4</td> <td>26.9</td> <td>4.03</td> <td>3.2</td></tr> <tr><td></td> <td>2.18</td> <td>5.28</td> <td>7.04</td> <td>63.1</td> <td>18.5</td> <td>4.31</td> <td>3.6</td></tr> <tr><td>LLaMA-2 70B</td> <td>2.07</td> <td>3.93</td> <td>5.72</td> <td>68.6</td> <td>9.7</td> <td>19.54</td> <td>19</td></tr> <tr><td></td> <td>2.11</td> <td>3.92</td> <td>5.71</td> <td>68.7</td> <td>9.7</td> <td>20.01</td> <td>19</td></tr></tbody>",K,P,tt,L,bt='See an example demo of VPTQ on the VPTQ Online Demo <a href="https://huggingface.co/spaces/microsoft/VPTQ" rel="nofollow">Space</a> or try running the VPTQ inference <a href="https://colab.research.google.com/github/microsoft/VPTQ/blob/main/notebooks/vptq_example.ipynb" rel="nofollow">notebook</a>.',et,_,vt='For more information, read the VPTQ <a href="https://arxiv.org/pdf/2409.17066" rel="nofollow">paper</a>.',lt,C,nt,z,it;return m=new at({props:{title:"VPTQ",local:"vptq",headingTag:"h1"}}),h=new $t({props:{code:"cGlwJTIwaW5zdGFsbCUyMHZwdHE=",highlighted:"pip install vptq",wrap:!1}}),g=new $t({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJWUFRRLWNvbW11bml0eSUyRk1ldGEtTGxhbWEtMy4xLTcwQi1JbnN0cnVjdC12MTYtazY1NTM2LTY1NTM2LXdvZnQlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRCUyMmF1dG8lMjIlMkMlMjAlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;VPTQ-community/Meta-Llama-3.1-70B-Instruct-v16-k65536-65536-woft&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, 
    device_map=<span class="hljs-string">&quot;auto&quot;</span>
)`,wrap:!1}}),$=new at({props:{title:"Benchmarks",local:"benchmarks",headingTag:"h2"}}),P=new at({props:{title:"Resources",local:"resources",headingTag:"h2"}}),C=new kt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/vptq.md"}}),{c(){s=o("meta"),E=i(),A=o("p"),B=i(),Q(m.$$.fragment),U=i(),u=o("p"),u.innerHTML=dt,F=i(),f=o("p"),f.textContent=rt,G=i(),Q(h.$$.fragment),W=i(),p=o("p"),p.innerHTML=st,R=i(),c=o("ul"),c.innerHTML=mt,Z=i(),T=o("p"),T.textContent=ut,I=i(),M=o("ul"),M.innerHTML=ft,S=i(),b=o("p"),b.textContent=ht,J=i(),v=o("p"),v.innerHTML=pt,N=i(),Q(g.$$.fragment),X=i(),w=o("p"),w.innerHTML=ct,Y=i(),Q($.$$.fragment),D=i(),y=o("p"),y.textContent=Tt,O=i(),x=o("table"),x.innerHTML=Mt,K=i(),Q(P.$$.fragment),tt=i(),L=o("p"),L.innerHTML=bt,et=i(),_=o("p"),_.innerHTML=vt,lt=i(),Q(C.$$.fragment),nt=i(),z=o("p"),this.h()},l(t){const e=Ct("svelte-u9bgzb",document.head);s=d(e,"META",{name:!0,content:!0}),e.forEach(l),E=a(t),A=d(t,"P",{}),gt(A).forEach(l),B=a(t),k(m.$$.fragment,t),U=a(t),u=d(t,"P",{"data-svelte-h":!0}),r(u)!=="svelte-1kvxjn7"&&(u.innerHTML=dt),F=a(t),f=d(t,"P",{"data-svelte-h":!0}),r(f)!=="svelte-17dfmu5"&&(f.textContent=rt),G=a(t),k(h.$$.fragment,t),W=a(t),p=d(t,"P",{"data-svelte-h":!0}),r(p)!=="svelte-1u1rjt1"&&(p.innerHTML=st),R=a(t),c=d(t,"UL",{"data-svelte-h":!0}),r(c)!=="svelte-1mdiby7"&&(c.innerHTML=mt),Z=a(t),T=d(t,"P",{"data-svelte-h":!0}),r(T)!=="svelte-yviqyq"&&(T.textContent=ut),I=a(t),M=d(t,"UL",{"data-svelte-h":!0}),r(M)!=="svelte-1s4gqbu"&&(M.innerHTML=ft),S=a(t),b=d(t,"P",{"data-svelte-h":!0}),r(b)!=="svelte-mv0147"&&(b.textContent=ht),J=a(t),v=d(t,"P",{"data-svelte-h":!0}),r(v)!=="svelte-1qwncgy"&&(v.innerHTML=pt),N=a(t),k(g.$$.fragment,t),X=a(t),w=d(t,"P",{"data-svelte-h":!0}),r(w)!=="svelte-1x0je1h"&&(w.innerHTML=ct),Y=a(t),k($.$$.fragment,t),D=a(t),y=d(t,"P",{"data-svelte-h":!0}),r(y)!=="svelte-l0x2ga"&&(y.textContent=Tt),O=a(t),x=d(t,"TABLE",{"data-svelte-h":!0}),r(x)!=="svelte-uqhmp2"&&(x.innerHTML=Mt),K=a(t),k(P.$$.fragment,t),tt=a(t),L=d(t,"P",{"data-svelte-h":!0}),r(L)!=="svelte-1vhx2x3"&&(L.innerHTML=bt),et=a(t),_=d(t,"P",{"data-svelte-h":!0}),r(_)!=="svelte-1enldrt"&&(_.innerHTML=vt),lt=a(t),k(C.$$.fragment,t),nt=a(t),z=d(t,"P",{}),gt(z).forEach(l),this.h()},h(){wt(s,"name","hf:doc:metadata"),wt(s,"content",Vt)},m(t,e){Qt(document.head,s),n(t,E,e),n(t,A,e),n(t,B,e),q(m,t,e),n(t,U,e),n(t,u,e),n(t,F,e),n(t,f,e),n(t,G,e),q(h,t,e),n(t,W,e),n(t,p,e),n(t,R,e),n(t,c,e),n(t,Z,e),n(t,T,e),n(t,I,e),n(t,M,e),n(t,S,e),n(t,b,e),n(t,J,e),n(t,v,e),n(t,N,e),q(g,t,e),n(t,X,e),n(t,w,e),n(t,Y,e),q($,t,e),n(t,D,e),n(t,y,e),n(t,O,e),n(t,x,e),n(t,K,e),q(P,t,e),n(t,tt,e),n(t,L,e),n(t,et,e),n(t,_,e),n(t,lt,e),q(C,t,e),n(t,nt,e),n(t,z,e),it=!0},p:xt,i(t){it||(V(m.$$.fragment,t),V(h.$$.fragment,t),V(g.$$.fragment,t),V($.$$.fragment,t),V(P.$$.fragment,t),V(C.$$.fragment,t),it=!0)},o(t){j(m.$$.fragment,t),j(h.$$.fragment,t),j(g.$$.fragment,t),j($.$$.fragment,t),j(P.$$.fragment,t),j(C.$$.fragment,t),it=!1},d(t){t&&(l(E),l(A),l(B),l(U),l(u),l(F),l(f),l(G),l(W),l(p),l(R),l(c),l(Z),l(T),l(I),l(M),l(S),l(b),l(J),l(v),l(N),l(X),l(w),l(Y),l(D),l(y),l(O),l(x),l(K),l(tt),l(L),l(et),l(_),l(lt),l(nt),l(z)),l(s),H(m,t),H(h,t),H(g,t),H($,t),H(P,t),H(C,t)}}}const Vt='{"title":"VPTQ","local":"vptq","sections":[{"title":"Benchmarks","local":"benchmarks","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function jt(ot){return Pt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Bt extends Lt{constructor(s){super(),_t(this,s,jt,qt,yt,{})}}export{Bt as component};
