import{s as K,n as O,o as ee}from"../chunks/scheduler.25b97de1.js";import{S as te,i as ae,g as p,s as l,r as y,A as se,h as o,f as a,c as n,j as P,u as T,x as J,k as D,y as le,a as s,v as E,d as $,t as U,w as v}from"../chunks/index.d9030fc9.js";import{C as Y}from"../chunks/CodeBlock.e6cd0d95.js";import{H as ne,E as ie}from"../chunks/EditOnGithub.91d95064.js";function pe(L){let i,j,Z,G,r,R,m,H='The <a href="https://github.com/NetEase-FuXi/EETQ" rel="nofollow">EETQ</a> library supports int8 per-channel weight-only quantization for NVIDIA GPUS. The high-performance GEMM and GEMV kernels are from FasterTransformer and TensorRT-LLM. It requires no calibration dataset and does not need to pre-quantize your model. Moreover, the accuracy degradation is negligible owing to the per-channel quantization.',_,u,I='Make sure you have eetq installed from the <a href="https://github.com/NetEase-FuXi/EETQ/releases" rel="nofollow">release page</a>',W,d,q,c,A='or via the source code <a href="https://github.com/NetEase-FuXi/EETQ" rel="nofollow">https://github.com/NetEase-FuXi/EETQ</a>. EETQ requires CUDA capability &lt;= 8.9 and &gt;= 7.0',F,h,X,f,S="An unquantized model can be quantized via “from_pretrained”.",x,M,k,b,B="A quantized model can be saved via “saved_pretrained” and be reused again via the “from_pretrained”.",Q,g,V,w,z,C,N;return r=new ne({props:{title:"EETQ",local:"eetq",headingTag:"h1"}}),d=new Y({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tbm8tY2FjaGUtZGlyJTIwaHR0cHMlM0ElMkYlMkZnaXRodWIuY29tJTJGTmV0RWFzZS1GdVhpJTJGRUVUUSUyRnJlbGVhc2VzJTJGZG93bmxvYWQlMkZ2MS4wLjAlMkZFRVRRLTEuMC4wJTJCY3UxMjElMkJ0b3JjaDIuMS4yLWNwMzEwLWNwMzEwLWxpbnV4X3g4Nl82NC53aGw=",highlighted:'pip install --no-cache-dir https:<span class="hljs-regexp">//gi</span>thub.com<span class="hljs-regexp">/NetEase-FuXi/</span>EETQ<span class="hljs-regexp">/releases/</span>download<span class="hljs-regexp">/v1.0.0/</span>EETQ-<span class="hljs-number">1.0</span>.<span class="hljs-number">0</span>+cu121+torch2.<span class="hljs-number">1.2</span>-cp310-cp310-linux_x86_64.whl',wrap:!1}}),h=new Y({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZOZXRFYXNlLUZ1WGklMkZFRVRRLmdpdCUwQWNkJTIwRUVUUSUyRiUwQWdpdCUyMHN1Ym1vZHVsZSUyMHVwZGF0ZSUyMC0taW5pdCUyMC0tcmVjdXJzaXZlJTBBcGlwJTIwaW5zdGFsbCUyMC4=",highlighted:`git clone https:<span class="hljs-regexp">//gi</span>thub.com<span class="hljs-regexp">/NetEase-FuXi/</span>EETQ.git
cd EETQ/
git submodule update --init --recursive
pip install .`,wrap:!1}}),M=new Y({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwRWV0cUNvbmZpZyUwQXBhdGglMjAlM0QlMjAlMjIlMkZwYXRoJTJGdG8lMkZtb2RlbCUyMiUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBFZXRxQ29uZmlnKCUyMmludDglMjIpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQocGF0aCUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, EetqConfig
path = <span class="hljs-string">&quot;/path/to/model&quot;</span>
quantization_config = EetqConfig(<span class="hljs-string">&quot;int8&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(path, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=quantization_config)`,wrap:!1}}),g=new Y({props:{code:"cXVhbnRfcGF0aCUyMCUzRCUyMCUyMiUyRnBhdGglMkZ0byUyRnNhdmUlMkZxdWFudGl6ZWQlMkZtb2RlbCUyMiUwQW1vZGVsLnNhdmVfcHJldHJhaW5lZChxdWFudF9wYXRoKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKHF1YW50X3BhdGglMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`quant_path = <span class="hljs-string">&quot;/path/to/save/quantized/model&quot;</span>
model.save_pretrained(quant_path)
model = AutoModelForCausalLM.from_pretrained(quant_path, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),w=new ie({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/eetq.md"}}),{c(){i=p("meta"),j=l(),Z=p("p"),G=l(),y(r.$$.fragment),R=l(),m=p("p"),m.innerHTML=H,_=l(),u=p("p"),u.innerHTML=I,W=l(),y(d.$$.fragment),q=l(),c=p("p"),c.innerHTML=A,F=l(),y(h.$$.fragment),X=l(),f=p("p"),f.textContent=S,x=l(),y(M.$$.fragment),k=l(),b=p("p"),b.textContent=B,Q=l(),y(g.$$.fragment),V=l(),y(w.$$.fragment),z=l(),C=p("p"),this.h()},l(e){const t=se("svelte-u9bgzb",document.head);i=o(t,"META",{name:!0,content:!0}),t.forEach(a),j=n(e),Z=o(e,"P",{}),P(Z).forEach(a),G=n(e),T(r.$$.fragment,e),R=n(e),m=o(e,"P",{"data-svelte-h":!0}),J(m)!=="svelte-ayxv4l"&&(m.innerHTML=H),_=n(e),u=o(e,"P",{"data-svelte-h":!0}),J(u)!=="svelte-7zl88"&&(u.innerHTML=I),W=n(e),T(d.$$.fragment,e),q=n(e),c=o(e,"P",{"data-svelte-h":!0}),J(c)!=="svelte-1jc8itq"&&(c.innerHTML=A),F=n(e),T(h.$$.fragment,e),X=n(e),f=o(e,"P",{"data-svelte-h":!0}),J(f)!=="svelte-1ti9l12"&&(f.textContent=S),x=n(e),T(M.$$.fragment,e),k=n(e),b=o(e,"P",{"data-svelte-h":!0}),J(b)!=="svelte-b8ansb"&&(b.textContent=B),Q=n(e),T(g.$$.fragment,e),V=n(e),T(w.$$.fragment,e),z=n(e),C=o(e,"P",{}),P(C).forEach(a),this.h()},h(){D(i,"name","hf:doc:metadata"),D(i,"content",oe)},m(e,t){le(document.head,i),s(e,j,t),s(e,Z,t),s(e,G,t),E(r,e,t),s(e,R,t),s(e,m,t),s(e,_,t),s(e,u,t),s(e,W,t),E(d,e,t),s(e,q,t),s(e,c,t),s(e,F,t),E(h,e,t),s(e,X,t),s(e,f,t),s(e,x,t),E(M,e,t),s(e,k,t),s(e,b,t),s(e,Q,t),E(g,e,t),s(e,V,t),E(w,e,t),s(e,z,t),s(e,C,t),N=!0},p:O,i(e){N||($(r.$$.fragment,e),$(d.$$.fragment,e),$(h.$$.fragment,e),$(M.$$.fragment,e),$(g.$$.fragment,e),$(w.$$.fragment,e),N=!0)},o(e){U(r.$$.fragment,e),U(d.$$.fragment,e),U(h.$$.fragment,e),U(M.$$.fragment,e),U(g.$$.fragment,e),U(w.$$.fragment,e),N=!1},d(e){e&&(a(j),a(Z),a(G),a(R),a(m),a(_),a(u),a(W),a(q),a(c),a(F),a(X),a(f),a(x),a(k),a(b),a(Q),a(V),a(z),a(C)),a(i),v(r,e),v(d,e),v(h,e),v(M,e),v(g,e),v(w,e)}}}const oe='{"title":"EETQ","local":"eetq","sections":[],"depth":1}';function re(L){return ee(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class he extends te{constructor(i){super(),ae(this,i,re,pe,K,{})}}export{he as component};
