import{s as I,n as q,o as Q}from"../chunks/scheduler.25b97de1.js";import{S as N,i as x,g as d,s,r as T,A as S,h as M,f as a,c as o,j as H,u as G,x as B,k as Y,y as A,a as n,v as k,d as v,t as R,w as F}from"../chunks/index.d9030fc9.js";import{C as E}from"../chunks/CodeBlock.e6cd0d95.js";import{H as L,E as P}from"../chunks/EditOnGithub.91d95064.js";function K(z){let l,f,b,w,i,Z,r,V='<a href="https://github.com/pytorch/ao" rel="nofollow">TorchAO</a> is an architecture optimization library for PyTorch, it provides high performance dtypes, optimization techniques and kernels for inference and training, featuring composability with native PyTorch features like <code>torch.compile</code>, FSDP etc.. Some benchmark numbers can be found <a href="https://github.com/pytorch/ao/tree/main?tab=readme-ov-file#without-intrusive-code-changes" rel="nofollow">here</a>',U,p,C="Before you begin, make sure the following libraries are installed with their latest version:",W,m,j,c,_,u,$='torchao quantization is implemented with tensor subclasses, currently it does not work with huggingface serialization, both the safetensor option and <a href="https://github.com/huggingface/transformers/issues/32364" rel="nofollow">non-safetensor option</a>, we’ll update here with instructions when it’s working.',X,h,g,y,J;return i=new L({props:{title:"TorchAO",local:"torchao",headingTag:"h1"}}),m=new E({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMHRvcmNoJTIwdG9yY2hhbw==",highlighted:"pip install --upgrade torch torchao",wrap:!1}}),c=new E({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRvcmNoQW9Db25maWclMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEF1dG9Ub2tlbml6ZXIlMEElMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIybWV0YS1sbGFtYSUyRk1ldGEtTGxhbWEtMy04QiUyMiUwQSUyMyUyMFdlJTIwc3VwcG9ydCUyMGludDRfd2VpZ2h0X29ubHklMkMlMjBpbnQ4X3dlaWdodF9vbmx5JTIwYW5kJTIwaW50OF9keW5hbWljX2FjdGl2YXRpb25faW50OF93ZWlnaHQlMEElMjMlMjBNb3JlJTIwZXhhbXBsZXMlMjBhbmQlMjBkb2N1bWVudGF0aW9ucyUyMGZvciUyMGFyZ3VtZW50cyUyMGNhbiUyMGJlJTIwZm91bmQlMjBpbiUyMGh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRnB5dG9yY2glMkZhbyUyRnRyZWUlMkZtYWluJTJGdG9yY2hhbyUyRnF1YW50aXphdGlvbiUyM290aGVyLWF2YWlsYWJsZS1xdWFudGl6YXRpb24tdGVjaG5pcXVlcyUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBUb3JjaEFvQ29uZmlnKCUyMmludDRfd2VpZ2h0X29ubHklMjIlMkMlMjBncm91cF9zaXplJTNEMTI4KSUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZChtb2RlbF9uYW1lJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyklMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9uYW1lKSUwQWlucHV0X3RleHQlMjAlM0QlMjAlMjJXaGF0JTIwYXJlJTIwd2UlMjBoYXZpbmclMjBmb3IlMjBkaW5uZXIlM0YlMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTIzJTIwY29tcGlsZSUyMHRoZSUyMHF1YW50aXplZCUyMG1vZGVsJTIwdG8lMjBnZXQlMjBzcGVlZHVwJTBBaW1wb3J0JTIwdG9yY2hhbyUwQXRvcmNoYW8ucXVhbnRpemF0aW9uLnV0aWxzLnJlY29tbWVuZGVkX2luZHVjdG9yX2NvbmZpZ19zZXR0ZXIoKSUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMHRvcmNoLmNvbXBpbGUocXVhbnRpemVkX21vZGVsJTJDJTIwbW9kZSUzRCUyMm1heC1hdXRvdHVuZSUyMiklMEElMEFvdXRwdXQlMjAlM0QlMjBxdWFudGl6ZWRfbW9kZWwuZ2VuZXJhdGUoKippbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0JTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer

model_name = <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B&quot;</span>
<span class="hljs-comment"># We support int4_weight_only, int8_weight_only and int8_dynamic_activation_int8_weight</span>
<span class="hljs-comment"># More examples and documentations for arguments can be found in https://github.com/pytorch/ao/tree/main/torchao/quantization#other-available-quantization-techniques</span>
quantization_config = TorchAoConfig(<span class="hljs-string">&quot;int4_weight_only&quot;</span>, group_size=<span class="hljs-number">128</span>)
quantized_model = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=quantization_config)

tokenizer = AutoTokenizer.from_pretrained(model_name)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># compile the quantized model to get speedup</span>
<span class="hljs-keyword">import</span> torchao
torchao.quantization.utils.recommended_inductor_config_setter()
quantized_model = torch.<span class="hljs-built_in">compile</span>(quantized_model, mode=<span class="hljs-string">&quot;max-autotune&quot;</span>)

output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),h=new P({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/torchao.md"}}),{c(){l=d("meta"),f=s(),b=d("p"),w=s(),T(i.$$.fragment),Z=s(),r=d("p"),r.innerHTML=V,U=s(),p=d("p"),p.textContent=C,W=s(),T(m.$$.fragment),j=s(),T(c.$$.fragment),_=s(),u=d("p"),u.innerHTML=$,X=s(),T(h.$$.fragment),g=s(),y=d("p"),this.h()},l(t){const e=S("svelte-u9bgzb",document.head);l=M(e,"META",{name:!0,content:!0}),e.forEach(a),f=o(t),b=M(t,"P",{}),H(b).forEach(a),w=o(t),G(i.$$.fragment,t),Z=o(t),r=M(t,"P",{"data-svelte-h":!0}),B(r)!=="svelte-f2tpfl"&&(r.innerHTML=V),U=o(t),p=M(t,"P",{"data-svelte-h":!0}),B(p)!=="svelte-1wrsrbm"&&(p.textContent=C),W=o(t),G(m.$$.fragment,t),j=o(t),G(c.$$.fragment,t),_=o(t),u=M(t,"P",{"data-svelte-h":!0}),B(u)!=="svelte-78510w"&&(u.innerHTML=$),X=o(t),G(h.$$.fragment,t),g=o(t),y=M(t,"P",{}),H(y).forEach(a),this.h()},h(){Y(l,"name","hf:doc:metadata"),Y(l,"content",D)},m(t,e){A(document.head,l),n(t,f,e),n(t,b,e),n(t,w,e),k(i,t,e),n(t,Z,e),n(t,r,e),n(t,U,e),n(t,p,e),n(t,W,e),k(m,t,e),n(t,j,e),k(c,t,e),n(t,_,e),n(t,u,e),n(t,X,e),k(h,t,e),n(t,g,e),n(t,y,e),J=!0},p:q,i(t){J||(v(i.$$.fragment,t),v(m.$$.fragment,t),v(c.$$.fragment,t),v(h.$$.fragment,t),J=!0)},o(t){R(i.$$.fragment,t),R(m.$$.fragment,t),R(c.$$.fragment,t),R(h.$$.fragment,t),J=!1},d(t){t&&(a(f),a(b),a(w),a(Z),a(r),a(U),a(p),a(W),a(j),a(_),a(u),a(X),a(g),a(y)),a(l),F(i,t),F(m,t),F(c,t),F(h,t)}}}const D='{"title":"TorchAO","local":"torchao","sections":[],"depth":1}';function O(z){return Q(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class lt extends N{constructor(l){super(),x(this,l,O,K,I,{})}}export{lt as component};
