import{s as ui,o as di,n as Q}from"../chunks/scheduler.9bc65507.js";import{S as fi,i as hi,g as r,s,r as d,A as ci,h as p,f as l,c as i,j as mi,u as f,x as m,k as et,y as gi,a,v as h,d as c,t as g,w as b}from"../chunks/index.707bf1b6.js";import{T as W}from"../chunks/Tip.c2ecdbf4.js";import{C as U}from"../chunks/CodeBlock.54a9f38d.js";import{H as G}from"../chunks/Heading.342b1fa6.js";import{H as Qn,a as ol}from"../chunks/HfOption.6d864328.js";function bi($){let n,M='Interested in adding a new quantization method to Transformers? Read the <a href="./hf_quantizer">HfQuantizer</a> guide to learn how!';return{c(){n=r("p"),n.innerHTML=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-px0yfi"&&(n.innerHTML=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function yi($){let n,M='Try Quanto + transformers with this <a href="https://colab.research.google.com/drive/16CXfVmtdQvciSh9BopZUDYcmXCDpvgrT?usp=sharing" rel="nofollow">notebook</a>!';return{c(){n=r("p"),n.innerHTML=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1m6vmvl"&&(n.innerHTML=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function Mi($){let n,M='Try AWQ quantization with this <a href="https://colab.research.google.com/drive/1HzZH89yAXJaZgwJDhQj9LqSBux932BvY" rel="nofollow">notebook</a>!';return{c(){n=r("p"),n.innerHTML=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1yzcfmc"&&(n.innerHTML=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function Ti($){let n,M="Fused modules cannot be combined with other optimization techniques such as FlashAttention-2.";return{c(){n=r("p"),n.textContent=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-hwgpn3"&&(n.textContent=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function wi($){let n,M='To enable fused modules for supported architectures, create an <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.AwqConfig">AwqConfig</a> and set the parameters <code>fuse_max_seq_len</code> and <code>do_fuse=True</code>. The <code>fuse_max_seq_len</code> parameter is the total sequence length and it should include the context length and the expected generation length. You can set it to a larger value to be safe.',o,y,u='For example, to fuse the AWQ modules of the <a href="https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-AWQ" rel="nofollow">TheBloke/Mistral-7B-OpenOrca-AWQ</a> model.',T,_,j;return _=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXdxQ29uZmlnJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMlRoZUJsb2tlJTJGTWlzdHJhbC03Qi1PcGVuT3JjYS1BV1ElMjIlMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQXdxQ29uZmlnKCUwQSUyMCUyMCUyMCUyMGJpdHMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwZnVzZV9tYXhfc2VxX2xlbiUzRDUxMiUyQyUwQSUyMCUyMCUyMCUyMGRvX2Z1c2UlM0RUcnVlJTJDJTBBKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcpLnRvKDAp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AwqConfig, AutoModelForCausalLM

model_id = <span class="hljs-string">&quot;TheBloke/Mistral-7B-OpenOrca-AWQ&quot;</span>

quantization_config = AwqConfig(
    bits=<span class="hljs-number">4</span>,
    fuse_max_seq_len=<span class="hljs-number">512</span>,
    do_fuse=<span class="hljs-literal">True</span>,
)

model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=quantization_config).to(<span class="hljs-number">0</span>)`,wrap:!1}}),{c(){n=r("p"),n.innerHTML=M,o=s(),y=r("p"),y.innerHTML=u,T=s(),d(_.$$.fragment)},l(w){n=p(w,"P",{"data-svelte-h":!0}),m(n)!=="svelte-y19lx3"&&(n.innerHTML=M),o=i(w),y=p(w,"P",{"data-svelte-h":!0}),m(y)!=="svelte-1zxz2i"&&(y.innerHTML=u),T=i(w),f(_.$$.fragment,w)},m(w,q){a(w,n,q),a(w,o,q),a(w,y,q),a(w,T,q),h(_,w,q),j=!0},p:Q,i(w){j||(c(_.$$.fragment,w),j=!0)},o(w){g(_.$$.fragment,w),j=!1},d(w){w&&(l(n),l(o),l(y),l(T)),b(_,w)}}}function $i($){let n,M='For architectures that don’t support fused modules yet, you need to create a custom fusing mapping to define which modules need to be fused with the <code>modules_to_fuse</code> parameter. For example, to fuse the AWQ modules of the <a href="https://huggingface.co/TheBloke/Yi-34B-AWQ" rel="nofollow">TheBloke/Yi-34B-AWQ</a> model.',o,y,u,T,_="The parameter <code>modules_to_fuse</code> should include:",j,w,q="<li><code>&quot;attention&quot;</code>: The names of the attention layers to fuse in the following order: query, key, value and output projection layer. If you don’t want to fuse these layers, pass an empty list.</li> <li><code>&quot;layernorm&quot;</code>: The names of all the LayerNorm layers you want to replace with a custom fused LayerNorm. If you don’t want to fuse these layers, pass an empty list.</li> <li><code>&quot;mlp&quot;</code>: The names of the MLP layers you want to fuse into a single MLP layer in the order: (gate (dense, layer, post-attention) / up / down layers).</li> <li><code>&quot;use_alibi&quot;</code>: If your model uses ALiBi positional embedding.</li> <li><code>&quot;num_attention_heads&quot;</code>: The number of attention heads.</li> <li><code>&quot;num_key_value_heads&quot;</code>: The number of key value heads that should be used to implement Grouped Query Attention (GQA). If <code>num_key_value_heads=num_attention_heads</code>, the model will use Multi Head Attention (MHA), if <code>num_key_value_heads=1</code> the model will use Multi Query Attention (MQA), otherwise GQA is used.</li> <li><code>&quot;hidden_size&quot;</code>: The dimension of the hidden representations.</li>",Z;return y=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXdxQ29uZmlnJTJDJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMlRoZUJsb2tlJTJGWWktMzRCLUFXUSUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBBd3FDb25maWcoJTBBJTIwJTIwJTIwJTIwYml0cyUzRDQlMkMlMEElMjAlMjAlMjAlMjBmdXNlX21heF9zZXFfbGVuJTNENTEyJTJDJTBBJTIwJTIwJTIwJTIwbW9kdWxlc190b19mdXNlJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYXR0ZW50aW9uJTIyJTNBJTIwJTVCJTIycV9wcm9qJTIyJTJDJTIwJTIya19wcm9qJTIyJTJDJTIwJTIydl9wcm9qJTIyJTJDJTIwJTIyb19wcm9qJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybGF5ZXJub3JtJTIyJTNBJTIwJTVCJTIybG4xJTIyJTJDJTIwJTIybG4yJTIyJTJDJTIwJTIybm9ybSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm1scCUyMiUzQSUyMCU1QiUyMmdhdGVfcHJvaiUyMiUyQyUyMCUyMnVwX3Byb2olMjIlMkMlMjAlMjJkb3duX3Byb2olMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ1c2VfYWxpYmklMjIlM0ElMjBGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm51bV9hdHRlbnRpb25faGVhZHMlMjIlM0ElMjA1NiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm51bV9rZXlfdmFsdWVfaGVhZHMlMjIlM0ElMjA4JTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyaGlkZGVuX3NpemUlMjIlM0ElMjA3MTY4JTBBJTIwJTIwJTIwJTIwJTdEJTBBKSUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcpLnRvKDAp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AwqConfig, AutoModelForCausalLM

model_id = <span class="hljs-string">&quot;TheBloke/Yi-34B-AWQ&quot;</span>

quantization_config = AwqConfig(
    bits=<span class="hljs-number">4</span>,
    fuse_max_seq_len=<span class="hljs-number">512</span>,
    modules_to_fuse={
        <span class="hljs-string">&quot;attention&quot;</span>: [<span class="hljs-string">&quot;q_proj&quot;</span>, <span class="hljs-string">&quot;k_proj&quot;</span>, <span class="hljs-string">&quot;v_proj&quot;</span>, <span class="hljs-string">&quot;o_proj&quot;</span>],
        <span class="hljs-string">&quot;layernorm&quot;</span>: [<span class="hljs-string">&quot;ln1&quot;</span>, <span class="hljs-string">&quot;ln2&quot;</span>, <span class="hljs-string">&quot;norm&quot;</span>],
        <span class="hljs-string">&quot;mlp&quot;</span>: [<span class="hljs-string">&quot;gate_proj&quot;</span>, <span class="hljs-string">&quot;up_proj&quot;</span>, <span class="hljs-string">&quot;down_proj&quot;</span>],
        <span class="hljs-string">&quot;use_alibi&quot;</span>: <span class="hljs-literal">False</span>,
        <span class="hljs-string">&quot;num_attention_heads&quot;</span>: <span class="hljs-number">56</span>,
        <span class="hljs-string">&quot;num_key_value_heads&quot;</span>: <span class="hljs-number">8</span>,
        <span class="hljs-string">&quot;hidden_size&quot;</span>: <span class="hljs-number">7168</span>
    }
)

model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=quantization_config).to(<span class="hljs-number">0</span>)`,wrap:!1}}),{c(){n=r("p"),n.innerHTML=M,o=s(),d(y.$$.fragment),u=s(),T=r("p"),T.innerHTML=_,j=s(),w=r("ul"),w.innerHTML=q},l(C){n=p(C,"P",{"data-svelte-h":!0}),m(n)!=="svelte-ecilwg"&&(n.innerHTML=M),o=i(C),f(y.$$.fragment,C),u=i(C),T=p(C,"P",{"data-svelte-h":!0}),m(T)!=="svelte-1divolh"&&(T.innerHTML=_),j=i(C),w=p(C,"UL",{"data-svelte-h":!0}),m(w)!=="svelte-ln3msn"&&(w.innerHTML=q)},m(C,k){a(C,n,k),a(C,o,k),h(y,C,k),a(C,u,k),a(C,T,k),a(C,j,k),a(C,w,k),Z=!0},p:Q,i(C){Z||(c(y.$$.fragment,C),Z=!0)},o(C){g(y.$$.fragment,C),Z=!1},d(C){C&&(l(n),l(o),l(u),l(T),l(j),l(w)),b(y,C)}}}function Ji($){let n,M,o,y;return n=new ol({props:{id:"fuse",option:"supported architectures",$$slots:{default:[wi]},$$scope:{ctx:$}}}),o=new ol({props:{id:"fuse",option:"unsupported architectures",$$slots:{default:[$i]},$$scope:{ctx:$}}}),{c(){d(n.$$.fragment),M=s(),d(o.$$.fragment)},l(u){f(n.$$.fragment,u),M=i(u),f(o.$$.fragment,u)},m(u,T){h(n,u,T),a(u,M,T),h(o,u,T),y=!0},p(u,T){const _={};T&2&&(_.$$scope={dirty:T,ctx:u}),n.$set(_);const j={};T&2&&(j.$$scope={dirty:T,ctx:u}),o.$set(j)},i(u){y||(c(n.$$.fragment,u),c(o.$$.fragment,u),y=!0)},o(u){g(n.$$.fragment,u),g(o.$$.fragment,u),y=!1},d(u){u&&l(M),b(n,u),b(o,u)}}}function vi($){let n,M="Note this feature is supported on AMD GPUs.";return{c(){n=r("p"),n.textContent=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-147si7w"&&(n.textContent=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function Ui($){let n,M='Try GPTQ quantization with PEFT in this <a href="https://colab.research.google.com/drive/1_TIrmuKOFhuRRiTWN94iLKUFu6ZX4ceb?usp=sharing" rel="nofollow">notebook</a> and learn more about it’s details in this <a href="https://huggingface.co/blog/gptq-integration" rel="nofollow">blog post</a>!';return{c(){n=r("p"),n.innerHTML=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1tfpiad"&&(n.innerHTML=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function _i($){let n,M='Depending on your hardware, it can take some time to quantize a model from scratch. It can take ~5 minutes to quantize the <a href="https://huggingface.co/facebook/opt-350m" rel="nofollow">facebook/opt-350m</a> model on a free-tier Google Colab GPU, but it’ll take ~4 hours to quantize a 175B parameter model on a NVIDIA A100. Before you quantize a model, it is a good idea to check the Hub if a GPTQ-quantized version of the model already exists.';return{c(){n=r("p"),n.innerHTML=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-jdzahk"&&(n.innerHTML=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function Ci($){let n,M="Only 4-bit models are supported, and we recommend deactivating the ExLlama kernels if you’re finetuning a quantized model with PEFT.";return{c(){n=r("p"),n.textContent=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-14ppi1y"&&(n.textContent=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function ji($){let n,M;return n=new U({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlM0UwLjM3LjA=",highlighted:"pip install transformers accelerate bitsandbytes&gt;0.37.0",wrap:!1}}),{c(){d(n.$$.fragment)},l(o){f(n.$$.fragment,o)},m(o,y){h(n,o,y),M=!0},p:Q,i(o){M||(c(n.$$.fragment,o),M=!0)},o(o){g(n.$$.fragment,o),M=!1},d(o){b(n,o)}}}function qi($){let n,M;return n=new U({props:{code:"cGlwJTIwaW5zdGFsbCUyMGJpdHNhbmRieXRlcyUzRSUzRDAuMzkuMCUwQXBpcCUyMGluc3RhbGwlMjAtLXVwZ3JhZGUlMjBhY2NlbGVyYXRlJTBBcGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMHRyYW5zZm9ybWVycw==",highlighted:`pip install bitsandbytes&gt;=0.39.0
pip install --upgrade accelerate
pip install --upgrade transformers`,wrap:!1}}),{c(){d(n.$$.fragment)},l(o){f(n.$$.fragment,o)},m(o,y){h(n,o,y),M=!0},p:Q,i(o){M||(c(n.$$.fragment,o),M=!0)},o(o){g(n.$$.fragment,o),M=!1},d(o){b(n,o)}}}function ki($){let n,M,o,y;return n=new ol({props:{id:"bnb",option:"8-bit",$$slots:{default:[ji]},$$scope:{ctx:$}}}),o=new ol({props:{id:"bnb",option:"4-bit",$$slots:{default:[qi]},$$scope:{ctx:$}}}),{c(){d(n.$$.fragment),M=s(),d(o.$$.fragment)},l(u){f(n.$$.fragment,u),M=i(u),f(o.$$.fragment,u)},m(u,T){h(n,u,T),a(u,M,T),h(o,u,T),y=!0},p(u,T){const _={};T&2&&(_.$$scope={dirty:T,ctx:u}),n.$set(_);const j={};T&2&&(j.$$scope={dirty:T,ctx:u}),o.$set(j)},i(u){y||(c(n.$$.fragment,u),c(o.$$.fragment,u),y=!0)},o(u){g(n.$$.fragment,u),g(o.$$.fragment,u),y=!1},d(u){u&&l(M),b(n,u),b(o,u)}}}function Zi($){let n,M="Quantizing a model in 8-bit halves the memory-usage, and for large models, set <code>device_map=&quot;auto&quot;</code> to efficiently use the GPUs available:",o,y,u,T,_="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",j,w,q,Z,C='Once a model is quantized to 8-bit, you can’t push the quantized weights to the Hub unless you’re using the latest version of Transformers and bitsandbytes. If you have the latest versions, then you can push the 8-bit model to the Hub with the <a href="/docs/transformers/v4.39.2/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.push_to_hub">push_to_hub()</a> method. The quantization config.json file is pushed first, followed by the quantized model weights.',k,J,z;return y=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_8bit = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)`,wrap:!1}}),w=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTM1MG0lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDMyKSUwQW1vZGVsXzhiaXQubW9kZWwuZGVjb2Rlci5sYXllcnMlNUItMSU1RC5maW5hbF9sYXllcl9ub3JtLndlaWdodC5kdHlwZQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_8bit = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>, torch_dtype=torch.float32)
model_8bit.model.decoder.layers[-<span class="hljs-number">1</span>].final_layer_norm.weight.dtype`,wrap:!1}}),J=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmJpZ3NjaWVuY2UlMkZibG9vbS01NjBtJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzhiaXQlM0RUcnVlKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmJpZ3NjaWVuY2UlMkZibG9vbS01NjBtJTIyKSUwQSUwQW1vZGVsLnB1c2hfdG9faHViKCUyMmJsb29tLTU2MG0tOGJpdCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_8bit=<span class="hljs-literal">True</span>)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>)

model.push_to_hub(<span class="hljs-string">&quot;bloom-560m-8bit&quot;</span>)`,wrap:!1}}),{c(){n=r("p"),n.innerHTML=M,o=s(),d(y.$$.fragment),u=s(),T=r("p"),T.innerHTML=_,j=s(),d(w.$$.fragment),q=s(),Z=r("p"),Z.innerHTML=C,k=s(),d(J.$$.fragment)},l(v){n=p(v,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1dz8xeq"&&(n.innerHTML=M),o=i(v),f(y.$$.fragment,v),u=i(v),T=p(v,"P",{"data-svelte-h":!0}),m(T)!=="svelte-kfzkum"&&(T.innerHTML=_),j=i(v),f(w.$$.fragment,v),q=i(v),Z=p(v,"P",{"data-svelte-h":!0}),m(Z)!=="svelte-1u2w7nj"&&(Z.innerHTML=C),k=i(v),f(J.$$.fragment,v)},m(v,I){a(v,n,I),a(v,o,I),h(y,v,I),a(v,u,I),a(v,T,I),a(v,j,I),h(w,v,I),a(v,q,I),a(v,Z,I),a(v,k,I),h(J,v,I),z=!0},p:Q,i(v){z||(c(y.$$.fragment,v),c(w.$$.fragment,v),c(J.$$.fragment,v),z=!0)},o(v){g(y.$$.fragment,v),g(w.$$.fragment,v),g(J.$$.fragment,v),z=!1},d(v){v&&(l(n),l(o),l(u),l(T),l(j),l(q),l(Z),l(k)),b(y,v),b(w,v),b(J,v)}}}function Gi($){let n,M="Quantizing a model in 4-bit reduces your memory-usage by 4x, and for large models, set <code>device_map=&quot;auto&quot;</code> to efficiently use the GPUs available:",o,y,u,T,_="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",j,w,q,Z,C="If you have <code>bitsandbytes&gt;=0.41.3</code>, you can serialize 4-bit models and push them on Hugging Face Hub. Simply call <code>model.push_to_hub()</code> after loading it in 4-bit precision. You can also save the serialized 4-bit models locally with <code>model.save_pretrained()</code> command.",k;return y=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBsb2FkX2luXzRiaXQlM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_4bit = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>)`,wrap:!1}}),w=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMEElMEFtb2RlbF80Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTM1MG0lMjIlMkMlMjBsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDMyKSUwQW1vZGVsXzRiaXQubW9kZWwuZGVjb2Rlci5sYXllcnMlNUItMSU1RC5maW5hbF9sYXllcl9ub3JtLndlaWdodC5kdHlwZQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model_4bit = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, load_in_4bit=<span class="hljs-literal">True</span>, torch_dtype=torch.float32)
model_4bit.model.decoder.layers[-<span class="hljs-number">1</span>].final_layer_norm.weight.dtype`,wrap:!1}}),{c(){n=r("p"),n.innerHTML=M,o=s(),d(y.$$.fragment),u=s(),T=r("p"),T.innerHTML=_,j=s(),d(w.$$.fragment),q=s(),Z=r("p"),Z.innerHTML=C},l(J){n=p(J,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1b6jp1p"&&(n.innerHTML=M),o=i(J),f(y.$$.fragment,J),u=i(J),T=p(J,"P",{"data-svelte-h":!0}),m(T)!=="svelte-kfzkum"&&(T.innerHTML=_),j=i(J),f(w.$$.fragment,J),q=i(J),Z=p(J,"P",{"data-svelte-h":!0}),m(Z)!=="svelte-1pmx2wb"&&(Z.innerHTML=C)},m(J,z){a(J,n,z),a(J,o,z),h(y,J,z),a(J,u,z),a(J,T,z),a(J,j,z),h(w,J,z),a(J,q,z),a(J,Z,z),k=!0},p:Q,i(J){k||(c(y.$$.fragment,J),c(w.$$.fragment,J),k=!0)},o(J){g(y.$$.fragment,J),g(w.$$.fragment,J),k=!1},d(J){J&&(l(n),l(o),l(u),l(T),l(j),l(q),l(Z)),b(y,J),b(w,J)}}}function zi($){let n,M,o,y;return n=new ol({props:{id:"bnb",option:"8-bit",$$slots:{default:[Zi]},$$scope:{ctx:$}}}),o=new ol({props:{id:"bnb",option:"4-bit",$$slots:{default:[Gi]},$$scope:{ctx:$}}}),{c(){d(n.$$.fragment),M=s(),d(o.$$.fragment)},l(u){f(n.$$.fragment,u),M=i(u),f(o.$$.fragment,u)},m(u,T){h(n,u,T),a(u,M,T),h(o,u,T),y=!0},p(u,T){const _={};T&2&&(_.$$scope={dirty:T,ctx:u}),n.$set(_);const j={};T&2&&(j.$$scope={dirty:T,ctx:u}),o.$set(j)},i(u){y||(c(n.$$.fragment,u),c(o.$$.fragment,u),y=!0)},o(u){g(n.$$.fragment,u),g(o.$$.fragment,u),y=!1},d(u){u&&l(M),b(n,u),b(o,u)}}}function Ii($){let n,M="Training with 8-bit and 4-bit weights are only supported for training <em>extra</em> parameters.";return{c(){n=r("p"),n.innerHTML=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-of9sym"&&(n.innerHTML=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function Qi($){let n,M='Learn more about the details of 8-bit quantization in this <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">blog post</a>!';return{c(){n=r("p"),n.innerHTML=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1bb05fp"&&(n.innerHTML=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function Wi($){let n,M='Try 4-bit quantization in this <a href="https://colab.research.google.com/drive/1ge2F1QSK8Q7h0hn3YKuBCOAS0bK8E0wf" rel="nofollow">notebook</a> and learn more about it’s details in this <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">blog post</a>.';return{c(){n=r("p"),n.innerHTML=M},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-rme94c"&&(n.innerHTML=M)},m(o,y){a(o,n,y)},p:Q,d(o){o&&l(n)}}}function Bi($){let n,M,o,y,u,T,_,j="Quantization techniques focus on representing data with less information while also trying to not lose too much accuracy. This often means converting a data type to represent the same information with fewer bits. For example, if your model weights are stored as 32-bit floating points and they’re quantized to 16-bit floating points, this halves the model size which makes it easier to store and reduces memory-usage. Lower precision can also speedup inference because it takes less time to perform calculations with fewer bits.",w,q,Z="Transformers supports several quantization schemes to help you run inference with large language models (LLMs) and finetune adapters on quantized models. This guide will show you how to use Activation-aware Weight Quantization (AWQ), AutoGPTQ, and bitsandbytes.",C,k,J,z,v,I,pl,lt,Wn='<a href="https://github.com/huggingface/quanto" rel="nofollow">🤗 Quanto</a> library is a versatile pytorch quantization toolkit. The quantization method used is the linear quantization. Quanto provides several unique features such as:',ml,at,Bn="<li>weights quantization (<code>float8</code>,<code>int8</code>,<code>int4</code>,<code>int2</code>)</li> <li>activation quantization (<code>float8</code>,<code>int8</code>)</li> <li>modality agnostic (e.g CV,LLM)</li> <li>device agnostic (e.g CUDA,MPS,CPU)</li> <li>compatibility with <code>torch.compile</code></li> <li>easy to add custom kernel for specific device</li> <li>supports quantization aware training</li>",ul,nt,Vn="Before you begin, make sure the following libraries are installed:",dl,st,fl,it,Xn='Now you can quantize a model by passing <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.QuantoConfig">QuantoConfig</a> object in the <a href="/docs/transformers/v4.39.2/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method. This works for any model in any modality, as long as it contains <code>torch.nn.Linear</code> layers.',hl,ot,Fn='The integration with transformers only supports weights quantization. For the more complex use case such as activation quantization, calibration and quantization aware training, you should use <a href="https://github.com/huggingface/quanto" rel="nofollow">quanto</a> library instead.',cl,rt,gl,pt,Rn="Note that serialization is not supported yet with transformers but it is coming soon! If you want to save the model, you can use quanto library instead.",bl,mt,xn='Quanto library uses linear quantization algorithm for quantization. Even though this is a basic quantization technique, we get very good results! Have a look at the following becnhmark (llama-2-7b on perplexity metric). You can find more benchamarks <a href="https://github.com/huggingface/quanto/tree/main/bench/generation" rel="nofollow">here</a>',yl,B,Ln='<div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/NousResearch-Llama-2-7b-hf_Perplexity.png" alt="llama-2-7b-quanto-perplexity"/></div>',Ml,ut,Nn="The library is versatible enough to be compatible with most PTQ optimization algorithms. The plan in the future is to integrate the most popular algorithms in the most seamless possible way (AWQ, Smoothquant).",Tl,dt,wl,ft,Hn='Try AQLM on <a href="https://colab.research.google.com/drive/1-xZmBRXT5Fm3Ghn4Mwa2KRypORXb855X?usp=sharing" rel="nofollow">Google Colab</a>!',$l,ht,An='Additive Quantization of Language Models (<a href="https://arxiv.org/abs/2401.06118" rel="nofollow">AQLM</a>) is a Large Language Models compression method. It quantizes multiple weights together and take advantage of interdependencies between them. AQLM represents groups of 8-16 weights as a sum of multiple vector codes.',Jl,ct,En="Inference support for AQLM is realised in the <code>aqlm</code> library. Make sure to install it to run the models (note aqlm works only with python&gt;=3.10):",vl,gt,Ul,bt,Yn="The library provides efficient kernels for both GPU and CPU inference and training.",_l,yt,Pn='The instructions on how to quantize models yourself, as well as all the relevant code can be found in the corresponding GitHub <a href="https://github.com/Vahe1994/AQLM" rel="nofollow">repository</a>.',Cl,Mt,jl,Tt,Sn='Starting with version <code>aqlm 1.0.2</code>, AQLM supports Parameter-Efficient Fine-Tuning in a form of <a href="https://huggingface.co/docs/peft/package_reference/lora" rel="nofollow">LoRA</a> integrated into the <a href="https://huggingface.co/blog/peft" rel="nofollow">PEFT</a> library.',ql,wt,kl,$t,Dn="AQLM quantization setups vary mainly on the number of codebooks used as well as codebook sizes in bits. The most popular setups, as well as inference kernels they support are:",Zl,Jt,Kn="<thead><tr><th>Kernel</th> <th>Number of codebooks</th> <th>Codebook size, bits</th> <th>Notation</th> <th>Accuracy</th> <th>Speedup</th> <th>Fast GPU inference</th> <th>Fast CPU inference</th></tr></thead> <tbody><tr><td>Triton</td> <td>K</td> <td>N</td> <td>KxN</td> <td>-</td> <td>Up to ~0.7x</td> <td>✅</td> <td>❌</td></tr> <tr><td>CUDA</td> <td>1</td> <td>16</td> <td>1x16</td> <td>Best</td> <td>Up to ~1.3x</td> <td>✅</td> <td>❌</td></tr> <tr><td>CUDA</td> <td>2</td> <td>8</td> <td>2x8</td> <td>OK</td> <td>Up to ~3.0x</td> <td>✅</td> <td>❌</td></tr> <tr><td>Numba</td> <td>K</td> <td>8</td> <td>Kx8</td> <td>Good</td> <td>Up to ~4.0x</td> <td>❌</td> <td>✅</td></tr></tbody>",Gl,vt,zl,V,Il,Ut,On='<a href="https://hf.co/papers/2306.00978" rel="nofollow">Activation-aware Weight Quantization (AWQ)</a> doesn’t quantize all the weights in a model, and instead, it preserves a small percentage of weights that are important for LLM performance. This significantly reduces quantization loss such that you can run models in 4-bit precision without experiencing any performance degradation.',Ql,_t,ts='There are several libraries for quantizing models with the AWQ algorithm, such as <a href="https://github.com/mit-han-lab/llm-awq" rel="nofollow">llm-awq</a>, <a href="https://github.com/casper-hansen/AutoAWQ" rel="nofollow">autoawq</a> or <a href="https://huggingface.co/docs/optimum/main/en/intel/optimization_inc" rel="nofollow">optimum-intel</a>. Transformers supports loading models quantized with the llm-awq and autoawq libraries. This guide will show you how to load models quantized with autoawq, but the process is similar for llm-awq quantized models.',Wl,Ct,es="Make sure you have autoawq installed:",Bl,jt,Vl,qt,ls='AWQ-quantized models can be identified by checking the <code>quantization_config</code> attribute in the model’s <a href="https://huggingface.co/TheBloke/zephyr-7B-alpha-AWQ/blob/main/config.json" rel="nofollow">config.json</a> file:',Xl,kt,Fl,Zt,as='A quantized model is loaded with the <a href="/docs/transformers/v4.39.2/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method. If you loaded your model on the CPU, make sure to move it to a GPU device first. Use the <code>device_map</code> parameter to specify where to place the model:',Rl,Gt,xl,zt,ns="Loading an AWQ-quantized model automatically sets other weights to fp16 by default for performance reasons. If you want to load these other weights in a different format, use the <code>torch_dtype</code> parameter:",Ll,It,Nl,Qt,ss='AWQ quantization can also be combined with <a href="perf_infer_gpu_one#flashattention-2">FlashAttention-2</a> to further accelerate inference:',Hl,Wt,Al,Bt,El,Vt,is='Fused modules offers improved accuracy and performance and it is supported out-of-the-box for AWQ modules for <a href="https://huggingface.co/meta-llama" rel="nofollow">Llama</a> and <a href="https://huggingface.co/mistralai/Mistral-7B-v0.1" rel="nofollow">Mistral</a> architectures, but you can also fuse AWQ modules for unsupported architectures.',Yl,X,Pl,F,Sl,Xt,Dl,Ft,os="Recent versions of <code>autoawq</code> supports exllama-v2 kernels for faster prefill and decoding. To get started, first install the latest version of <code>autoawq</code> by running:",Kl,Rt,Ol,xt,rs="Get started by passing an <code>AwqConfig()</code> with <code>version=&quot;exllama&quot;</code>.",ta,Lt,ea,R,la,Nt,aa,x,na,Ht,ps='The <a href="https://github.com/PanQiWei/AutoGPTQ" rel="nofollow">AutoGPTQ</a> library implements the GPTQ algorithm, a post-training quantization technique where each row of the weight matrix is quantized independently to find a version of the weights that minimizes the error. These weights are quantized to int4, but they’re restored to fp16 on the fly during inference. This can save your memory-usage by 4x because the int4 weights are dequantized in a fused kernel rather than a GPU’s global memory, and you can also expect a speedup in inference because using a lower bitwidth takes less time to communicate.',sa,At,ms="Before you begin, make sure the following libraries are installed:",ia,Et,oa,Yt,us='To quantize a model (currently only supported for text models), you need to create a <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.GPTQConfig">GPTQConfig</a> class and set the number of bits to quantize to, a dataset to calibrate the weights for quantization, and a tokenizer to prepare the dataset.',ra,Pt,pa,St,ds="You could also pass your own dataset as a list of strings, but it is highly recommended to use the same dataset from the GPTQ paper.",ma,Dt,ua,Kt,fs='Load a model to quantize and pass the <code>gptq_config</code> to the <a href="/docs/transformers/v4.39.2/en/model_doc/auto#transformers.FlaxAutoModelForVision2Seq.from_pretrained">from_pretrained()</a> method. Set <code>device_map=&quot;auto&quot;</code> to automatically offload the model to a CPU to help fit the model in memory, and allow the model modules to be moved between the CPU and GPU for quantization.',da,Ot,fa,te,hs="If you’re running out of memory because a dataset is too large, disk offloading is not supported. If this is the case, try passing the <code>max_memory</code> parameter to allocate the amount of memory to use on your device (GPU and CPU):",ha,ee,ca,L,ga,le,cs='Once your model is quantized, you can push the model and tokenizer to the Hub where it can be easily shared and accessed. Use the <a href="/docs/transformers/v4.39.2/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.push_to_hub">push_to_hub()</a> method to save the <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.GPTQConfig">GPTQConfig</a>:',ba,ae,ya,ne,gs='You could also save your quantized model locally with the <a href="/docs/transformers/v4.39.2/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a> method. If the model was quantized with the <code>device_map</code> parameter, make sure to move the entire model to a GPU or CPU before saving it. For example, to save the model on a CPU:',Ma,se,Ta,ie,bs='Reload a quantized model with the <a href="/docs/transformers/v4.39.2/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method, and set <code>device_map=&quot;auto&quot;</code> to automatically distribute the model on all available GPUs to load the model faster without using more memory than needed.',wa,oe,$a,re,Ja,pe,ys='<a href="https://github.com/turboderp/exllama" rel="nofollow">ExLlama</a> is a Python/C++/CUDA implementation of the <a href="model_doc/llama">Llama</a> model that is designed for faster inference with 4-bit GPTQ weights (check out these <a href="https://github.com/huggingface/optimum/tree/main/tests/benchmark#gptq-benchmark" rel="nofollow">benchmarks</a>). The ExLlama kernel is activated by default when you create a <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.GPTQConfig">GPTQConfig</a> object. To boost inference speed even further, use the <a href="https://github.com/turboderp/exllamav2" rel="nofollow">ExLlamaV2</a> kernels by configuring the <code>exllama_config</code> parameter:',va,me,Ua,N,_a,ue,Ms="The ExLlama kernels are only supported when the entire model is on the GPU. If you’re doing inference on a CPU with AutoGPTQ (version > 0.4.2), then you’ll need to disable the ExLlama kernel. This overwrites the attributes related to the ExLlama kernels in the quantization config of the config.json file.",Ca,de,ja,fe,qa,he,Ts='<a href="https://github.com/TimDettmers/bitsandbytes" rel="nofollow">bitsandbytes</a> is the easiest option for quantizing a model to 8 and 4-bit. 8-bit quantization multiplies outliers in fp16 with non-outliers in int8, converts the non-outlier values back to fp16, and then adds them together to return the weights in fp16. This reduces the degradative effect outlier values have on a model’s performance. 4-bit quantization compresses a model even further, and it is commonly used with <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> to finetune quantized LLMs.',ka,ce,ws="To use bitsandbytes, make sure you have the following libraries installed:",Za,H,Ga,ge,$s='Now you can quantize a model with the <code>load_in_8bit</code> or <code>load_in_4bit</code> parameters in the <a href="/docs/transformers/v4.39.2/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method. This works for any model in any modality, as long as it supports loading with Accelerate and contains <code>torch.nn.Linear</code> layers.',za,A,Ia,E,Qa,be,Js="You can check your memory footprint with the <code>get_memory_footprint</code> method:",Wa,ye,Ba,Me,vs='Quantized models can be loaded from the <a href="/docs/transformers/v4.39.2/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method without needing to specify the <code>load_in_8bit</code> or <code>load_in_4bit</code> parameters:',Va,Te,Xa,we,Fa,Y,Ra,$e,Us="This section explores some of the specific features of 8-bit models, such as offloading, outlier thresholds, skipping module conversion, and finetuning.",xa,Je,La,ve,_s='8-bit models can offload weights between the CPU and GPU to support fitting very large models into memory. The weights dispatched to the CPU are actually stored in <strong>float32</strong>, and aren’t converted to 8-bit. For example, to enable offloading for the <a href="https://huggingface.co/bigscience/bloom-1b7" rel="nofollow">bigscience/bloom-1b7</a> model, start by creating a <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Na,Ue,Ha,_e,Cs="Design a custom device map to fit everything on your GPU except for the <code>lm_head</code>, which you’ll dispatch to the CPU:",Aa,Ce,Ea,je,js="Now load your model with the custom <code>device_map</code> and <code>quantization_config</code>:",Ya,qe,Pa,ke,Sa,Ze,qs="An “outlier” is a hidden state value greater than a certain threshold, and these values are computed in fp16. While the values are usually normally distributed ([-3.5, 3.5]), this distribution can be very different for large models ([-60, 6] or [6, 60]). 8-bit quantization works well for values ~5, but beyond that, there is a significant performance penalty. A good default threshold value is 6, but a lower threshold may be needed for more unstable models (small models or finetuning).",Da,Ge,ks='To find the best threshold for your model, we recommend experimenting with the <code>llm_int8_threshold</code> parameter in <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Ka,ze,Oa,Ie,tn,Qe,Zs='For some models, like <a href="model_doc/jukebox">Jukebox</a>, you don’t need to quantize every module to 8-bit which can actually cause instability. With Jukebox, there are several <code>lm_head</code> modules that should be skipped using the <code>llm_int8_skip_modules</code> parameter in <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',en,We,ln,Be,an,Ve,Gs='With the <a href="https://github.com/huggingface/peft" rel="nofollow">PEFT</a> library, you can finetune large models like <a href="https://huggingface.co/google/flan-t5-large" rel="nofollow">flan-t5-large</a> and <a href="https://huggingface.co/facebook/opt-6.7b" rel="nofollow">facebook/opt-6.7b</a> with 8-bit quantization. You don’t need to pass the <code>device_map</code> parameter for training because it’ll automatically load your model on a GPU. However, you can still customize the device map with the <code>device_map</code> parameter if you want to (<code>device_map=&quot;auto&quot;</code> should only be used for inference).',nn,Xe,sn,P,on,Fe,zs="This section explores some of the specific features of 4-bit models, such as changing the compute data type, using the Normal Float 4 (NF4) data type, and using nested quantization.",rn,Re,pn,xe,Is='To speedup computation, you can change the data type from float32 (the default value) to bf16 using the <code>bnb_4bit_compute_dtype</code> parameter in <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',mn,Le,un,Ne,dn,He,Qs='NF4 is a 4-bit data type from the <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> paper, adapted for weights initialized from a normal distribution. You should use NF4 for training 4-bit base models. This can be configured with the <code>bnb_4bit_quant_type</code> parameter in the <a href="/docs/transformers/v4.39.2/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',fn,Ae,hn,Ee,Ws="For inference, the <code>bnb_4bit_quant_type</code> does not have a huge impact on performance. However, to remain consistent with the model weights, you should use the <code>bnb_4bit_compute_dtype</code> and <code>torch_dtype</code> values.",cn,Ye,gn,Pe,Bs='Nested quantization is a technique that can save additional memory at no additional performance cost. This feature performs a second quantization of the already quantized weights to save an addition 0.4 bits/parameter. For example, with nested quantization, you can finetune a <a href="https://huggingface.co/meta-llama/Llama-2-13b" rel="nofollow">Llama-13b</a> model on a 16GB NVIDIA T4 GPU with a sequence length of 1024, a batch size of 1, and enabling gradient accumulation with 4 steps.',bn,Se,yn,De,Mn,Ke,Vs='The <a href="https://huggingface.co/docs/optimum/index" rel="nofollow">Optimum</a> library supports quantization for Intel, Furiosa, ONNX Runtime, GPTQ, and lower-level PyTorch quantization functions. Consider using Optimum for quantization if you’re using specific and optimized hardware like Intel CPUs, Furiosa NPUs or a model accelerator like ONNX Runtime.',Tn,Oe,wn,tl,Xs='To compare the speed, throughput, and latency of each quantization scheme, check the following benchmarks obtained from the <a href="https://github.com/huggingface/optimum-benchmark" rel="nofollow">optimum-benchmark</a> library. The benchmark was run on a NVIDIA A1000 for the <a href="https://huggingface.co/TheBloke/Mistral-7B-v0.1-AWQ" rel="nofollow">TheBloke/Mistral-7B-v0.1-AWQ</a> and <a href="https://huggingface.co/TheBloke/Mistral-7B-v0.1-GPTQ" rel="nofollow">TheBloke/Mistral-7B-v0.1-GPTQ</a> models. These were also tested against the bitsandbytes quantization methods as well as a native fp16 model.',$n,S,Fs='<div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/forward_memory_plot.png" alt="forward peak memory per batch size"/> <figcaption class="mt-2 text-center text-sm text-gray-500">forward peak memory/batch size</figcaption></div> <div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/generate_memory_plot.png" alt="generate peak memory per batch size"/> <figcaption class="mt-2 text-center text-sm text-gray-500">generate peak memory/batch size</figcaption></div>',Jn,D,Rs='<div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/generate_throughput_plot.png" alt="generate throughput per batch size"/> <figcaption class="mt-2 text-center text-sm text-gray-500">generate throughput/batch size</figcaption></div> <div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/forward_latency_plot.png" alt="forward latency per batch size"/> <figcaption class="mt-2 text-center text-sm text-gray-500">forward latency/batch size</figcaption></div>',vn,el,xs='The benchmarks indicate AWQ quantization is the fastest for inference, text generation, and has the lowest peak memory for text generation. However, AWQ has the largest forward latency per batch size. For a more detailed discussion about the pros and cons of each quantization method, read the <a href="https://huggingface.co/blog/overview-quantization-transformers" rel="nofollow">Overview of natively supported quantization schemes in 🤗 Transformers</a> blog post.',Un,ll,_n,al,Ls='The <a href="https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-AWQ" rel="nofollow">TheBloke/Mistral-7B-OpenOrca-AWQ</a> model was benchmarked with <code>batch_size=1</code> with and without fused modules.',Cn,K,Ns="Unfused module",jn,nl,Hs='<thead><tr><th align="right">Batch Size</th> <th align="right">Prefill Length</th> <th align="right">Decode Length</th> <th align="right">Prefill tokens/s</th> <th align="right">Decode tokens/s</th> <th align="left">Memory (VRAM)</th></tr></thead> <tbody><tr><td align="right">1</td> <td align="right">32</td> <td align="right">32</td> <td align="right">60.0984</td> <td align="right">38.4537</td> <td align="left">4.50 GB (5.68%)</td></tr> <tr><td align="right">1</td> <td align="right">64</td> <td align="right">64</td> <td align="right">1333.67</td> <td align="right">31.6604</td> <td align="left">4.50 GB (5.68%)</td></tr> <tr><td align="right">1</td> <td align="right">128</td> <td align="right">128</td> <td align="right">2434.06</td> <td align="right">31.6272</td> <td align="left">4.50 GB (5.68%)</td></tr> <tr><td align="right">1</td> <td align="right">256</td> <td align="right">256</td> <td align="right">3072.26</td> <td align="right">38.1731</td> <td align="left">4.50 GB (5.68%)</td></tr> <tr><td align="right">1</td> <td align="right">512</td> <td align="right">512</td> <td align="right">3184.74</td> <td align="right">31.6819</td> <td align="left">4.59 GB (5.80%)</td></tr> <tr><td align="right">1</td> <td align="right">1024</td> <td align="right">1024</td> <td align="right">3148.18</td> <td align="right">36.8031</td> <td align="left">4.81 GB (6.07%)</td></tr> <tr><td align="right">1</td> <td align="right">2048</td> <td align="right">2048</td> <td align="right">2927.33</td> <td align="right">35.2676</td> <td align="left">5.73 GB (7.23%)</td></tr></tbody>',qn,O,As="Fused module",kn,sl,Es='<thead><tr><th align="right">Batch Size</th> <th align="right">Prefill Length</th> <th align="right">Decode Length</th> <th align="right">Prefill tokens/s</th> <th align="right">Decode tokens/s</th> <th align="left">Memory (VRAM)</th></tr></thead> <tbody><tr><td align="right">1</td> <td align="right">32</td> <td align="right">32</td> <td align="right">81.4899</td> <td align="right">80.2569</td> <td align="left">4.00 GB (5.05%)</td></tr> <tr><td align="right">1</td> <td align="right">64</td> <td align="right">64</td> <td align="right">1756.1</td> <td align="right">106.26</td> <td align="left">4.00 GB (5.05%)</td></tr> <tr><td align="right">1</td> <td align="right">128</td> <td align="right">128</td> <td align="right">2479.32</td> <td align="right">105.631</td> <td align="left">4.00 GB (5.06%)</td></tr> <tr><td align="right">1</td> <td align="right">256</td> <td align="right">256</td> <td align="right">1813.6</td> <td align="right">85.7485</td> <td align="left">4.01 GB (5.06%)</td></tr> <tr><td align="right">1</td> <td align="right">512</td> <td align="right">512</td> <td align="right">2848.9</td> <td align="right">97.701</td> <td align="left">4.11 GB (5.19%)</td></tr> <tr><td align="right">1</td> <td align="right">1024</td> <td align="right">1024</td> <td align="right">3044.35</td> <td align="right">87.7323</td> <td align="left">4.41 GB (5.57%)</td></tr> <tr><td align="right">1</td> <td align="right">2048</td> <td align="right">2048</td> <td align="right">2715.11</td> <td align="right">89.4709</td> <td align="left">5.57 GB (7.04%)</td></tr></tbody>',Zn,il,Ys='The speed and throughput of fused and unfused modules were also tested with the <a href="https://github.com/huggingface/optimum-benchmark" rel="nofollow">optimum-benchmark</a> library.',Gn,tt,Ps='<div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/fused_forward_memory_plot.png" alt="generate throughput per batch size"/> <figcaption class="mt-2 text-center text-sm text-gray-500">forward peak memory/batch size</figcaption></div> <div><img class="rounded-xl" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/quantization/fused_generate_throughput_plot.png" alt="forward latency per batch size"/> <figcaption class="mt-2 text-center text-sm text-gray-500">generate throughput/batch size</figcaption></div>',zn,rl,In;return u=new G({props:{title:"Quantization",local:"quantization",headingTag:"h1"}}),k=new W({props:{$$slots:{default:[bi]},$$scope:{ctx:$}}}),z=new G({props:{title:"Quanto",local:"quanto",headingTag:"h2"}}),I=new W({props:{$$slots:{default:[yi]},$$scope:{ctx:$}}}),st=new U({props:{code:"cGlwJTIwaW5zdGFsbCUyMHF1YW50byUwQXBpcCUyMGluc3RhbGwlMjBnaXQlMkJodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRmFjY2VsZXJhdGUuZ2l0JTBBcGlwJTIwaW5zdGFsbCUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRmh1Z2dpbmdmYWNlJTJGdHJhbnNmb3JtZXJzLmdpdA==",highlighted:`pip install quanto
pip install git+https://github.com/huggingface/accelerate.git
pip install git+https://github.com/huggingface/transformers.git`,wrap:!1}}),rt=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMFF1YW50b0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyZmFjZWJvb2slMkZvcHQtMTI1bSUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBRdWFudG9Db25maWcod2VpZ2h0cyUzRCUyMmludDglMjIpJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZGV2aWNlX21hcCUzRCUyMmN1ZGElM0EwJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, QuantoConfig

model_id = <span class="hljs-string">&quot;facebook/opt-125m&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
quantization_config = QuantoConfig(weights=<span class="hljs-string">&quot;int8&quot;</span>)
quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;cuda:0&quot;</span>, quantization_config=quantization_config)`,wrap:!1}}),dt=new G({props:{title:"AQLM",local:"aqlm",headingTag:"h2"}}),gt=new U({props:{code:"cGlwJTIwaW5zdGFsbCUyMGFxbG0lNUJncHUlMkNjcHUlNUQ=",highlighted:"pip install aqlm[gpu,cpu]",wrap:!1}}),Mt=new G({props:{title:"PEFT",local:"peft",headingTag:"h3"}}),wt=new G({props:{title:"AQLM configurations",local:"aqlm-configurations",headingTag:"h3"}}),vt=new G({props:{title:"AWQ",local:"awq",headingTag:"h2"}}),V=new W({props:{$$slots:{default:[Mi]},$$scope:{ctx:$}}}),jt=new U({props:{code:"cGlwJTIwaW5zdGFsbCUyMGF1dG9hd3E=",highlighted:"pip install autoawq",wrap:!1}}),kt=new U({props:{code:"JTdCJTBBJTIwJTIwJTIyX25hbWVfb3JfcGF0aCUyMiUzQSUyMCUyMiUyRndvcmtzcGFjZSUyRnByb2Nlc3MlMkZodWdnaW5nZmFjZWg0X3plcGh5ci03Yi1hbHBoYSUyRnNvdXJjZSUyMiUyQyUwQSUyMCUyMCUyMmFyY2hpdGVjdHVyZXMlMjIlM0ElMjAlNUIlMEElMjAlMjAlMjAlMjAlMjJNaXN0cmFsRm9yQ2F1c2FsTE0lMjIlMEElMjAlMjAlNUQlMkMlMEElMjAlMjAuLi4lMEElMjAlMjAuLi4lMEElMjAlMjAuLi4lMEElMjAlMjAlMjJxdWFudGl6YXRpb25fY29uZmlnJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIycXVhbnRfbWV0aG9kJTIyJTNBJTIwJTIyYXdxJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19wb2ludCUyMiUzQSUyMHRydWUlMkMlMEElMjAlMjAlMjAlMjAlMjJncm91cF9zaXplJTIyJTNBJTIwMTI4JTJDJTBBJTIwJTIwJTIwJTIwJTIyYml0cyUyMiUzQSUyMDQlMkMlMEElMjAlMjAlMjAlMjAlMjJ2ZXJzaW9uJTIyJTNBJTIwJTIyZ2VtbSUyMiUwQSUyMCUyMCU3RCUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
  <span class="hljs-attr">&quot;_name_or_path&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;/workspace/process/huggingfaceh4_zephyr-7b-alpha/source&quot;</span><span class="hljs-punctuation">,</span>
  <span class="hljs-attr">&quot;architectures&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">[</span>
    <span class="hljs-string">&quot;MistralForCausalLM&quot;</span>
  <span class="hljs-punctuation">]</span><span class="hljs-punctuation">,</span>
  ...
  ...
  ...
  <span class="hljs-attr">&quot;quantization_config&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;quant_method&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;awq&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_point&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;group_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">128</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;bits&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">4</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;version&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;gemm&quot;</span>
  <span class="hljs-punctuation">}</span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),Gt=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyVGhlQmxva2UlMkZ6ZXBoeXItN0ItYWxwaGEtQVdRJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEJTIyY3VkYSUzQTAlMjIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;TheBloke/zephyr-7B-alpha-AWQ&quot;</span>
model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;cuda:0&quot;</span>)`,wrap:!1}}),It=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyVGhlQmxva2UlMkZ6ZXBoeXItN0ItYWxwaGEtQVdRJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model_id = <span class="hljs-string">&quot;TheBloke/zephyr-7B-alpha-AWQ&quot;</span>
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.float32)`,wrap:!1}}),Wt=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMlRoZUJsb2tlJTJGemVwaHlyLTdCLWFscGhhLUFXUSUyMiUyQyUyMGF0dG5faW1wbGVtZW50YXRpb24lM0QlMjJmbGFzaF9hdHRlbnRpb25fMiUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJjdWRhJTNBMCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;TheBloke/zephyr-7B-alpha-AWQ&quot;</span>, attn_implementation=<span class="hljs-string">&quot;flash_attention_2&quot;</span>, device_map=<span class="hljs-string">&quot;cuda:0&quot;</span>)`,wrap:!1}}),Bt=new G({props:{title:"Fused modules",local:"fused-modules",headingTag:"h3"}}),X=new W({props:{warning:!0,$$slots:{default:[Ti]},$$scope:{ctx:$}}}),F=new Qn({props:{id:"fuse",options:["supported architectures","unsupported architectures"],$$slots:{default:[Ji]},$$scope:{ctx:$}}}),Xt=new G({props:{title:"Exllama-v2 support",local:"exllama-v2-support",headingTag:"h3"}}),Rt=new U({props:{code:"cGlwJTIwaW5zdGFsbCUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRmNhc3Blci1oYW5zZW4lMkZBdXRvQVdRLmdpdA==",highlighted:"pip install git+https://github.com/casper-hansen/AutoAWQ.git",wrap:!1}}),Lt=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwQXdxQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEF3cUNvbmZpZyh2ZXJzaW9uJTNEJTIyZXhsbGFtYSUyMiklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJUaGVCbG9rZSUyRk1pc3RyYWwtN0ItSW5zdHJ1Y3QtdjAuMS1BV1ElMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKSUwQSUwQWlucHV0X2lkcyUyMCUzRCUyMHRvcmNoLnJhbmRpbnQoMCUyQyUyMDEwMCUyQyUyMCgxJTJDJTIwMTI4KSUyQyUyMGR0eXBlJTNEdG9yY2gubG9uZyUyQyUyMGRldmljZSUzRCUyMmN1ZGElMjIpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwoaW5wdXRfaWRzKSUwQXByaW50KG91dHB1dC5sb2dpdHMpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyVGhlQmxva2UlMkZNaXN0cmFsLTdCLUluc3RydWN0LXYwLjEtQVdRJTIyKSUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplci5lbmNvZGUoJTIySG93JTIwdG8lMjBtYWtlJTIwYSUyMGNha2UlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBb3V0cHV0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoaW5wdXRfaWRzJTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSUyQyUyMG1heF9sZW5ndGglM0Q1MCUyQyUyMHBhZF90b2tlbl9pZCUzRDUwMjU2KSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0JTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, AwqConfig

quantization_config = AwqConfig(version=<span class="hljs-string">&quot;exllama&quot;</span>)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;TheBloke/Mistral-7B-Instruct-v0.1-AWQ&quot;</span>,
    quantization_config=quantization_config,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)

input_ids = torch.randint(<span class="hljs-number">0</span>, <span class="hljs-number">100</span>, (<span class="hljs-number">1</span>, <span class="hljs-number">128</span>), dtype=torch.long, device=<span class="hljs-string">&quot;cuda&quot;</span>)
output = model(input_ids)
<span class="hljs-built_in">print</span>(output.logits)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;TheBloke/Mistral-7B-Instruct-v0.1-AWQ&quot;</span>)
input_ids = tokenizer.encode(<span class="hljs-string">&quot;How to make a cake&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
output = model.generate(input_ids, do_sample=<span class="hljs-literal">True</span>, max_length=<span class="hljs-number">50</span>, pad_token_id=<span class="hljs-number">50256</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),R=new W({props:{warning:!0,$$slots:{default:[vi]},$$scope:{ctx:$}}}),Nt=new G({props:{title:"AutoGPTQ",local:"autogptq",headingTag:"h2"}}),x=new W({props:{$$slots:{default:[Ui]},$$scope:{ctx:$}}}),Et=new U({props:{code:"cGlwJTIwaW5zdGFsbCUyMGF1dG8tZ3B0cSUwQXBpcCUyMGluc3RhbGwlMjBnaXQlMkJodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZodWdnaW5nZmFjZSUyRm9wdGltdW0uZ2l0JTBBcGlwJTIwaW5zdGFsbCUyMGdpdCUyQmh0dHBzJTNBJTJGJTJGZ2l0aHViLmNvbSUyRmh1Z2dpbmdmYWNlJTJGdHJhbnNmb3JtZXJzLmdpdCUwQXBpcCUyMGluc3RhbGwlMjAtLXVwZ3JhZGUlMjBhY2NlbGVyYXRl",highlighted:`pip install auto-gptq
pip install git+https://github.com/huggingface/optimum.git
pip install git+https://github.com/huggingface/transformers.git
pip install --upgrade accelerate`,wrap:!1}}),Pt=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEdQVFFDb25maWclMEElMEFtb2RlbF9pZCUyMCUzRCUyMCUyMmZhY2Vib29rJTJGb3B0LTEyNW0lMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCklMEFncHRxX2NvbmZpZyUyMCUzRCUyMEdQVFFDb25maWcoYml0cyUzRDQlMkMlMjBkYXRhc2V0JTNEJTIyYzQlMjIlMkMlMjB0b2tlbml6ZXIlM0R0b2tlbml6ZXIp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, GPTQConfig

model_id = <span class="hljs-string">&quot;facebook/opt-125m&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(model_id)
gptq_config = GPTQConfig(bits=<span class="hljs-number">4</span>, dataset=<span class="hljs-string">&quot;c4&quot;</span>, tokenizer=tokenizer)`,wrap:!1}}),Dt=new U({props:{code:"ZGF0YXNldCUyMCUzRCUyMCU1QiUyMmF1dG8tZ3B0cSUyMGlzJTIwYW4lMjBlYXN5LXRvLXVzZSUyMG1vZGVsJTIwcXVhbnRpemF0aW9uJTIwbGlicmFyeSUyMHdpdGglMjB1c2VyLWZyaWVuZGx5JTIwYXBpcyUyQyUyMGJhc2VkJTIwb24lMjBHUFRRJTIwYWxnb3JpdGhtLiUyMiU1RCUwQWdwdHFfY29uZmlnJTIwJTNEJTIwR1BUUUNvbmZpZyhiaXRzJTNENCUyQyUyMGRhdGFzZXQlM0RkYXRhc2V0JTJDJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyKQ==",highlighted:`dataset = [<span class="hljs-string">&quot;auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm.&quot;</span>]
gptq_config = GPTQConfig(bits=<span class="hljs-number">4</span>, dataset=dataset, tokenizer=tokenizer)`,wrap:!1}}),Ot=new U({props:{code:"cXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEZ3B0cV9jb25maWcp",highlighted:'quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=gptq_config)',wrap:!1}}),ee=new U({props:{code:"cXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjBtYXhfbWVtb3J5JTNEJTdCMCUzQSUyMCUyMjMwR2lCJTIyJTJDJTIwMSUzQSUyMCUyMjQ2R2lCJTIyJTJDJTIwJTIyY3B1JTIyJTNBJTIwJTIyMzBHaUIlMjIlN0QlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEZ3B0cV9jb25maWcp",highlighted:'quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>, max_memory={<span class="hljs-number">0</span>: <span class="hljs-string">&quot;30GiB&quot;</span>, <span class="hljs-number">1</span>: <span class="hljs-string">&quot;46GiB&quot;</span>, <span class="hljs-string">&quot;cpu&quot;</span>: <span class="hljs-string">&quot;30GiB&quot;</span>}, quantization_config=gptq_config)',wrap:!1}}),L=new W({props:{warning:!0,$$slots:{default:[_i]},$$scope:{ctx:$}}}),ae=new U({props:{code:"cXVhbnRpemVkX21vZGVsLnB1c2hfdG9faHViKCUyMm9wdC0xMjVtLWdwdHElMjIpJTBBdG9rZW5pemVyLnB1c2hfdG9faHViKCUyMm9wdC0xMjVtLWdwdHElMjIp",highlighted:`quantized_model.push_to_hub(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)
tokenizer.push_to_hub(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)`,wrap:!1}}),se=new U({props:{code:"cXVhbnRpemVkX21vZGVsLnNhdmVfcHJldHJhaW5lZCglMjJvcHQtMTI1bS1ncHRxJTIyKSUwQXRva2VuaXplci5zYXZlX3ByZXRyYWluZWQoJTIyb3B0LTEyNW0tZ3B0cSUyMiklMEElMEElMjMlMjBpZiUyMHF1YW50aXplZCUyMHdpdGglMjBkZXZpY2VfbWFwJTIwc2V0JTBBcXVhbnRpemVkX21vZGVsLnRvKCUyMmNwdSUyMiklMEFxdWFudGl6ZWRfbW9kZWwuc2F2ZV9wcmV0cmFpbmVkKCUyMm9wdC0xMjVtLWdwdHElMjIp",highlighted:`quantized_model.save_pretrained(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)
tokenizer.save_pretrained(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)

<span class="hljs-comment"># if quantized with device_map set</span>
quantized_model.to(<span class="hljs-string">&quot;cpu&quot;</span>)
quantized_model.save_pretrained(<span class="hljs-string">&quot;opt-125m-gptq&quot;</span>)`,wrap:!1}}),oe=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyJTdCeW91cl91c2VybmFtZSU3RCUyRm9wdC0xMjVtLWdwdHElMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/opt-125m-gptq&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),re=new G({props:{title:"ExLlama",local:"exllama",headingTag:"h3"}}),me=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBHUFRRQ29uZmlnJTBBJTBBZ3B0cV9jb25maWclMjAlM0QlMjBHUFRRQ29uZmlnKGJpdHMlM0Q0JTJDJTIwZXhsbGFtYV9jb25maWclM0QlN0IlMjJ2ZXJzaW9uJTIyJTNBMiU3RCklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjIlN0J5b3VyX3VzZXJuYW1lJTdEJTJGb3B0LTEyNW0tZ3B0cSUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRGdwdHFfY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, GPTQConfig

gptq_config = GPTQConfig(bits=<span class="hljs-number">4</span>, exllama_config={<span class="hljs-string">&quot;version&quot;</span>:<span class="hljs-number">2</span>})
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/opt-125m-gptq&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>, quantization_config=gptq_config)`,wrap:!1}}),N=new W({props:{warning:!0,$$slots:{default:[Ci]},$$scope:{ctx:$}}}),de=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBHUFRRQ29uZmlnJTBBZ3B0cV9jb25maWclMjAlM0QlMjBHUFRRQ29uZmlnKGJpdHMlM0Q0JTJDJTIwdXNlX2V4bGxhbWElM0RGYWxzZSklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjIlN0J5b3VyX3VzZXJuYW1lJTdEJTJGb3B0LTEyNW0tZ3B0cSUyMiUyQyUyMGRldmljZV9tYXAlM0QlMjJjcHUlMjIlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEZ3B0cV9jb25maWcp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, GPTQConfig
gptq_config = GPTQConfig(bits=<span class="hljs-number">4</span>, use_exllama=<span class="hljs-literal">False</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/opt-125m-gptq&quot;</span>, device_map=<span class="hljs-string">&quot;cpu&quot;</span>, quantization_config=gptq_config)`,wrap:!1}}),fe=new G({props:{title:"bitsandbytes",local:"bitsandbytes",headingTag:"h2"}}),H=new Qn({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[ki]},$$scope:{ctx:$}}}),A=new Qn({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[zi]},$$scope:{ctx:$}}}),E=new W({props:{warning:!0,$$slots:{default:[Ii]},$$scope:{ctx:$}}}),ye=new U({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())',wrap:!1}}),Te=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMiU3QnlvdXJfdXNlcm5hbWUlN0QlMkZibG9vbS01NjBtLThiaXQlMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/bloom-560m-8bit&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),we=new G({props:{title:"8-bit",local:"8-bit",headingTag:"h3"}}),Y=new W({props:{$$slots:{default:[Qi]},$$scope:{ctx:$}}}),Je=new G({props:{title:"Offloading",local:"offloading",headingTag:"h4"}}),Ue=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsbG1faW50OF9lbmFibGVfZnAzMl9jcHVfb2ZmbG9hZCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=<span class="hljs-literal">True</span>)`,wrap:!1}}),Ce=new U({props:{code:"ZGV2aWNlX21hcCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLndvcmRfZW1iZWRkaW5ncyUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci53b3JkX2VtYmVkZGluZ3NfbGF5ZXJub3JtJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMmxtX2hlYWQlMjIlM0ElMjAlMjJjcHUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci5oJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLmxuX2YlMjIlM0ElMjAwJTJDJTBBJTdE",highlighted:`device_map = {
    <span class="hljs-string">&quot;transformer.word_embeddings&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.word_embeddings_layernorm&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;lm_head&quot;</span>: <span class="hljs-string">&quot;cpu&quot;</span>,
    <span class="hljs-string">&quot;transformer.h&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.ln_f&quot;</span>: <span class="hljs-number">0</span>,
}`,wrap:!1}}),qe=new U({props:{code:"bW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    device_map=device_map,
    quantization_config=quantization_config,
)`,wrap:!1}}),ke=new G({props:{title:"Outlier threshold",local:"outlier-threshold",headingTag:"h4"}}),ze=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsbG1faW50OF90aHJlc2hvbGQlM0QxMCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)`,wrap:!1}}),Ie=new G({props:{title:"Skip module conversion",local:"skip-module-conversion",headingTag:"h4"}}),We=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbGxtX2ludDhfc2tpcF9tb2R1bGVzJTNEJTVCJTIybG1faGVhZCUyMiU1RCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_skip_modules=[<span class="hljs-string">&quot;lm_head&quot;</span>],
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),Be=new G({props:{title:"Finetuning",local:"finetuning",headingTag:"h4"}}),Xe=new G({props:{title:"4-bit",local:"4-bit",headingTag:"h3"}}),P=new W({props:{$$slots:{default:[Wi]},$$scope:{ctx:$}}}),Re=new G({props:{title:"Compute data type",local:"compute-data-type",headingTag:"h4"}}),Le=new U({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`,wrap:!1}}),Ne=new G({props:{title:"Normal Float 4 (NF4)",local:"normal-float-4-nf4",headingTag:"h4"}}),Ae=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW5mNF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBbW9kZWxfbmY0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRG5mNF9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

nf4_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

model_nf4 = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=nf4_config)`,wrap:!1}}),Ye=new G({props:{title:"Nested quantization",local:"nested-quantization",headingTag:"h4"}}),Se=new U({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQW1vZGVsX2RvdWJsZV9xdWFudCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi0xM2IlMjIlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEZG91YmxlX3F1YW50X2NvbmZpZyk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

model_double_quant = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-13b&quot;</span>, quantization_config=double_quant_config)`,wrap:!1}}),De=new G({props:{title:"Optimum",local:"optimum",headingTag:"h2"}}),Oe=new G({props:{title:"Benchmarks",local:"benchmarks",headingTag:"h2"}}),ll=new G({props:{title:"Fused AWQ modules",local:"fused-awq-modules",headingTag:"h3"}}),{c(){n=r("meta"),M=s(),o=r("p"),y=s(),d(u.$$.fragment),T=s(),_=r("p"),_.textContent=j,w=s(),q=r("p"),q.textContent=Z,C=s(),d(k.$$.fragment),J=s(),d(z.$$.fragment),v=s(),d(I.$$.fragment),pl=s(),lt=r("p"),lt.innerHTML=Wn,ml=s(),at=r("ul"),at.innerHTML=Bn,ul=s(),nt=r("p"),nt.textContent=Vn,dl=s(),d(st.$$.fragment),fl=s(),it=r("p"),it.innerHTML=Xn,hl=s(),ot=r("p"),ot.innerHTML=Fn,cl=s(),d(rt.$$.fragment),gl=s(),pt=r("p"),pt.textContent=Rn,bl=s(),mt=r("p"),mt.innerHTML=xn,yl=s(),B=r("div"),B.innerHTML=Ln,Ml=s(),ut=r("p"),ut.textContent=Nn,Tl=s(),d(dt.$$.fragment),wl=s(),ft=r("p"),ft.innerHTML=Hn,$l=s(),ht=r("p"),ht.innerHTML=An,Jl=s(),ct=r("p"),ct.innerHTML=En,vl=s(),d(gt.$$.fragment),Ul=s(),bt=r("p"),bt.textContent=Yn,_l=s(),yt=r("p"),yt.innerHTML=Pn,Cl=s(),d(Mt.$$.fragment),jl=s(),Tt=r("p"),Tt.innerHTML=Sn,ql=s(),d(wt.$$.fragment),kl=s(),$t=r("p"),$t.textContent=Dn,Zl=s(),Jt=r("table"),Jt.innerHTML=Kn,Gl=s(),d(vt.$$.fragment),zl=s(),d(V.$$.fragment),Il=s(),Ut=r("p"),Ut.innerHTML=On,Ql=s(),_t=r("p"),_t.innerHTML=ts,Wl=s(),Ct=r("p"),Ct.textContent=es,Bl=s(),d(jt.$$.fragment),Vl=s(),qt=r("p"),qt.innerHTML=ls,Xl=s(),d(kt.$$.fragment),Fl=s(),Zt=r("p"),Zt.innerHTML=as,Rl=s(),d(Gt.$$.fragment),xl=s(),zt=r("p"),zt.innerHTML=ns,Ll=s(),d(It.$$.fragment),Nl=s(),Qt=r("p"),Qt.innerHTML=ss,Hl=s(),d(Wt.$$.fragment),Al=s(),d(Bt.$$.fragment),El=s(),Vt=r("p"),Vt.innerHTML=is,Yl=s(),d(X.$$.fragment),Pl=s(),d(F.$$.fragment),Sl=s(),d(Xt.$$.fragment),Dl=s(),Ft=r("p"),Ft.innerHTML=os,Kl=s(),d(Rt.$$.fragment),Ol=s(),xt=r("p"),xt.innerHTML=rs,ta=s(),d(Lt.$$.fragment),ea=s(),d(R.$$.fragment),la=s(),d(Nt.$$.fragment),aa=s(),d(x.$$.fragment),na=s(),Ht=r("p"),Ht.innerHTML=ps,sa=s(),At=r("p"),At.textContent=ms,ia=s(),d(Et.$$.fragment),oa=s(),Yt=r("p"),Yt.innerHTML=us,ra=s(),d(Pt.$$.fragment),pa=s(),St=r("p"),St.textContent=ds,ma=s(),d(Dt.$$.fragment),ua=s(),Kt=r("p"),Kt.innerHTML=fs,da=s(),d(Ot.$$.fragment),fa=s(),te=r("p"),te.innerHTML=hs,ha=s(),d(ee.$$.fragment),ca=s(),d(L.$$.fragment),ga=s(),le=r("p"),le.innerHTML=cs,ba=s(),d(ae.$$.fragment),ya=s(),ne=r("p"),ne.innerHTML=gs,Ma=s(),d(se.$$.fragment),Ta=s(),ie=r("p"),ie.innerHTML=bs,wa=s(),d(oe.$$.fragment),$a=s(),d(re.$$.fragment),Ja=s(),pe=r("p"),pe.innerHTML=ys,va=s(),d(me.$$.fragment),Ua=s(),d(N.$$.fragment),_a=s(),ue=r("p"),ue.textContent=Ms,Ca=s(),d(de.$$.fragment),ja=s(),d(fe.$$.fragment),qa=s(),he=r("p"),he.innerHTML=Ts,ka=s(),ce=r("p"),ce.textContent=ws,Za=s(),d(H.$$.fragment),Ga=s(),ge=r("p"),ge.innerHTML=$s,za=s(),d(A.$$.fragment),Ia=s(),d(E.$$.fragment),Qa=s(),be=r("p"),be.innerHTML=Js,Wa=s(),d(ye.$$.fragment),Ba=s(),Me=r("p"),Me.innerHTML=vs,Va=s(),d(Te.$$.fragment),Xa=s(),d(we.$$.fragment),Fa=s(),d(Y.$$.fragment),Ra=s(),$e=r("p"),$e.textContent=Us,xa=s(),d(Je.$$.fragment),La=s(),ve=r("p"),ve.innerHTML=_s,Na=s(),d(Ue.$$.fragment),Ha=s(),_e=r("p"),_e.innerHTML=Cs,Aa=s(),d(Ce.$$.fragment),Ea=s(),je=r("p"),je.innerHTML=js,Ya=s(),d(qe.$$.fragment),Pa=s(),d(ke.$$.fragment),Sa=s(),Ze=r("p"),Ze.textContent=qs,Da=s(),Ge=r("p"),Ge.innerHTML=ks,Ka=s(),d(ze.$$.fragment),Oa=s(),d(Ie.$$.fragment),tn=s(),Qe=r("p"),Qe.innerHTML=Zs,en=s(),d(We.$$.fragment),ln=s(),d(Be.$$.fragment),an=s(),Ve=r("p"),Ve.innerHTML=Gs,nn=s(),d(Xe.$$.fragment),sn=s(),d(P.$$.fragment),on=s(),Fe=r("p"),Fe.textContent=zs,rn=s(),d(Re.$$.fragment),pn=s(),xe=r("p"),xe.innerHTML=Is,mn=s(),d(Le.$$.fragment),un=s(),d(Ne.$$.fragment),dn=s(),He=r("p"),He.innerHTML=Qs,fn=s(),d(Ae.$$.fragment),hn=s(),Ee=r("p"),Ee.innerHTML=Ws,cn=s(),d(Ye.$$.fragment),gn=s(),Pe=r("p"),Pe.innerHTML=Bs,bn=s(),d(Se.$$.fragment),yn=s(),d(De.$$.fragment),Mn=s(),Ke=r("p"),Ke.innerHTML=Vs,Tn=s(),d(Oe.$$.fragment),wn=s(),tl=r("p"),tl.innerHTML=Xs,$n=s(),S=r("div"),S.innerHTML=Fs,Jn=s(),D=r("div"),D.innerHTML=Rs,vn=s(),el=r("p"),el.innerHTML=xs,Un=s(),d(ll.$$.fragment),_n=s(),al=r("p"),al.innerHTML=Ls,Cn=s(),K=r("figcaption"),K.textContent=Ns,jn=s(),nl=r("table"),nl.innerHTML=Hs,qn=s(),O=r("figcaption"),O.textContent=As,kn=s(),sl=r("table"),sl.innerHTML=Es,Zn=s(),il=r("p"),il.innerHTML=Ys,Gn=s(),tt=r("div"),tt.innerHTML=Ps,zn=s(),rl=r("p"),this.h()},l(t){const e=ci("svelte-u9bgzb",document.head);n=p(e,"META",{name:!0,content:!0}),e.forEach(l),M=i(t),o=p(t,"P",{}),mi(o).forEach(l),y=i(t),f(u.$$.fragment,t),T=i(t),_=p(t,"P",{"data-svelte-h":!0}),m(_)!=="svelte-1euzkei"&&(_.textContent=j),w=i(t),q=p(t,"P",{"data-svelte-h":!0}),m(q)!=="svelte-1f5pv07"&&(q.textContent=Z),C=i(t),f(k.$$.fragment,t),J=i(t),f(z.$$.fragment,t),v=i(t),f(I.$$.fragment,t),pl=i(t),lt=p(t,"P",{"data-svelte-h":!0}),m(lt)!=="svelte-1uxysak"&&(lt.innerHTML=Wn),ml=i(t),at=p(t,"UL",{"data-svelte-h":!0}),m(at)!=="svelte-1rrl1p8"&&(at.innerHTML=Bn),ul=i(t),nt=p(t,"P",{"data-svelte-h":!0}),m(nt)!=="svelte-13c61n1"&&(nt.textContent=Vn),dl=i(t),f(st.$$.fragment,t),fl=i(t),it=p(t,"P",{"data-svelte-h":!0}),m(it)!=="svelte-5dlwxe"&&(it.innerHTML=Xn),hl=i(t),ot=p(t,"P",{"data-svelte-h":!0}),m(ot)!=="svelte-acy0np"&&(ot.innerHTML=Fn),cl=i(t),f(rt.$$.fragment,t),gl=i(t),pt=p(t,"P",{"data-svelte-h":!0}),m(pt)!=="svelte-1sscg20"&&(pt.textContent=Rn),bl=i(t),mt=p(t,"P",{"data-svelte-h":!0}),m(mt)!=="svelte-1woqmtc"&&(mt.innerHTML=xn),yl=i(t),B=p(t,"DIV",{class:!0,"data-svelte-h":!0}),m(B)!=="svelte-196rp13"&&(B.innerHTML=Ln),Ml=i(t),ut=p(t,"P",{"data-svelte-h":!0}),m(ut)!=="svelte-5v5thh"&&(ut.textContent=Nn),Tl=i(t),f(dt.$$.fragment,t),wl=i(t),ft=p(t,"P",{"data-svelte-h":!0}),m(ft)!=="svelte-vu1jh7"&&(ft.innerHTML=Hn),$l=i(t),ht=p(t,"P",{"data-svelte-h":!0}),m(ht)!=="svelte-11fbd25"&&(ht.innerHTML=An),Jl=i(t),ct=p(t,"P",{"data-svelte-h":!0}),m(ct)!=="svelte-1rdkef8"&&(ct.innerHTML=En),vl=i(t),f(gt.$$.fragment,t),Ul=i(t),bt=p(t,"P",{"data-svelte-h":!0}),m(bt)!=="svelte-6sf1ac"&&(bt.textContent=Yn),_l=i(t),yt=p(t,"P",{"data-svelte-h":!0}),m(yt)!=="svelte-mrigan"&&(yt.innerHTML=Pn),Cl=i(t),f(Mt.$$.fragment,t),jl=i(t),Tt=p(t,"P",{"data-svelte-h":!0}),m(Tt)!=="svelte-1uurbap"&&(Tt.innerHTML=Sn),ql=i(t),f(wt.$$.fragment,t),kl=i(t),$t=p(t,"P",{"data-svelte-h":!0}),m($t)!=="svelte-1ouechy"&&($t.textContent=Dn),Zl=i(t),Jt=p(t,"TABLE",{"data-svelte-h":!0}),m(Jt)!=="svelte-mgp7ro"&&(Jt.innerHTML=Kn),Gl=i(t),f(vt.$$.fragment,t),zl=i(t),f(V.$$.fragment,t),Il=i(t),Ut=p(t,"P",{"data-svelte-h":!0}),m(Ut)!=="svelte-1c5vyni"&&(Ut.innerHTML=On),Ql=i(t),_t=p(t,"P",{"data-svelte-h":!0}),m(_t)!=="svelte-vomabp"&&(_t.innerHTML=ts),Wl=i(t),Ct=p(t,"P",{"data-svelte-h":!0}),m(Ct)!=="svelte-1ozln1k"&&(Ct.textContent=es),Bl=i(t),f(jt.$$.fragment,t),Vl=i(t),qt=p(t,"P",{"data-svelte-h":!0}),m(qt)!=="svelte-1lxz9t"&&(qt.innerHTML=ls),Xl=i(t),f(kt.$$.fragment,t),Fl=i(t),Zt=p(t,"P",{"data-svelte-h":!0}),m(Zt)!=="svelte-1v4hy46"&&(Zt.innerHTML=as),Rl=i(t),f(Gt.$$.fragment,t),xl=i(t),zt=p(t,"P",{"data-svelte-h":!0}),m(zt)!=="svelte-19un5li"&&(zt.innerHTML=ns),Ll=i(t),f(It.$$.fragment,t),Nl=i(t),Qt=p(t,"P",{"data-svelte-h":!0}),m(Qt)!=="svelte-ptrt24"&&(Qt.innerHTML=ss),Hl=i(t),f(Wt.$$.fragment,t),Al=i(t),f(Bt.$$.fragment,t),El=i(t),Vt=p(t,"P",{"data-svelte-h":!0}),m(Vt)!=="svelte-jywid1"&&(Vt.innerHTML=is),Yl=i(t),f(X.$$.fragment,t),Pl=i(t),f(F.$$.fragment,t),Sl=i(t),f(Xt.$$.fragment,t),Dl=i(t),Ft=p(t,"P",{"data-svelte-h":!0}),m(Ft)!=="svelte-1fw2pai"&&(Ft.innerHTML=os),Kl=i(t),f(Rt.$$.fragment,t),Ol=i(t),xt=p(t,"P",{"data-svelte-h":!0}),m(xt)!=="svelte-in32kr"&&(xt.innerHTML=rs),ta=i(t),f(Lt.$$.fragment,t),ea=i(t),f(R.$$.fragment,t),la=i(t),f(Nt.$$.fragment,t),aa=i(t),f(x.$$.fragment,t),na=i(t),Ht=p(t,"P",{"data-svelte-h":!0}),m(Ht)!=="svelte-sq0x7n"&&(Ht.innerHTML=ps),sa=i(t),At=p(t,"P",{"data-svelte-h":!0}),m(At)!=="svelte-13c61n1"&&(At.textContent=ms),ia=i(t),f(Et.$$.fragment,t),oa=i(t),Yt=p(t,"P",{"data-svelte-h":!0}),m(Yt)!=="svelte-1q1rjwz"&&(Yt.innerHTML=us),ra=i(t),f(Pt.$$.fragment,t),pa=i(t),St=p(t,"P",{"data-svelte-h":!0}),m(St)!=="svelte-1cjo88z"&&(St.textContent=ds),ma=i(t),f(Dt.$$.fragment,t),ua=i(t),Kt=p(t,"P",{"data-svelte-h":!0}),m(Kt)!=="svelte-1kh6ftl"&&(Kt.innerHTML=fs),da=i(t),f(Ot.$$.fragment,t),fa=i(t),te=p(t,"P",{"data-svelte-h":!0}),m(te)!=="svelte-1tle3a0"&&(te.innerHTML=hs),ha=i(t),f(ee.$$.fragment,t),ca=i(t),f(L.$$.fragment,t),ga=i(t),le=p(t,"P",{"data-svelte-h":!0}),m(le)!=="svelte-rk1pzp"&&(le.innerHTML=cs),ba=i(t),f(ae.$$.fragment,t),ya=i(t),ne=p(t,"P",{"data-svelte-h":!0}),m(ne)!=="svelte-1hpcnk9"&&(ne.innerHTML=gs),Ma=i(t),f(se.$$.fragment,t),Ta=i(t),ie=p(t,"P",{"data-svelte-h":!0}),m(ie)!=="svelte-1grftev"&&(ie.innerHTML=bs),wa=i(t),f(oe.$$.fragment,t),$a=i(t),f(re.$$.fragment,t),Ja=i(t),pe=p(t,"P",{"data-svelte-h":!0}),m(pe)!=="svelte-1p6qc9q"&&(pe.innerHTML=ys),va=i(t),f(me.$$.fragment,t),Ua=i(t),f(N.$$.fragment,t),_a=i(t),ue=p(t,"P",{"data-svelte-h":!0}),m(ue)!=="svelte-m9nw5y"&&(ue.textContent=Ms),Ca=i(t),f(de.$$.fragment,t),ja=i(t),f(fe.$$.fragment,t),qa=i(t),he=p(t,"P",{"data-svelte-h":!0}),m(he)!=="svelte-5a92xi"&&(he.innerHTML=Ts),ka=i(t),ce=p(t,"P",{"data-svelte-h":!0}),m(ce)!=="svelte-gf36q7"&&(ce.textContent=ws),Za=i(t),f(H.$$.fragment,t),Ga=i(t),ge=p(t,"P",{"data-svelte-h":!0}),m(ge)!=="svelte-1hsciom"&&(ge.innerHTML=$s),za=i(t),f(A.$$.fragment,t),Ia=i(t),f(E.$$.fragment,t),Qa=i(t),be=p(t,"P",{"data-svelte-h":!0}),m(be)!=="svelte-1bxp667"&&(be.innerHTML=Js),Wa=i(t),f(ye.$$.fragment,t),Ba=i(t),Me=p(t,"P",{"data-svelte-h":!0}),m(Me)!=="svelte-d3brck"&&(Me.innerHTML=vs),Va=i(t),f(Te.$$.fragment,t),Xa=i(t),f(we.$$.fragment,t),Fa=i(t),f(Y.$$.fragment,t),Ra=i(t),$e=p(t,"P",{"data-svelte-h":!0}),m($e)!=="svelte-1yx1x2g"&&($e.textContent=Us),xa=i(t),f(Je.$$.fragment,t),La=i(t),ve=p(t,"P",{"data-svelte-h":!0}),m(ve)!=="svelte-1ks4tm4"&&(ve.innerHTML=_s),Na=i(t),f(Ue.$$.fragment,t),Ha=i(t),_e=p(t,"P",{"data-svelte-h":!0}),m(_e)!=="svelte-15oy7am"&&(_e.innerHTML=Cs),Aa=i(t),f(Ce.$$.fragment,t),Ea=i(t),je=p(t,"P",{"data-svelte-h":!0}),m(je)!=="svelte-k1g8c2"&&(je.innerHTML=js),Ya=i(t),f(qe.$$.fragment,t),Pa=i(t),f(ke.$$.fragment,t),Sa=i(t),Ze=p(t,"P",{"data-svelte-h":!0}),m(Ze)!=="svelte-ur5rgd"&&(Ze.textContent=qs),Da=i(t),Ge=p(t,"P",{"data-svelte-h":!0}),m(Ge)!=="svelte-kw4eq"&&(Ge.innerHTML=ks),Ka=i(t),f(ze.$$.fragment,t),Oa=i(t),f(Ie.$$.fragment,t),tn=i(t),Qe=p(t,"P",{"data-svelte-h":!0}),m(Qe)!=="svelte-v8c3hd"&&(Qe.innerHTML=Zs),en=i(t),f(We.$$.fragment,t),ln=i(t),f(Be.$$.fragment,t),an=i(t),Ve=p(t,"P",{"data-svelte-h":!0}),m(Ve)!=="svelte-4nvx4q"&&(Ve.innerHTML=Gs),nn=i(t),f(Xe.$$.fragment,t),sn=i(t),f(P.$$.fragment,t),on=i(t),Fe=p(t,"P",{"data-svelte-h":!0}),m(Fe)!=="svelte-7ob7j"&&(Fe.textContent=zs),rn=i(t),f(Re.$$.fragment,t),pn=i(t),xe=p(t,"P",{"data-svelte-h":!0}),m(xe)!=="svelte-1s9vynt"&&(xe.innerHTML=Is),mn=i(t),f(Le.$$.fragment,t),un=i(t),f(Ne.$$.fragment,t),dn=i(t),He=p(t,"P",{"data-svelte-h":!0}),m(He)!=="svelte-1u3ixrn"&&(He.innerHTML=Qs),fn=i(t),f(Ae.$$.fragment,t),hn=i(t),Ee=p(t,"P",{"data-svelte-h":!0}),m(Ee)!=="svelte-1qoc2ct"&&(Ee.innerHTML=Ws),cn=i(t),f(Ye.$$.fragment,t),gn=i(t),Pe=p(t,"P",{"data-svelte-h":!0}),m(Pe)!=="svelte-l0u9eq"&&(Pe.innerHTML=Bs),bn=i(t),f(Se.$$.fragment,t),yn=i(t),f(De.$$.fragment,t),Mn=i(t),Ke=p(t,"P",{"data-svelte-h":!0}),m(Ke)!=="svelte-opujmz"&&(Ke.innerHTML=Vs),Tn=i(t),f(Oe.$$.fragment,t),wn=i(t),tl=p(t,"P",{"data-svelte-h":!0}),m(tl)!=="svelte-10oh9lm"&&(tl.innerHTML=Xs),$n=i(t),S=p(t,"DIV",{class:!0,"data-svelte-h":!0}),m(S)!=="svelte-10jjm5l"&&(S.innerHTML=Fs),Jn=i(t),D=p(t,"DIV",{class:!0,"data-svelte-h":!0}),m(D)!=="svelte-48at03"&&(D.innerHTML=Rs),vn=i(t),el=p(t,"P",{"data-svelte-h":!0}),m(el)!=="svelte-1sm7gom"&&(el.innerHTML=xs),Un=i(t),f(ll.$$.fragment,t),_n=i(t),al=p(t,"P",{"data-svelte-h":!0}),m(al)!=="svelte-81zmsw"&&(al.innerHTML=Ls),Cn=i(t),K=p(t,"FIGCAPTION",{class:!0,"data-svelte-h":!0}),m(K)!=="svelte-grefv0"&&(K.textContent=Ns),jn=i(t),nl=p(t,"TABLE",{"data-svelte-h":!0}),m(nl)!=="svelte-19aactm"&&(nl.innerHTML=Hs),qn=i(t),O=p(t,"FIGCAPTION",{class:!0,"data-svelte-h":!0}),m(O)!=="svelte-1r15bg7"&&(O.textContent=As),kn=i(t),sl=p(t,"TABLE",{"data-svelte-h":!0}),m(sl)!=="svelte-19fczbk"&&(sl.innerHTML=Es),Zn=i(t),il=p(t,"P",{"data-svelte-h":!0}),m(il)!=="svelte-gu8e8k"&&(il.innerHTML=Ys),Gn=i(t),tt=p(t,"DIV",{class:!0,"data-svelte-h":!0}),m(tt)!=="svelte-1ke50ja"&&(tt.innerHTML=Ps),zn=i(t),rl=p(t,"P",{}),mi(rl).forEach(l),this.h()},h(){et(n,"name","hf:doc:metadata"),et(n,"content",Vi),et(B,"class","flex gap-4"),et(S,"class","flex gap-4"),et(D,"class","flex gap-4"),et(K,"class","text-center text-gray-500 text-lg"),et(O,"class","text-center text-gray-500 text-lg"),et(tt,"class","flex gap-4")},m(t,e){gi(document.head,n),a(t,M,e),a(t,o,e),a(t,y,e),h(u,t,e),a(t,T,e),a(t,_,e),a(t,w,e),a(t,q,e),a(t,C,e),h(k,t,e),a(t,J,e),h(z,t,e),a(t,v,e),h(I,t,e),a(t,pl,e),a(t,lt,e),a(t,ml,e),a(t,at,e),a(t,ul,e),a(t,nt,e),a(t,dl,e),h(st,t,e),a(t,fl,e),a(t,it,e),a(t,hl,e),a(t,ot,e),a(t,cl,e),h(rt,t,e),a(t,gl,e),a(t,pt,e),a(t,bl,e),a(t,mt,e),a(t,yl,e),a(t,B,e),a(t,Ml,e),a(t,ut,e),a(t,Tl,e),h(dt,t,e),a(t,wl,e),a(t,ft,e),a(t,$l,e),a(t,ht,e),a(t,Jl,e),a(t,ct,e),a(t,vl,e),h(gt,t,e),a(t,Ul,e),a(t,bt,e),a(t,_l,e),a(t,yt,e),a(t,Cl,e),h(Mt,t,e),a(t,jl,e),a(t,Tt,e),a(t,ql,e),h(wt,t,e),a(t,kl,e),a(t,$t,e),a(t,Zl,e),a(t,Jt,e),a(t,Gl,e),h(vt,t,e),a(t,zl,e),h(V,t,e),a(t,Il,e),a(t,Ut,e),a(t,Ql,e),a(t,_t,e),a(t,Wl,e),a(t,Ct,e),a(t,Bl,e),h(jt,t,e),a(t,Vl,e),a(t,qt,e),a(t,Xl,e),h(kt,t,e),a(t,Fl,e),a(t,Zt,e),a(t,Rl,e),h(Gt,t,e),a(t,xl,e),a(t,zt,e),a(t,Ll,e),h(It,t,e),a(t,Nl,e),a(t,Qt,e),a(t,Hl,e),h(Wt,t,e),a(t,Al,e),h(Bt,t,e),a(t,El,e),a(t,Vt,e),a(t,Yl,e),h(X,t,e),a(t,Pl,e),h(F,t,e),a(t,Sl,e),h(Xt,t,e),a(t,Dl,e),a(t,Ft,e),a(t,Kl,e),h(Rt,t,e),a(t,Ol,e),a(t,xt,e),a(t,ta,e),h(Lt,t,e),a(t,ea,e),h(R,t,e),a(t,la,e),h(Nt,t,e),a(t,aa,e),h(x,t,e),a(t,na,e),a(t,Ht,e),a(t,sa,e),a(t,At,e),a(t,ia,e),h(Et,t,e),a(t,oa,e),a(t,Yt,e),a(t,ra,e),h(Pt,t,e),a(t,pa,e),a(t,St,e),a(t,ma,e),h(Dt,t,e),a(t,ua,e),a(t,Kt,e),a(t,da,e),h(Ot,t,e),a(t,fa,e),a(t,te,e),a(t,ha,e),h(ee,t,e),a(t,ca,e),h(L,t,e),a(t,ga,e),a(t,le,e),a(t,ba,e),h(ae,t,e),a(t,ya,e),a(t,ne,e),a(t,Ma,e),h(se,t,e),a(t,Ta,e),a(t,ie,e),a(t,wa,e),h(oe,t,e),a(t,$a,e),h(re,t,e),a(t,Ja,e),a(t,pe,e),a(t,va,e),h(me,t,e),a(t,Ua,e),h(N,t,e),a(t,_a,e),a(t,ue,e),a(t,Ca,e),h(de,t,e),a(t,ja,e),h(fe,t,e),a(t,qa,e),a(t,he,e),a(t,ka,e),a(t,ce,e),a(t,Za,e),h(H,t,e),a(t,Ga,e),a(t,ge,e),a(t,za,e),h(A,t,e),a(t,Ia,e),h(E,t,e),a(t,Qa,e),a(t,be,e),a(t,Wa,e),h(ye,t,e),a(t,Ba,e),a(t,Me,e),a(t,Va,e),h(Te,t,e),a(t,Xa,e),h(we,t,e),a(t,Fa,e),h(Y,t,e),a(t,Ra,e),a(t,$e,e),a(t,xa,e),h(Je,t,e),a(t,La,e),a(t,ve,e),a(t,Na,e),h(Ue,t,e),a(t,Ha,e),a(t,_e,e),a(t,Aa,e),h(Ce,t,e),a(t,Ea,e),a(t,je,e),a(t,Ya,e),h(qe,t,e),a(t,Pa,e),h(ke,t,e),a(t,Sa,e),a(t,Ze,e),a(t,Da,e),a(t,Ge,e),a(t,Ka,e),h(ze,t,e),a(t,Oa,e),h(Ie,t,e),a(t,tn,e),a(t,Qe,e),a(t,en,e),h(We,t,e),a(t,ln,e),h(Be,t,e),a(t,an,e),a(t,Ve,e),a(t,nn,e),h(Xe,t,e),a(t,sn,e),h(P,t,e),a(t,on,e),a(t,Fe,e),a(t,rn,e),h(Re,t,e),a(t,pn,e),a(t,xe,e),a(t,mn,e),h(Le,t,e),a(t,un,e),h(Ne,t,e),a(t,dn,e),a(t,He,e),a(t,fn,e),h(Ae,t,e),a(t,hn,e),a(t,Ee,e),a(t,cn,e),h(Ye,t,e),a(t,gn,e),a(t,Pe,e),a(t,bn,e),h(Se,t,e),a(t,yn,e),h(De,t,e),a(t,Mn,e),a(t,Ke,e),a(t,Tn,e),h(Oe,t,e),a(t,wn,e),a(t,tl,e),a(t,$n,e),a(t,S,e),a(t,Jn,e),a(t,D,e),a(t,vn,e),a(t,el,e),a(t,Un,e),h(ll,t,e),a(t,_n,e),a(t,al,e),a(t,Cn,e),a(t,K,e),a(t,jn,e),a(t,nl,e),a(t,qn,e),a(t,O,e),a(t,kn,e),a(t,sl,e),a(t,Zn,e),a(t,il,e),a(t,Gn,e),a(t,tt,e),a(t,zn,e),a(t,rl,e),In=!0},p(t,[e]){const Ss={};e&2&&(Ss.$$scope={dirty:e,ctx:t}),k.$set(Ss);const Ds={};e&2&&(Ds.$$scope={dirty:e,ctx:t}),I.$set(Ds);const Ks={};e&2&&(Ks.$$scope={dirty:e,ctx:t}),V.$set(Ks);const Os={};e&2&&(Os.$$scope={dirty:e,ctx:t}),X.$set(Os);const ti={};e&2&&(ti.$$scope={dirty:e,ctx:t}),F.$set(ti);const ei={};e&2&&(ei.$$scope={dirty:e,ctx:t}),R.$set(ei);const li={};e&2&&(li.$$scope={dirty:e,ctx:t}),x.$set(li);const ai={};e&2&&(ai.$$scope={dirty:e,ctx:t}),L.$set(ai);const ni={};e&2&&(ni.$$scope={dirty:e,ctx:t}),N.$set(ni);const si={};e&2&&(si.$$scope={dirty:e,ctx:t}),H.$set(si);const ii={};e&2&&(ii.$$scope={dirty:e,ctx:t}),A.$set(ii);const oi={};e&2&&(oi.$$scope={dirty:e,ctx:t}),E.$set(oi);const ri={};e&2&&(ri.$$scope={dirty:e,ctx:t}),Y.$set(ri);const pi={};e&2&&(pi.$$scope={dirty:e,ctx:t}),P.$set(pi)},i(t){In||(c(u.$$.fragment,t),c(k.$$.fragment,t),c(z.$$.fragment,t),c(I.$$.fragment,t),c(st.$$.fragment,t),c(rt.$$.fragment,t),c(dt.$$.fragment,t),c(gt.$$.fragment,t),c(Mt.$$.fragment,t),c(wt.$$.fragment,t),c(vt.$$.fragment,t),c(V.$$.fragment,t),c(jt.$$.fragment,t),c(kt.$$.fragment,t),c(Gt.$$.fragment,t),c(It.$$.fragment,t),c(Wt.$$.fragment,t),c(Bt.$$.fragment,t),c(X.$$.fragment,t),c(F.$$.fragment,t),c(Xt.$$.fragment,t),c(Rt.$$.fragment,t),c(Lt.$$.fragment,t),c(R.$$.fragment,t),c(Nt.$$.fragment,t),c(x.$$.fragment,t),c(Et.$$.fragment,t),c(Pt.$$.fragment,t),c(Dt.$$.fragment,t),c(Ot.$$.fragment,t),c(ee.$$.fragment,t),c(L.$$.fragment,t),c(ae.$$.fragment,t),c(se.$$.fragment,t),c(oe.$$.fragment,t),c(re.$$.fragment,t),c(me.$$.fragment,t),c(N.$$.fragment,t),c(de.$$.fragment,t),c(fe.$$.fragment,t),c(H.$$.fragment,t),c(A.$$.fragment,t),c(E.$$.fragment,t),c(ye.$$.fragment,t),c(Te.$$.fragment,t),c(we.$$.fragment,t),c(Y.$$.fragment,t),c(Je.$$.fragment,t),c(Ue.$$.fragment,t),c(Ce.$$.fragment,t),c(qe.$$.fragment,t),c(ke.$$.fragment,t),c(ze.$$.fragment,t),c(Ie.$$.fragment,t),c(We.$$.fragment,t),c(Be.$$.fragment,t),c(Xe.$$.fragment,t),c(P.$$.fragment,t),c(Re.$$.fragment,t),c(Le.$$.fragment,t),c(Ne.$$.fragment,t),c(Ae.$$.fragment,t),c(Ye.$$.fragment,t),c(Se.$$.fragment,t),c(De.$$.fragment,t),c(Oe.$$.fragment,t),c(ll.$$.fragment,t),In=!0)},o(t){g(u.$$.fragment,t),g(k.$$.fragment,t),g(z.$$.fragment,t),g(I.$$.fragment,t),g(st.$$.fragment,t),g(rt.$$.fragment,t),g(dt.$$.fragment,t),g(gt.$$.fragment,t),g(Mt.$$.fragment,t),g(wt.$$.fragment,t),g(vt.$$.fragment,t),g(V.$$.fragment,t),g(jt.$$.fragment,t),g(kt.$$.fragment,t),g(Gt.$$.fragment,t),g(It.$$.fragment,t),g(Wt.$$.fragment,t),g(Bt.$$.fragment,t),g(X.$$.fragment,t),g(F.$$.fragment,t),g(Xt.$$.fragment,t),g(Rt.$$.fragment,t),g(Lt.$$.fragment,t),g(R.$$.fragment,t),g(Nt.$$.fragment,t),g(x.$$.fragment,t),g(Et.$$.fragment,t),g(Pt.$$.fragment,t),g(Dt.$$.fragment,t),g(Ot.$$.fragment,t),g(ee.$$.fragment,t),g(L.$$.fragment,t),g(ae.$$.fragment,t),g(se.$$.fragment,t),g(oe.$$.fragment,t),g(re.$$.fragment,t),g(me.$$.fragment,t),g(N.$$.fragment,t),g(de.$$.fragment,t),g(fe.$$.fragment,t),g(H.$$.fragment,t),g(A.$$.fragment,t),g(E.$$.fragment,t),g(ye.$$.fragment,t),g(Te.$$.fragment,t),g(we.$$.fragment,t),g(Y.$$.fragment,t),g(Je.$$.fragment,t),g(Ue.$$.fragment,t),g(Ce.$$.fragment,t),g(qe.$$.fragment,t),g(ke.$$.fragment,t),g(ze.$$.fragment,t),g(Ie.$$.fragment,t),g(We.$$.fragment,t),g(Be.$$.fragment,t),g(Xe.$$.fragment,t),g(P.$$.fragment,t),g(Re.$$.fragment,t),g(Le.$$.fragment,t),g(Ne.$$.fragment,t),g(Ae.$$.fragment,t),g(Ye.$$.fragment,t),g(Se.$$.fragment,t),g(De.$$.fragment,t),g(Oe.$$.fragment,t),g(ll.$$.fragment,t),In=!1},d(t){t&&(l(M),l(o),l(y),l(T),l(_),l(w),l(q),l(C),l(J),l(v),l(pl),l(lt),l(ml),l(at),l(ul),l(nt),l(dl),l(fl),l(it),l(hl),l(ot),l(cl),l(gl),l(pt),l(bl),l(mt),l(yl),l(B),l(Ml),l(ut),l(Tl),l(wl),l(ft),l($l),l(ht),l(Jl),l(ct),l(vl),l(Ul),l(bt),l(_l),l(yt),l(Cl),l(jl),l(Tt),l(ql),l(kl),l($t),l(Zl),l(Jt),l(Gl),l(zl),l(Il),l(Ut),l(Ql),l(_t),l(Wl),l(Ct),l(Bl),l(Vl),l(qt),l(Xl),l(Fl),l(Zt),l(Rl),l(xl),l(zt),l(Ll),l(Nl),l(Qt),l(Hl),l(Al),l(El),l(Vt),l(Yl),l(Pl),l(Sl),l(Dl),l(Ft),l(Kl),l(Ol),l(xt),l(ta),l(ea),l(la),l(aa),l(na),l(Ht),l(sa),l(At),l(ia),l(oa),l(Yt),l(ra),l(pa),l(St),l(ma),l(ua),l(Kt),l(da),l(fa),l(te),l(ha),l(ca),l(ga),l(le),l(ba),l(ya),l(ne),l(Ma),l(Ta),l(ie),l(wa),l($a),l(Ja),l(pe),l(va),l(Ua),l(_a),l(ue),l(Ca),l(ja),l(qa),l(he),l(ka),l(ce),l(Za),l(Ga),l(ge),l(za),l(Ia),l(Qa),l(be),l(Wa),l(Ba),l(Me),l(Va),l(Xa),l(Fa),l(Ra),l($e),l(xa),l(La),l(ve),l(Na),l(Ha),l(_e),l(Aa),l(Ea),l(je),l(Ya),l(Pa),l(Sa),l(Ze),l(Da),l(Ge),l(Ka),l(Oa),l(tn),l(Qe),l(en),l(ln),l(an),l(Ve),l(nn),l(sn),l(on),l(Fe),l(rn),l(pn),l(xe),l(mn),l(un),l(dn),l(He),l(fn),l(hn),l(Ee),l(cn),l(gn),l(Pe),l(bn),l(yn),l(Mn),l(Ke),l(Tn),l(wn),l(tl),l($n),l(S),l(Jn),l(D),l(vn),l(el),l(Un),l(_n),l(al),l(Cn),l(K),l(jn),l(nl),l(qn),l(O),l(kn),l(sl),l(Zn),l(il),l(Gn),l(tt),l(zn),l(rl)),l(n),b(u,t),b(k,t),b(z,t),b(I,t),b(st,t),b(rt,t),b(dt,t),b(gt,t),b(Mt,t),b(wt,t),b(vt,t),b(V,t),b(jt,t),b(kt,t),b(Gt,t),b(It,t),b(Wt,t),b(Bt,t),b(X,t),b(F,t),b(Xt,t),b(Rt,t),b(Lt,t),b(R,t),b(Nt,t),b(x,t),b(Et,t),b(Pt,t),b(Dt,t),b(Ot,t),b(ee,t),b(L,t),b(ae,t),b(se,t),b(oe,t),b(re,t),b(me,t),b(N,t),b(de,t),b(fe,t),b(H,t),b(A,t),b(E,t),b(ye,t),b(Te,t),b(we,t),b(Y,t),b(Je,t),b(Ue,t),b(Ce,t),b(qe,t),b(ke,t),b(ze,t),b(Ie,t),b(We,t),b(Be,t),b(Xe,t),b(P,t),b(Re,t),b(Le,t),b(Ne,t),b(Ae,t),b(Ye,t),b(Se,t),b(De,t),b(Oe,t),b(ll,t)}}}const Vi='{"title":"Quantization","local":"quantization","sections":[{"title":"Quanto","local":"quanto","sections":[],"depth":2},{"title":"AQLM","local":"aqlm","sections":[{"title":"PEFT","local":"peft","sections":[],"depth":3},{"title":"AQLM configurations","local":"aqlm-configurations","sections":[],"depth":3}],"depth":2},{"title":"AWQ","local":"awq","sections":[{"title":"Fused modules","local":"fused-modules","sections":[],"depth":3},{"title":"Exllama-v2 support","local":"exllama-v2-support","sections":[],"depth":3}],"depth":2},{"title":"AutoGPTQ","local":"autogptq","sections":[{"title":"ExLlama","local":"exllama","sections":[],"depth":3}],"depth":2},{"title":"bitsandbytes","local":"bitsandbytes","sections":[{"title":"8-bit","local":"8-bit","sections":[{"title":"Offloading","local":"offloading","sections":[],"depth":4},{"title":"Outlier threshold","local":"outlier-threshold","sections":[],"depth":4},{"title":"Skip module conversion","local":"skip-module-conversion","sections":[],"depth":4},{"title":"Finetuning","local":"finetuning","sections":[],"depth":4}],"depth":3},{"title":"4-bit","local":"4-bit","sections":[{"title":"Compute data type","local":"compute-data-type","sections":[],"depth":4},{"title":"Normal Float 4 (NF4)","local":"normal-float-4-nf4","sections":[],"depth":4},{"title":"Nested quantization","local":"nested-quantization","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Optimum","local":"optimum","sections":[],"depth":2},{"title":"Benchmarks","local":"benchmarks","sections":[{"title":"Fused AWQ modules","local":"fused-awq-modules","sections":[],"depth":3}],"depth":2}],"depth":1}';function Xi($){return di(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ai extends fi{constructor(n){super(),hi(this,n,Xi,Bi,ui,{})}}export{Ai as component};
