import{s as Vt,o as Et,n as Te}from"../chunks/scheduler.25b97de1.js";import{S as qt,i as At,g,s as o,r as u,A as It,h as J,f as l,c as i,j as Gt,u as b,x as w,k as Ft,y as Ht,a as n,v as c,d as y,t as h,w as M}from"../chunks/index.d9030fc9.js";import{T as ct}from"../chunks/Tip.baa67368.js";import{C as B}from"../chunks/CodeBlock.e6cd0d95.js";import{H as R,E as Lt}from"../chunks/EditOnGithub.91d95064.js";import{H as Qt,a as bt}from"../chunks/HfOption.1e589c90.js";function xt(_){let s,f;return s=new B({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGFjY2VsZXJhdGUlMjBiaXRzYW5kYnl0ZXMlM0UwLjM3LjA=",highlighted:"pip install transformers accelerate bitsandbytes&gt;0.37.0",wrap:!1}}),{c(){u(s.$$.fragment)},l(a){b(s.$$.fragment,a)},m(a,p){c(s,a,p),f=!0},p:Te,i(a){f||(y(s.$$.fragment,a),f=!0)},o(a){h(s.$$.fragment,a),f=!1},d(a){M(s,a)}}}function Yt(_){let s,f;return s=new B({props:{code:"cGlwJTIwaW5zdGFsbCUyMGJpdHNhbmRieXRlcyUzRSUzRDAuMzkuMCUwQXBpcCUyMGluc3RhbGwlMjAtLXVwZ3JhZGUlMjBhY2NlbGVyYXRlJTIwdHJhbnNmb3JtZXJz",highlighted:`pip install bitsandbytes&gt;=0.39.0
pip install --upgrade accelerate transformers`,wrap:!1}}),{c(){u(s.$$.fragment)},l(a){b(s.$$.fragment,a)},m(a,p){c(s,a,p),f=!0},p:Te,i(a){f||(y(s.$$.fragment,a),f=!0)},o(a){h(s.$$.fragment,a),f=!1},d(a){M(s,a)}}}function St(_){let s,f,a,p;return s=new bt({props:{id:"bnb",option:"8-bit",$$slots:{default:[xt]},$$scope:{ctx:_}}}),a=new bt({props:{id:"bnb",option:"4-bit",$$slots:{default:[Yt]},$$scope:{ctx:_}}}),{c(){u(s.$$.fragment),f=o(),u(a.$$.fragment)},l(m){b(s.$$.fragment,m),f=i(m),b(a.$$.fragment,m)},m(m,$){c(s,m,$),n(m,f,$),c(a,m,$),p=!0},p(m,$){const v={};$&2&&(v.$$scope={dirty:$,ctx:m}),s.$set(v);const Z={};$&2&&(Z.$$scope={dirty:$,ctx:m}),a.$set(Z)},i(m){p||(y(s.$$.fragment,m),y(a.$$.fragment,m),p=!0)},o(m){h(s.$$.fragment,m),h(a.$$.fragment,m),p=!1},d(m){m&&l(f),M(s,m),M(a,m)}}}function Pt(_){let s,f="Quantizing a model in 8-bit halves the memory-usage, and for large models, set <code>device_map=&quot;auto&quot;</code> to efficiently use the GPUs available:",a,p,m,$,v="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",Z,U,W,C,X='Once a model is quantized to 8-bit, you can’t push the quantized weights to the Hub unless you’re using the latest version of Transformers and bitsandbytes. If you have the latest versions, then you can push the 8-bit model to the Hub with the <a href="/docs/transformers/v4.42.4/en/main_classes/model#transformers.utils.PushToHubMixin.push_to_hub">push_to_hub()</a> method. The quantization config.json file is pushed first, followed by the quantized model weights.',j,r,T;return p=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzhiaXQlM0RUcnVlKSUwQSUwQW1vZGVsXzhiaXQlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>, 
    quantization_config=quantization_config
)`,wrap:!1}}),U=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTIwJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMjAlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIlMEEpJTBBbW9kZWxfOGJpdC5tb2RlbC5kZWNvZGVyLmxheWVycyU1Qi0xJTVELmZpbmFsX2xheWVyX25vcm0ud2VpZ2h0LmR0eXBl",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;facebook/opt-350m&quot;</span>, 
    quantization_config=quantization_config, 
    torch_dtype=torch.float32
)
model_8bit.model.decoder.layers[-<span class="hljs-number">1</span>].final_layer_norm.weight.dtype`,wrap:!1}}),r=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcobG9hZF9pbl84Yml0JTNEVHJ1ZSklMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tNTYwbSUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmJpZ3NjaWVuY2UlMkZibG9vbS01NjBtJTIyKSUwQSUwQW1vZGVsLnB1c2hfdG9faHViKCUyMmJsb29tLTU2MG0tOGJpdCUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>, 
    quantization_config=quantization_config
)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bigscience/bloom-560m&quot;</span>)

model.push_to_hub(<span class="hljs-string">&quot;bloom-560m-8bit&quot;</span>)`,wrap:!1}}),{c(){s=g("p"),s.innerHTML=f,a=o(),u(p.$$.fragment),m=o(),$=g("p"),$.innerHTML=v,Z=o(),u(U.$$.fragment),W=o(),C=g("p"),C.innerHTML=X,j=o(),u(r.$$.fragment)},l(d){s=J(d,"P",{"data-svelte-h":!0}),w(s)!=="svelte-1dz8xeq"&&(s.innerHTML=f),a=i(d),b(p.$$.fragment,d),m=i(d),$=J(d,"P",{"data-svelte-h":!0}),w($)!=="svelte-kfzkum"&&($.innerHTML=v),Z=i(d),b(U.$$.fragment,d),W=i(d),C=J(d,"P",{"data-svelte-h":!0}),w(C)!=="svelte-lmcmjo"&&(C.innerHTML=X),j=i(d),b(r.$$.fragment,d)},m(d,k){n(d,s,k),n(d,a,k),c(p,d,k),n(d,m,k),n(d,$,k),n(d,Z,k),c(U,d,k),n(d,W,k),n(d,C,k),n(d,j,k),c(r,d,k),T=!0},p:Te,i(d){T||(y(p.$$.fragment,d),y(U.$$.fragment,d),y(r.$$.fragment,d),T=!0)},o(d){h(p.$$.fragment,d),h(U.$$.fragment,d),h(r.$$.fragment,d),T=!1},d(d){d&&(l(s),l(a),l(m),l($),l(Z),l(W),l(C),l(j)),M(p,d),M(U,d),M(r,d)}}}function Dt(_){let s,f="Quantizing a model in 4-bit reduces your memory-usage by 4x, and for large models, set <code>device_map=&quot;auto&quot;</code> to efficiently use the GPUs available:",a,p,m,$,v="By default, all the other modules such as <code>torch.nn.LayerNorm</code> are converted to <code>torch.float16</code>. You can change the data type of these modules with the <code>torch_dtype</code> parameter if you want:",Z,U,W,C,X="If you have <code>bitsandbytes&gt;=0.41.3</code>, you can serialize 4-bit models and push them on Hugging Face Hub. Simply call <code>model.push_to_hub()</code> after loading it in 4-bit precision. You can also save the serialized 4-bit models locally with <code>model.save_pretrained()</code> command.",j;return p=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlKSUwQSUwQW1vZGVsXzRiaXQlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),U=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBCaXRzQW5kQnl0ZXNDb25maWclMEElMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpJTBBJTBBbW9kZWxfNGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJmYWNlYm9vayUyRm9wdC0zNTBtJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMkMlMjAlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MzIlMEEpJTBBbW9kZWxfNGJpdC5tb2RlbC5kZWNvZGVyLmxheWVycyU1Qi0xJTVELmZpbmFsX2xheWVyX25vcm0ud2VpZ2h0LmR0eXBl",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>)

model_4bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;facebook/opt-350m&quot;</span>,
    quantization_config=quantization_config, 
    torch_dtype=torch.float32
)
model_4bit.model.decoder.layers[-<span class="hljs-number">1</span>].final_layer_norm.weight.dtype`,wrap:!1}}),{c(){s=g("p"),s.innerHTML=f,a=o(),u(p.$$.fragment),m=o(),$=g("p"),$.innerHTML=v,Z=o(),u(U.$$.fragment),W=o(),C=g("p"),C.innerHTML=X},l(r){s=J(r,"P",{"data-svelte-h":!0}),w(s)!=="svelte-1b6jp1p"&&(s.innerHTML=f),a=i(r),b(p.$$.fragment,r),m=i(r),$=J(r,"P",{"data-svelte-h":!0}),w($)!=="svelte-kfzkum"&&($.innerHTML=v),Z=i(r),b(U.$$.fragment,r),W=i(r),C=J(r,"P",{"data-svelte-h":!0}),w(C)!=="svelte-1pmx2wb"&&(C.innerHTML=X)},m(r,T){n(r,s,T),n(r,a,T),c(p,r,T),n(r,m,T),n(r,$,T),n(r,Z,T),c(U,r,T),n(r,W,T),n(r,C,T),j=!0},p:Te,i(r){j||(y(p.$$.fragment,r),y(U.$$.fragment,r),j=!0)},o(r){h(p.$$.fragment,r),h(U.$$.fragment,r),j=!1},d(r){r&&(l(s),l(a),l(m),l($),l(Z),l(W),l(C)),M(p,r),M(U,r)}}}function Kt(_){let s,f,a,p;return s=new bt({props:{id:"bnb",option:"8-bit",$$slots:{default:[Pt]},$$scope:{ctx:_}}}),a=new bt({props:{id:"bnb",option:"4-bit",$$slots:{default:[Dt]},$$scope:{ctx:_}}}),{c(){u(s.$$.fragment),f=o(),u(a.$$.fragment)},l(m){b(s.$$.fragment,m),f=i(m),b(a.$$.fragment,m)},m(m,$){c(s,m,$),n(m,f,$),c(a,m,$),p=!0},p(m,$){const v={};$&2&&(v.$$scope={dirty:$,ctx:m}),s.$set(v);const Z={};$&2&&(Z.$$scope={dirty:$,ctx:m}),a.$set(Z)},i(m){p||(y(s.$$.fragment,m),y(a.$$.fragment,m),p=!0)},o(m){h(s.$$.fragment,m),h(a.$$.fragment,m),p=!1},d(m){m&&l(f),M(s,m),M(a,m)}}}function Ot(_){let s,f="Training with 8-bit and 4-bit weights are only supported for training <em>extra</em> parameters.";return{c(){s=g("p"),s.innerHTML=f},l(a){s=J(a,"P",{"data-svelte-h":!0}),w(s)!=="svelte-of9sym"&&(s.innerHTML=f)},m(a,p){n(a,s,p)},p:Te,d(a){a&&l(s)}}}function el(_){let s,f='Learn more about the details of 8-bit quantization in this <a href="https://huggingface.co/blog/hf-bitsandbytes-integration" rel="nofollow">blog post</a>!';return{c(){s=g("p"),s.innerHTML=f},l(a){s=J(a,"P",{"data-svelte-h":!0}),w(s)!=="svelte-1bb05fp"&&(s.innerHTML=f)},m(a,p){n(a,s,p)},p:Te,d(a){a&&l(s)}}}function tl(_){let s,f='Try 4-bit quantization in this <a href="https://colab.research.google.com/drive/1ge2F1QSK8Q7h0hn3YKuBCOAS0bK8E0wf" rel="nofollow">notebook</a> and learn more about it’s details in this <a href="https://huggingface.co/blog/4bit-transformers-bitsandbytes" rel="nofollow">blog post</a>.';return{c(){s=g("p"),s.innerHTML=f},l(a){s=J(a,"P",{"data-svelte-h":!0}),w(s)!=="svelte-rme94c"&&(s.innerHTML=f)},m(a,p){n(a,s,p)},p:Te,d(a){a&&l(s)}}}function ll(_){let s,f,a,p,m,$,v,Z='<a href="https://github.com/TimDettmers/bitsandbytes" rel="nofollow">bitsandbytes</a> is the easiest option for quantizing a model to 8 and 4-bit. 8-bit quantization multiplies outliers in fp16 with non-outliers in int8, converts the non-outlier values back to fp16, and then adds them together to return the weights in fp16. This reduces the degradative effect outlier values have on a model’s performance. 4-bit quantization compresses a model even further, and it is commonly used with <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> to finetune quantized LLMs.',U,W,C="To use bitsandbytes, make sure you have the following libraries installed:",X,j,r,T,d='Now you can quantize a model by passing a <code>BitsAndBytesConfig</code> to <a href="/docs/transformers/v4.42.4/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method. This works for any model in any modality, as long as it supports loading with Accelerate and contains <code>torch.nn.Linear</code> layers.',k,z,Ue,N,ve,Q,yt="You can check your memory footprint with the <code>get_memory_footprint</code> method:",Ze,V,Ce,E,ht='Quantized models can be loaded from the <a href="/docs/transformers/v4.42.4/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method without needing to specify the <code>load_in_8bit</code> or <code>load_in_4bit</code> parameters:',je,q,Be,A,We,G,ke,I,Mt="This section explores some of the specific features of 8-bit models, such as offloading, outlier thresholds, skipping module conversion, and finetuning.",Re,H,Xe,L,$t='8-bit models can offload weights between the CPU and GPU to support fitting very large models into memory. The weights dispatched to the CPU are actually stored in <strong>float32</strong>, and aren’t converted to 8-bit. For example, to enable offloading for the <a href="https://huggingface.co/bigscience/bloom-1b7" rel="nofollow">bigscience/bloom-1b7</a> model, start by creating a <a href="/docs/transformers/v4.42.4/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',ze,x,Ne,Y,gt="Design a custom device map to fit everything on your GPU except for the <code>lm_head</code>, which you’ll dispatch to the CPU:",Ge,S,Fe,P,Jt="Now load your model with the custom <code>device_map</code> and <code>quantization_config</code>:",Qe,D,Ve,K,Ee,O,wt="An “outlier” is a hidden state value greater than a certain threshold, and these values are computed in fp16. While the values are usually normally distributed ([-3.5, 3.5]), this distribution can be very different for large models ([-60, 6] or [6, 60]). 8-bit quantization works well for values ~5, but beyond that, there is a significant performance penalty. A good default threshold value is 6, but a lower threshold may be needed for more unstable models (small models or finetuning).",qe,ee,Tt='To find the best threshold for your model, we recommend experimenting with the <code>llm_int8_threshold</code> parameter in <a href="/docs/transformers/v4.42.4/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Ae,te,Ie,le,He,ne,_t='For some models, like <a href="model_doc/jukebox">Jukebox</a>, you don’t need to quantize every module to 8-bit which can actually cause instability. With Jukebox, there are several <code>lm_head</code> modules that should be skipped using the <code>llm_int8_skip_modules</code> parameter in <a href="/docs/transformers/v4.42.4/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',Le,se,xe,ae,Ye,oe,Ut='With the <a href="https://github.com/huggingface/peft" rel="nofollow">PEFT</a> library, you can finetune large models like <a href="https://huggingface.co/google/flan-t5-large" rel="nofollow">flan-t5-large</a> and <a href="https://huggingface.co/facebook/opt-6.7b" rel="nofollow">facebook/opt-6.7b</a> with 8-bit quantization. You don’t need to pass the <code>device_map</code> parameter for training because it’ll automatically load your model on a GPU. However, you can still customize the device map with the <code>device_map</code> parameter if you want to (<code>device_map=&quot;auto&quot;</code> should only be used for inference).',Se,ie,Pe,F,De,me,vt="This section explores some of the specific features of 4-bit models, such as changing the compute data type, using the Normal Float 4 (NF4) data type, and using nested quantization.",Ke,re,Oe,pe,Zt='To speedup computation, you can change the data type from float32 (the default value) to bf16 using the <code>bnb_4bit_compute_dtype</code> parameter in <a href="/docs/transformers/v4.42.4/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',et,fe,tt,de,lt,ue,Ct='NF4 is a 4-bit data type from the <a href="https://hf.co/papers/2305.14314" rel="nofollow">QLoRA</a> paper, adapted for weights initialized from a normal distribution. You should use NF4 for training 4-bit base models. This can be configured with the <code>bnb_4bit_quant_type</code> parameter in the <a href="/docs/transformers/v4.42.4/en/main_classes/quantization#transformers.BitsAndBytesConfig">BitsAndBytesConfig</a>:',nt,be,st,ce,jt="For inference, the <code>bnb_4bit_quant_type</code> does not have a huge impact on performance. However, to remain consistent with the model weights, you should use the <code>bnb_4bit_compute_dtype</code> and <code>torch_dtype</code> values.",at,ye,ot,he,Bt='Nested quantization is a technique that can save additional memory at no additional performance cost. This feature performs a second quantization of the already quantized weights to save an addition 0.4 bits/parameter. For example, with nested quantization, you can finetune a <a href="https://huggingface.co/meta-llama/Llama-2-13b" rel="nofollow">Llama-13b</a> model on a 16GB NVIDIA T4 GPU with a sequence length of 1024, a batch size of 1, and enabling gradient accumulation with 4 steps.',it,Me,mt,$e,rt,ge,Wt="Once quantized, you can dequantize the model to the original precision but this might result in a small quality loss of the model. Make sure you have enough GPU RAM to fit the dequantized model.",pt,Je,ft,we,dt,_e,ut;return m=new R({props:{title:"bitsandbytes",local:"bitsandbytes",headingTag:"h1"}}),j=new Qt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[St]},$$scope:{ctx:_}}}),z=new Qt({props:{id:"bnb",options:["8-bit","4-bit"],$$slots:{default:[Kt]},$$scope:{ctx:_}}}),N=new ct({props:{warning:!0,$$slots:{default:[Ot]},$$scope:{ctx:_}}}),V=new B({props:{code:"cHJpbnQobW9kZWwuZ2V0X21lbW9yeV9mb290cHJpbnQoKSk=",highlighted:'<span class="hljs-built_in">print</span>(model.get_memory_footprint())',wrap:!1}}),q=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMiU3QnlvdXJfdXNlcm5hbWUlN0QlMkZibG9vbS01NjBtLThiaXQlMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;{your_username}/bloom-560m-8bit&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),A=new R({props:{title:"8-bit (LLM.int8() algorithm)",local:"8-bit-llmint8-algorithm",headingTag:"h2"}}),G=new ct({props:{$$slots:{default:[el]},$$scope:{ctx:_}}}),H=new R({props:{title:"Offloading",local:"offloading",headingTag:"h3"}}),x=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsbG1faW50OF9lbmFibGVfZnAzMl9jcHVfb2ZmbG9hZCUzRFRydWUp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(llm_int8_enable_fp32_cpu_offload=<span class="hljs-literal">True</span>)`,wrap:!1}}),S=new B({props:{code:"ZGV2aWNlX21hcCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLndvcmRfZW1iZWRkaW5ncyUyMiUzQSUyMDAlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci53b3JkX2VtYmVkZGluZ3NfbGF5ZXJub3JtJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMmxtX2hlYWQlMjIlM0ElMjAlMjJjcHUlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJ0cmFuc2Zvcm1lci5oJTIyJTNBJTIwMCUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYW5zZm9ybWVyLmxuX2YlMjIlM0ElMjAwJTJDJTBBJTdE",highlighted:`device_map = {
    <span class="hljs-string">&quot;transformer.word_embeddings&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.word_embeddings_layernorm&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;lm_head&quot;</span>: <span class="hljs-string">&quot;cpu&quot;</span>,
    <span class="hljs-string">&quot;transformer.h&quot;</span>: <span class="hljs-number">0</span>,
    <span class="hljs-string">&quot;transformer.ln_f&quot;</span>: <span class="hljs-number">0</span>,
}`,wrap:!1}}),D=new B({props:{code:"bW9kZWxfOGJpdCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`model_8bit = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>,
    device_map=device_map,
    quantization_config=quantization_config,
)`,wrap:!1}}),K=new R({props:{title:"Outlier threshold",local:"outlier-threshold",headingTag:"h3"}}),te=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJiaWdzY2llbmNlJTJGYmxvb20tMWI3JTIyJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyglMEElMjAlMjAlMjAlMjBsbG1faW50OF90aHJlc2hvbGQlM0QxMCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_threshold=<span class="hljs-number">10</span>,
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=device_map,
    quantization_config=quantization_config,
)`,wrap:!1}}),le=new R({props:{title:"Skip module conversion",local:"skip-module-conversion",headingTag:"h3"}}),se=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyYmlnc2NpZW5jZSUyRmJsb29tLTFiNyUyMiUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbGxtX2ludDhfc2tpcF9tb2R1bGVzJTNEJTVCJTIybG1faGVhZCUyMiU1RCUyQyUwQSklMEElMEFtb2RlbF84Yml0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX2lkJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig

model_id = <span class="hljs-string">&quot;bigscience/bloom-1b7&quot;</span>

quantization_config = BitsAndBytesConfig(
    llm_int8_skip_modules=[<span class="hljs-string">&quot;lm_head&quot;</span>],
)

model_8bit = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config,
)`,wrap:!1}}),ae=new R({props:{title:"Finetuning",local:"finetuning",headingTag:"h3"}}),ie=new R({props:{title:"4-bit (QLoRA algorithm)",local:"4-bit-qlora-algorithm",headingTag:"h2"}}),F=new ct({props:{$$slots:{default:[tl]},$$scope:{ctx:_}}}),re=new R({props:{title:"Compute data type",local:"compute-data-type",headingTag:"h3"}}),fe=new B({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16)`,wrap:!1}}),de=new R({props:{title:"Normal Float 4 (NF4)",local:"normal-float-4-nf4",headingTag:"h3"}}),be=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQW5mNF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3F1YW50X3R5cGUlM0QlMjJuZjQlMjIlMkMlMEEpJTBBJTBBbW9kZWxfbmY0JTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRG5mNF9jb25maWcp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

nf4_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>,
)

model_nf4 = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=nf4_config)`,wrap:!1}}),ye=new R({props:{title:"Nested quantization",local:"nested-quantization",headingTag:"h3"}}),Me=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQSUwQWRvdWJsZV9xdWFudF9jb25maWclMjAlM0QlMjBCaXRzQW5kQnl0ZXNDb25maWcoJTBBJTIwJTIwJTIwJTIwbG9hZF9pbl80Yml0JTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlJTJDJTBBKSUwQSUwQW1vZGVsX2RvdWJsZV9xdWFudCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi0xM2IlMjIlMkMlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEZG91YmxlX3F1YW50X2NvbmZpZyk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BitsAndBytesConfig

double_quant_config = BitsAndBytesConfig(
    load_in_4bit=<span class="hljs-literal">True</span>,
    bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>,
)

model_double_quant = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-13b&quot;</span>, quantization_config=double_quant_config)`,wrap:!1}}),$e=new R({props:{title:"Dequantizing bitsandbytes models",local:"dequantizing-bitsandbytes-models",headingTag:"h2"}}),Je=new B({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyZmFjZWJvb2slMkZvcHQtMTI1bSUyMiUwQSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fNGJpdCUzRFRydWUpKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQW1vZGVsLmRlcXVhbnRpemUoKSUwQSUwQXRleHQlMjAlM0QlMjB0b2tlbml6ZXIoJTIySGVsbG8lMjBteSUyMG5hbWUlMjBpcyUyMiUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKDApJTBBJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKip0ZXh0KSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0JTVCMCU1RCkp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig, AutoTokenizer

model_id = <span class="hljs-string">&quot;facebook/opt-125m&quot;</span>

model = AutoModelForCausalLM.from_pretrained(model_id, BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>))
tokenizer = AutoTokenizer.from_pretrained(model_id)

model.dequantize()

text = tokenizer(<span class="hljs-string">&quot;Hello my name is&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-number">0</span>)

out = model.generate(**text)
<span class="hljs-built_in">print</span>(tokenizer.decode(out[<span class="hljs-number">0</span>]))`,wrap:!1}}),we=new Lt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/bitsandbytes.md"}}),{c(){s=g("meta"),f=o(),a=g("p"),p=o(),u(m.$$.fragment),$=o(),v=g("p"),v.innerHTML=Z,U=o(),W=g("p"),W.textContent=C,X=o(),u(j.$$.fragment),r=o(),T=g("p"),T.innerHTML=d,k=o(),u(z.$$.fragment),Ue=o(),u(N.$$.fragment),ve=o(),Q=g("p"),Q.innerHTML=yt,Ze=o(),u(V.$$.fragment),Ce=o(),E=g("p"),E.innerHTML=ht,je=o(),u(q.$$.fragment),Be=o(),u(A.$$.fragment),We=o(),u(G.$$.fragment),ke=o(),I=g("p"),I.textContent=Mt,Re=o(),u(H.$$.fragment),Xe=o(),L=g("p"),L.innerHTML=$t,ze=o(),u(x.$$.fragment),Ne=o(),Y=g("p"),Y.innerHTML=gt,Ge=o(),u(S.$$.fragment),Fe=o(),P=g("p"),P.innerHTML=Jt,Qe=o(),u(D.$$.fragment),Ve=o(),u(K.$$.fragment),Ee=o(),O=g("p"),O.textContent=wt,qe=o(),ee=g("p"),ee.innerHTML=Tt,Ae=o(),u(te.$$.fragment),Ie=o(),u(le.$$.fragment),He=o(),ne=g("p"),ne.innerHTML=_t,Le=o(),u(se.$$.fragment),xe=o(),u(ae.$$.fragment),Ye=o(),oe=g("p"),oe.innerHTML=Ut,Se=o(),u(ie.$$.fragment),Pe=o(),u(F.$$.fragment),De=o(),me=g("p"),me.textContent=vt,Ke=o(),u(re.$$.fragment),Oe=o(),pe=g("p"),pe.innerHTML=Zt,et=o(),u(fe.$$.fragment),tt=o(),u(de.$$.fragment),lt=o(),ue=g("p"),ue.innerHTML=Ct,nt=o(),u(be.$$.fragment),st=o(),ce=g("p"),ce.innerHTML=jt,at=o(),u(ye.$$.fragment),ot=o(),he=g("p"),he.innerHTML=Bt,it=o(),u(Me.$$.fragment),mt=o(),u($e.$$.fragment),rt=o(),ge=g("p"),ge.textContent=Wt,pt=o(),u(Je.$$.fragment),ft=o(),u(we.$$.fragment),dt=o(),_e=g("p"),this.h()},l(e){const t=It("svelte-u9bgzb",document.head);s=J(t,"META",{name:!0,content:!0}),t.forEach(l),f=i(e),a=J(e,"P",{}),Gt(a).forEach(l),p=i(e),b(m.$$.fragment,e),$=i(e),v=J(e,"P",{"data-svelte-h":!0}),w(v)!=="svelte-5a92xi"&&(v.innerHTML=Z),U=i(e),W=J(e,"P",{"data-svelte-h":!0}),w(W)!=="svelte-gf36q7"&&(W.textContent=C),X=i(e),b(j.$$.fragment,e),r=i(e),T=J(e,"P",{"data-svelte-h":!0}),w(T)!=="svelte-140z17t"&&(T.innerHTML=d),k=i(e),b(z.$$.fragment,e),Ue=i(e),b(N.$$.fragment,e),ve=i(e),Q=J(e,"P",{"data-svelte-h":!0}),w(Q)!=="svelte-1bxp667"&&(Q.innerHTML=yt),Ze=i(e),b(V.$$.fragment,e),Ce=i(e),E=J(e,"P",{"data-svelte-h":!0}),w(E)!=="svelte-43govc"&&(E.innerHTML=ht),je=i(e),b(q.$$.fragment,e),Be=i(e),b(A.$$.fragment,e),We=i(e),b(G.$$.fragment,e),ke=i(e),I=J(e,"P",{"data-svelte-h":!0}),w(I)!=="svelte-1yx1x2g"&&(I.textContent=Mt),Re=i(e),b(H.$$.fragment,e),Xe=i(e),L=J(e,"P",{"data-svelte-h":!0}),w(L)!=="svelte-1rruy6k"&&(L.innerHTML=$t),ze=i(e),b(x.$$.fragment,e),Ne=i(e),Y=J(e,"P",{"data-svelte-h":!0}),w(Y)!=="svelte-15oy7am"&&(Y.innerHTML=gt),Ge=i(e),b(S.$$.fragment,e),Fe=i(e),P=J(e,"P",{"data-svelte-h":!0}),w(P)!=="svelte-k1g8c2"&&(P.innerHTML=Jt),Qe=i(e),b(D.$$.fragment,e),Ve=i(e),b(K.$$.fragment,e),Ee=i(e),O=J(e,"P",{"data-svelte-h":!0}),w(O)!=="svelte-ur5rgd"&&(O.textContent=wt),qe=i(e),ee=J(e,"P",{"data-svelte-h":!0}),w(ee)!=="svelte-1rchmcy"&&(ee.innerHTML=Tt),Ae=i(e),b(te.$$.fragment,e),Ie=i(e),b(le.$$.fragment,e),He=i(e),ne=J(e,"P",{"data-svelte-h":!0}),w(ne)!=="svelte-e3hcsh"&&(ne.innerHTML=_t),Le=i(e),b(se.$$.fragment,e),xe=i(e),b(ae.$$.fragment,e),Ye=i(e),oe=J(e,"P",{"data-svelte-h":!0}),w(oe)!=="svelte-4nvx4q"&&(oe.innerHTML=Ut),Se=i(e),b(ie.$$.fragment,e),Pe=i(e),b(F.$$.fragment,e),De=i(e),me=J(e,"P",{"data-svelte-h":!0}),w(me)!=="svelte-7ob7j"&&(me.textContent=vt),Ke=i(e),b(re.$$.fragment,e),Oe=i(e),pe=J(e,"P",{"data-svelte-h":!0}),w(pe)!=="svelte-12vpx5l"&&(pe.innerHTML=Zt),et=i(e),b(fe.$$.fragment,e),tt=i(e),b(de.$$.fragment,e),lt=i(e),ue=J(e,"P",{"data-svelte-h":!0}),w(ue)!=="svelte-jdr043"&&(ue.innerHTML=Ct),nt=i(e),b(be.$$.fragment,e),st=i(e),ce=J(e,"P",{"data-svelte-h":!0}),w(ce)!=="svelte-1qoc2ct"&&(ce.innerHTML=jt),at=i(e),b(ye.$$.fragment,e),ot=i(e),he=J(e,"P",{"data-svelte-h":!0}),w(he)!=="svelte-l0u9eq"&&(he.innerHTML=Bt),it=i(e),b(Me.$$.fragment,e),mt=i(e),b($e.$$.fragment,e),rt=i(e),ge=J(e,"P",{"data-svelte-h":!0}),w(ge)!=="svelte-1gygypd"&&(ge.textContent=Wt),pt=i(e),b(Je.$$.fragment,e),ft=i(e),b(we.$$.fragment,e),dt=i(e),_e=J(e,"P",{}),Gt(_e).forEach(l),this.h()},h(){Ft(s,"name","hf:doc:metadata"),Ft(s,"content",nl)},m(e,t){Ht(document.head,s),n(e,f,t),n(e,a,t),n(e,p,t),c(m,e,t),n(e,$,t),n(e,v,t),n(e,U,t),n(e,W,t),n(e,X,t),c(j,e,t),n(e,r,t),n(e,T,t),n(e,k,t),c(z,e,t),n(e,Ue,t),c(N,e,t),n(e,ve,t),n(e,Q,t),n(e,Ze,t),c(V,e,t),n(e,Ce,t),n(e,E,t),n(e,je,t),c(q,e,t),n(e,Be,t),c(A,e,t),n(e,We,t),c(G,e,t),n(e,ke,t),n(e,I,t),n(e,Re,t),c(H,e,t),n(e,Xe,t),n(e,L,t),n(e,ze,t),c(x,e,t),n(e,Ne,t),n(e,Y,t),n(e,Ge,t),c(S,e,t),n(e,Fe,t),n(e,P,t),n(e,Qe,t),c(D,e,t),n(e,Ve,t),c(K,e,t),n(e,Ee,t),n(e,O,t),n(e,qe,t),n(e,ee,t),n(e,Ae,t),c(te,e,t),n(e,Ie,t),c(le,e,t),n(e,He,t),n(e,ne,t),n(e,Le,t),c(se,e,t),n(e,xe,t),c(ae,e,t),n(e,Ye,t),n(e,oe,t),n(e,Se,t),c(ie,e,t),n(e,Pe,t),c(F,e,t),n(e,De,t),n(e,me,t),n(e,Ke,t),c(re,e,t),n(e,Oe,t),n(e,pe,t),n(e,et,t),c(fe,e,t),n(e,tt,t),c(de,e,t),n(e,lt,t),n(e,ue,t),n(e,nt,t),c(be,e,t),n(e,st,t),n(e,ce,t),n(e,at,t),c(ye,e,t),n(e,ot,t),n(e,he,t),n(e,it,t),c(Me,e,t),n(e,mt,t),c($e,e,t),n(e,rt,t),n(e,ge,t),n(e,pt,t),c(Je,e,t),n(e,ft,t),c(we,e,t),n(e,dt,t),n(e,_e,t),ut=!0},p(e,[t]){const kt={};t&2&&(kt.$$scope={dirty:t,ctx:e}),j.$set(kt);const Rt={};t&2&&(Rt.$$scope={dirty:t,ctx:e}),z.$set(Rt);const Xt={};t&2&&(Xt.$$scope={dirty:t,ctx:e}),N.$set(Xt);const zt={};t&2&&(zt.$$scope={dirty:t,ctx:e}),G.$set(zt);const Nt={};t&2&&(Nt.$$scope={dirty:t,ctx:e}),F.$set(Nt)},i(e){ut||(y(m.$$.fragment,e),y(j.$$.fragment,e),y(z.$$.fragment,e),y(N.$$.fragment,e),y(V.$$.fragment,e),y(q.$$.fragment,e),y(A.$$.fragment,e),y(G.$$.fragment,e),y(H.$$.fragment,e),y(x.$$.fragment,e),y(S.$$.fragment,e),y(D.$$.fragment,e),y(K.$$.fragment,e),y(te.$$.fragment,e),y(le.$$.fragment,e),y(se.$$.fragment,e),y(ae.$$.fragment,e),y(ie.$$.fragment,e),y(F.$$.fragment,e),y(re.$$.fragment,e),y(fe.$$.fragment,e),y(de.$$.fragment,e),y(be.$$.fragment,e),y(ye.$$.fragment,e),y(Me.$$.fragment,e),y($e.$$.fragment,e),y(Je.$$.fragment,e),y(we.$$.fragment,e),ut=!0)},o(e){h(m.$$.fragment,e),h(j.$$.fragment,e),h(z.$$.fragment,e),h(N.$$.fragment,e),h(V.$$.fragment,e),h(q.$$.fragment,e),h(A.$$.fragment,e),h(G.$$.fragment,e),h(H.$$.fragment,e),h(x.$$.fragment,e),h(S.$$.fragment,e),h(D.$$.fragment,e),h(K.$$.fragment,e),h(te.$$.fragment,e),h(le.$$.fragment,e),h(se.$$.fragment,e),h(ae.$$.fragment,e),h(ie.$$.fragment,e),h(F.$$.fragment,e),h(re.$$.fragment,e),h(fe.$$.fragment,e),h(de.$$.fragment,e),h(be.$$.fragment,e),h(ye.$$.fragment,e),h(Me.$$.fragment,e),h($e.$$.fragment,e),h(Je.$$.fragment,e),h(we.$$.fragment,e),ut=!1},d(e){e&&(l(f),l(a),l(p),l($),l(v),l(U),l(W),l(X),l(r),l(T),l(k),l(Ue),l(ve),l(Q),l(Ze),l(Ce),l(E),l(je),l(Be),l(We),l(ke),l(I),l(Re),l(Xe),l(L),l(ze),l(Ne),l(Y),l(Ge),l(Fe),l(P),l(Qe),l(Ve),l(Ee),l(O),l(qe),l(ee),l(Ae),l(Ie),l(He),l(ne),l(Le),l(xe),l(Ye),l(oe),l(Se),l(Pe),l(De),l(me),l(Ke),l(Oe),l(pe),l(et),l(tt),l(lt),l(ue),l(nt),l(st),l(ce),l(at),l(ot),l(he),l(it),l(mt),l(rt),l(ge),l(pt),l(ft),l(dt),l(_e)),l(s),M(m,e),M(j,e),M(z,e),M(N,e),M(V,e),M(q,e),M(A,e),M(G,e),M(H,e),M(x,e),M(S,e),M(D,e),M(K,e),M(te,e),M(le,e),M(se,e),M(ae,e),M(ie,e),M(F,e),M(re,e),M(fe,e),M(de,e),M(be,e),M(ye,e),M(Me,e),M($e,e),M(Je,e),M(we,e)}}}const nl='{"title":"bitsandbytes","local":"bitsandbytes","sections":[{"title":"8-bit (LLM.int8() algorithm)","local":"8-bit-llmint8-algorithm","sections":[{"title":"Offloading","local":"offloading","sections":[],"depth":3},{"title":"Outlier threshold","local":"outlier-threshold","sections":[],"depth":3},{"title":"Skip module conversion","local":"skip-module-conversion","sections":[],"depth":3},{"title":"Finetuning","local":"finetuning","sections":[],"depth":3}],"depth":2},{"title":"4-bit (QLoRA algorithm)","local":"4-bit-qlora-algorithm","sections":[{"title":"Compute data type","local":"compute-data-type","sections":[],"depth":3},{"title":"Normal Float 4 (NF4)","local":"normal-float-4-nf4","sections":[],"depth":3},{"title":"Nested quantization","local":"nested-quantization","sections":[],"depth":3}],"depth":2},{"title":"Dequantizing bitsandbytes models","local":"dequantizing-bitsandbytes-models","sections":[],"depth":2}],"depth":1}';function sl(_){return Et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class fl extends qt{constructor(s){super(),At(this,s,sl,ll,Vt,{})}}export{fl as component};
