import{s as He,n as Je,o as Ue}from"../chunks/scheduler.25b97de1.js";import{S as ze,i as qe,g as a,s as n,r as f,A as Ie,h as r,f as l,c as o,j as Qe,u,x as s,k as Ge,y as Se,a as i,v as m,d,t as h,w as c}from"../chunks/index.d9030fc9.js";import{C as Ze}from"../chunks/CodeBlock.e6cd0d95.js";import{H as B,E as Xe}from"../chunks/EditOnGithub.91d95064.js";function Be(he){let p,E,S,P,g,F,w,ce=`The GGUF file format is used to store models for inference with <a href="https://github.com/ggerganov/ggml" rel="nofollow">GGML</a> and other
libraries that depend on it, like the very popular <a href="https://github.com/ggerganov/llama.cpp" rel="nofollow">llama.cpp</a> or
<a href="https://github.com/ggerganov/whisper.cpp" rel="nofollow">whisper.cpp</a>.`,V,b,ge=`It is a file format <a href="https://huggingface.co/docs/hub/en/gguf" rel="nofollow">supported by the Hugging Face Hub</a> with features
allowing for quick inspection of tensors and metadata within the file.`,j,T,we=`This file format is designed as a “single-file-format” where a single file usually contains both the configuration
attributes, the tokenizer vocabulary and other attributes, as well as all tensors to be loaded in the model. These
files come in different formats according to the quantization type of the file. We briefly go over some of them
<a href="https://huggingface.co/docs/hub/en/gguf#quantization-types" rel="nofollow">here</a>.`,Y,y,R,v,be=`We have added the ability to load <code>gguf</code> files within <code>transformers</code> in order to offer further training/fine-tuning
capabilities to gguf models, before converting back those models to <code>gguf</code> to use within the <code>ggml</code> ecosystem. When
loading a model, we first dequantize it to fp32, before loading the weights to be used in PyTorch.`,N,$,Te=`<p>[!NOTE]
The support is still very exploratory and we welcome contributions in order to solidify it across quantization types
and model architectures.</p>`,K,_,ye="For now, here are the supported model architectures and quantization types:",A,M,O,x,ve=`The initial supported quantization types are decided according to the popular quantized files that have been shared
on the Hub.`,D,C,$e="<li>F32</li> <li>F16</li> <li>BF16</li> <li>Q4_0</li> <li>Q4_1</li> <li>Q5_0</li> <li>Q5_1</li> <li>Q8_0</li> <li>Q2_K</li> <li>Q3_K</li> <li>Q4_K</li> <li>Q5_K</li> <li>Q6_K</li> <li>IQ1_S</li> <li>IQ1_M</li> <li>IQ2_XXS</li> <li>IQ2_XS</li> <li>IQ2_S</li> <li>IQ3_XXS</li> <li>IQ3_S</li> <li>IQ4_XS</li> <li>IQ4_NL</li>",ee,k,_e=`<p>[!NOTE]
To support gguf dequantization, <code>gguf&gt;=0.10.0</code> installation is required.</p>`,te,L,le,W,Me="For now the supported model architectures are the architectures that have been very popular on the Hub, namely:",ie,Q,xe="<li>LLaMa</li> <li>Mistral</li> <li>Qwen2</li> <li>Qwen2Moe</li> <li>Phi3</li>",ne,G,oe,Z,Ce=`In order to load <code>gguf</code> files in <code>transformers</code>, you should specify the <code>gguf_file</code> argument to the <code>from_pretrained</code>
methods of both tokenizers and models. Here is how one would load a tokenizer and a model, which can be loaded
from the exact same file:`,ae,H,re,J,ke=`Now you have access to the full, unquantized version of the model in the PyTorch ecosystem, where you can combine it
with a plethora of other tools.`,se,U,Le=`In order to convert back to a <code>gguf</code> file, we recommend using the
<a href="https://github.com/ggerganov/llama.cpp/blob/master/convert-hf-to-gguf.py" rel="nofollow"><code>convert-hf-to-gguf.py</code> file</a> from llama.cpp.`,pe,z,We="Here’s how you would complete the script above to save the model and export it back to <code>gguf</code>:",fe,q,ue,I,me,X,de;return g=new B({props:{title:"GGUF and interaction with Transformers",local:"gguf-and-interaction-with-transformers",headingTag:"h1"}}),y=new B({props:{title:"Support within Transformers",local:"support-within-transformers",headingTag:"h2"}}),M=new B({props:{title:"Supported quantization types",local:"supported-quantization-types",headingTag:"h3"}}),L=new B({props:{title:"Supported model architectures",local:"supported-model-architectures",headingTag:"h3"}}),G=new B({props:{title:"Example usage",local:"example-usage",headingTag:"h2"}}),H=new Ze({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyVGhlQmxva2UlMkZUaW55TGxhbWEtMS4xQi1DaGF0LXYxLjAtR0dVRiUyMiUwQWZpbGVuYW1lJTIwJTNEJTIwJTIydGlueWxsYW1hLTEuMWItY2hhdC12MS4wLlE2X0suZ2d1ZiUyMiUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZ2d1Zl9maWxlJTNEZmlsZW5hbWUpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBnZ3VmX2ZpbGUlM0RmaWxlbmFtZSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

model_id = <span class="hljs-string">&quot;TheBloke/TinyLlama-1.1B-Chat-v1.0-GGUF&quot;</span>
filename = <span class="hljs-string">&quot;tinyllama-1.1b-chat-v1.0.Q6_K.gguf&quot;</span>

tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename)
model = AutoModelForCausalLM.from_pretrained(model_id, gguf_file=filename)`,wrap:!1}}),q=new Ze({props:{code:"dG9rZW5pemVyLnNhdmVfcHJldHJhaW5lZCgnZGlyZWN0b3J5JyklMEFtb2RlbC5zYXZlX3ByZXRyYWluZWQoJ2RpcmVjdG9yeScpJTBBJTBBIXB5dGhvbiUyMCUyNCU3QnBhdGhfdG9fbGxhbWFfY3BwJTdEJTJGY29udmVydC1oZi10by1nZ3VmLnB5JTIwJTI0JTdCZGlyZWN0b3J5JTdE",highlighted:`tokenizer.save_pretrained(<span class="hljs-string">&#x27;directory&#x27;</span>)
model.save_pretrained(<span class="hljs-string">&#x27;directory&#x27;</span>)

!python \${path_to_llama_cpp}/convert-hf-to-gguf.py \${directory}`,wrap:!1}}),I=new Xe({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/gguf.md"}}),{c(){p=a("meta"),E=n(),S=a("p"),P=n(),f(g.$$.fragment),F=n(),w=a("p"),w.innerHTML=ce,V=n(),b=a("p"),b.innerHTML=ge,j=n(),T=a("p"),T.innerHTML=we,Y=n(),f(y.$$.fragment),R=n(),v=a("p"),v.innerHTML=be,N=n(),$=a("blockquote"),$.innerHTML=Te,K=n(),_=a("p"),_.textContent=ye,A=n(),f(M.$$.fragment),O=n(),x=a("p"),x.textContent=ve,D=n(),C=a("ul"),C.innerHTML=$e,ee=n(),k=a("blockquote"),k.innerHTML=_e,te=n(),f(L.$$.fragment),le=n(),W=a("p"),W.textContent=Me,ie=n(),Q=a("ul"),Q.innerHTML=xe,ne=n(),f(G.$$.fragment),oe=n(),Z=a("p"),Z.innerHTML=Ce,ae=n(),f(H.$$.fragment),re=n(),J=a("p"),J.textContent=ke,se=n(),U=a("p"),U.innerHTML=Le,pe=n(),z=a("p"),z.innerHTML=We,fe=n(),f(q.$$.fragment),ue=n(),f(I.$$.fragment),me=n(),X=a("p"),this.h()},l(e){const t=Ie("svelte-u9bgzb",document.head);p=r(t,"META",{name:!0,content:!0}),t.forEach(l),E=o(e),S=r(e,"P",{}),Qe(S).forEach(l),P=o(e),u(g.$$.fragment,e),F=o(e),w=r(e,"P",{"data-svelte-h":!0}),s(w)!=="svelte-117klwz"&&(w.innerHTML=ce),V=o(e),b=r(e,"P",{"data-svelte-h":!0}),s(b)!=="svelte-14alsnt"&&(b.innerHTML=ge),j=o(e),T=r(e,"P",{"data-svelte-h":!0}),s(T)!=="svelte-1lcntb8"&&(T.innerHTML=we),Y=o(e),u(y.$$.fragment,e),R=o(e),v=r(e,"P",{"data-svelte-h":!0}),s(v)!=="svelte-1fcaojo"&&(v.innerHTML=be),N=o(e),$=r(e,"BLOCKQUOTE",{"data-svelte-h":!0}),s($)!=="svelte-13g437o"&&($.innerHTML=Te),K=o(e),_=r(e,"P",{"data-svelte-h":!0}),s(_)!=="svelte-lue5cc"&&(_.textContent=ye),A=o(e),u(M.$$.fragment,e),O=o(e),x=r(e,"P",{"data-svelte-h":!0}),s(x)!=="svelte-1r2kpez"&&(x.textContent=ve),D=o(e),C=r(e,"UL",{"data-svelte-h":!0}),s(C)!=="svelte-1f29qgp"&&(C.innerHTML=$e),ee=o(e),k=r(e,"BLOCKQUOTE",{"data-svelte-h":!0}),s(k)!=="svelte-tc3a5k"&&(k.innerHTML=_e),te=o(e),u(L.$$.fragment,e),le=o(e),W=r(e,"P",{"data-svelte-h":!0}),s(W)!=="svelte-tmyftl"&&(W.textContent=Me),ie=o(e),Q=r(e,"UL",{"data-svelte-h":!0}),s(Q)!=="svelte-xxd66r"&&(Q.innerHTML=xe),ne=o(e),u(G.$$.fragment,e),oe=o(e),Z=r(e,"P",{"data-svelte-h":!0}),s(Z)!=="svelte-1lj1hm3"&&(Z.innerHTML=Ce),ae=o(e),u(H.$$.fragment,e),re=o(e),J=r(e,"P",{"data-svelte-h":!0}),s(J)!=="svelte-16kc502"&&(J.textContent=ke),se=o(e),U=r(e,"P",{"data-svelte-h":!0}),s(U)!=="svelte-uk9y6w"&&(U.innerHTML=Le),pe=o(e),z=r(e,"P",{"data-svelte-h":!0}),s(z)!=="svelte-eicl62"&&(z.innerHTML=We),fe=o(e),u(q.$$.fragment,e),ue=o(e),u(I.$$.fragment,e),me=o(e),X=r(e,"P",{}),Qe(X).forEach(l),this.h()},h(){Ge(p,"name","hf:doc:metadata"),Ge(p,"content",Ee)},m(e,t){Se(document.head,p),i(e,E,t),i(e,S,t),i(e,P,t),m(g,e,t),i(e,F,t),i(e,w,t),i(e,V,t),i(e,b,t),i(e,j,t),i(e,T,t),i(e,Y,t),m(y,e,t),i(e,R,t),i(e,v,t),i(e,N,t),i(e,$,t),i(e,K,t),i(e,_,t),i(e,A,t),m(M,e,t),i(e,O,t),i(e,x,t),i(e,D,t),i(e,C,t),i(e,ee,t),i(e,k,t),i(e,te,t),m(L,e,t),i(e,le,t),i(e,W,t),i(e,ie,t),i(e,Q,t),i(e,ne,t),m(G,e,t),i(e,oe,t),i(e,Z,t),i(e,ae,t),m(H,e,t),i(e,re,t),i(e,J,t),i(e,se,t),i(e,U,t),i(e,pe,t),i(e,z,t),i(e,fe,t),m(q,e,t),i(e,ue,t),m(I,e,t),i(e,me,t),i(e,X,t),de=!0},p:Je,i(e){de||(d(g.$$.fragment,e),d(y.$$.fragment,e),d(M.$$.fragment,e),d(L.$$.fragment,e),d(G.$$.fragment,e),d(H.$$.fragment,e),d(q.$$.fragment,e),d(I.$$.fragment,e),de=!0)},o(e){h(g.$$.fragment,e),h(y.$$.fragment,e),h(M.$$.fragment,e),h(L.$$.fragment,e),h(G.$$.fragment,e),h(H.$$.fragment,e),h(q.$$.fragment,e),h(I.$$.fragment,e),de=!1},d(e){e&&(l(E),l(S),l(P),l(F),l(w),l(V),l(b),l(j),l(T),l(Y),l(R),l(v),l(N),l($),l(K),l(_),l(A),l(O),l(x),l(D),l(C),l(ee),l(k),l(te),l(le),l(W),l(ie),l(Q),l(ne),l(oe),l(Z),l(ae),l(re),l(J),l(se),l(U),l(pe),l(z),l(fe),l(ue),l(me),l(X)),l(p),c(g,e),c(y,e),c(M,e),c(L,e),c(G,e),c(H,e),c(q,e),c(I,e)}}}const Ee='{"title":"GGUF and interaction with Transformers","local":"gguf-and-interaction-with-transformers","sections":[{"title":"Support within Transformers","local":"support-within-transformers","sections":[{"title":"Supported quantization types","local":"supported-quantization-types","sections":[],"depth":3},{"title":"Supported model architectures","local":"supported-model-architectures","sections":[],"depth":3}],"depth":2},{"title":"Example usage","local":"example-usage","sections":[],"depth":2}],"depth":1}';function Pe(he){return Ue(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Re extends ze{constructor(p){super(),qe(this,p,Pe,Be,He,{})}}export{Re as component};
