import{s as Ce,o as Ie,n as Ge}from"../chunks/scheduler.d6170356.js";import{S as ve,i as je,g as o,s as n,r as m,A as xe,h as p,f as l,c as i,j as Je,u as c,x as y,k as ge,y as Le,a,v as u,d as f,t as M,w}from"../chunks/index.fcd4cc08.js";import{T as He}from"../chunks/Tip.b09c67cf.js";import{C as E}from"../chunks/CodeBlock.db16bf50.js";import{H as ce,E as Ye}from"../chunks/index.c1f68a2b.js";function _e(A){let s,h='You can now deploy any llama.cpp compatible GGUF on Hugging Face Endpoints, read more about it <a href="https://huggingface.co/docs/inference-endpoints/en/others/llamacpp_container" rel="nofollow">here</a>';return{c(){s=o("p"),s.innerHTML=h},l(r){s=p(r,"P",{"data-svelte-h":!0}),y(s)!=="svelte-t4s0t"&&(s.innerHTML=h)},m(r,q){a(r,s,q)},p:Ge,d(r){r&&l(s)}}}function Be(A){let s,h,r,q,U,F,d,W,T,ue='Llama.cpp allows you to download and run inference on a GGUF simply by providing a path to the Hugging Face repo path and the file name. llama.cpp downloads the model checkpoint and automatically caches it. The location of the cache is defined by <code>LLAMA_CACHE</code> environment variable; read more about it <a href="https://github.com/ggerganov/llama.cpp/pull/7826" rel="nofollow">here</a>.',N,b,fe='You can install llama.cpp through brew (works on Mac and Linux), or you can build it from source. There are also pre-built binaries and Docker images that you can <a href="https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#usage" rel="nofollow">check in the official documentation</a>.',V,$,P,J,Z,g,X,C,Me="Step 1: Clone llama.cpp from GitHub.",z,I,D,G,we="Step 2: Move into the llama.cpp folder and build it. You can also add hardware-specific flags (for ex: <code>-DGGML_CUDA=1</code> for Nvidia GPUs).",O,v,K,j,ye='Note: for other hardware support (for ex: AMD ROCm, Intel SYCL), please refer to <a href="https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md" rel="nofollow">llama.cpp’s build guide</a>',ee,x,de="Once installed, you can use the <code>llama-cli</code> or <code>llama-server</code> as follows:",te,L,le,H,he="Note: You can explicitly add <code>-no-cnv</code> to run the CLI in raw completion mode (non-chat mode).",ae,Y,Ue="Additionally, you can invoke an OpenAI spec chat completions endpoint directly using the llama.cpp server:",ne,_,ie,B,Te="After running the server you can simply utilise the endpoint as below:",se,k,oe,Q,be="Replace <code>-hf</code> with any valid Hugging Face hub repo name - off you go! 🦙",pe,S,re,R,me;return U=new ce({props:{title:"GGUF usage with llama.cpp",local:"gguf-usage-with-llamacpp",headingTag:"h1"}}),d=new He({props:{warning:!1,$$slots:{default:[_e]},$$scope:{ctx:A}}}),$=new ce({props:{title:"Option 1: Install with brew",local:"option-1-install-with-brew",headingTag:"h3"}}),J=new E({props:{code:"YnJldyUyMGluc3RhbGwlMjBsbGFtYS5jcHA=",highlighted:"brew install llama.cpp",wrap:!1}}),g=new ce({props:{title:"Option 2: build from source",local:"option-2-build-from-source",headingTag:"h3"}}),I=new E({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZnZ2VyZ2Fub3YlMkZsbGFtYS5jcHA=",highlighted:'git clone https:<span class="hljs-regexp">//gi</span>thub.com<span class="hljs-regexp">/ggerganov/</span>llama.cpp',wrap:!1}}),v=new E({props:{code:"Y2QlMjBsbGFtYS5jcHAlMEFjbWFrZSUyMC1CJTIwYnVpbGQlMjAlMjAlMjAlMjMlMjBvcHRpb25hbGx5JTJDJTIwYWRkJTIwLURHR01MX0NVREElM0RPTiUyMHRvJTIwYWN0aXZhdGUlMjBDVURBJTBBY21ha2UlMjAtLWJ1aWxkJTIwYnVpbGQlMjAtLWNvbmZpZyUyMFJlbGVhc2U=",highlighted:`cd <span class="hljs-keyword">llama.cpp
</span>cmake -<span class="hljs-keyword">B </span><span class="hljs-keyword">build </span>  <span class="hljs-comment"># optionally, add -DGGML_CUDA=ON to activate CUDA</span>
cmake --<span class="hljs-keyword">build </span><span class="hljs-keyword">build </span>--<span class="hljs-built_in">config</span> Release`,wrap:!1}}),L=new E({props:{code:"bGxhbWEtY2xpJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),_=new E({props:{code:"bGxhbWEtc2VydmVyJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),k=new E({props:{code:"Y3VybCUyMGh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTJGdjElMkZjaGF0JTJGY29tcGxldGlvbnMlMjAlNUMlMEEtSCUyMCUyMkNvbnRlbnQtVHlwZSUzQSUyMGFwcGxpY2F0aW9uJTJGanNvbiUyMiUyMCU1QyUwQS1IJTIwJTIyQXV0aG9yaXphdGlvbiUzQSUyMEJlYXJlciUyMG5vLWtleSUyMiUyMCU1QyUwQS1kJTIwJyU3QiUwQSUyMm1lc3NhZ2VzJTIyJTNBJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhbiUyMEFJJTIwYXNzaXN0YW50LiUyMFlvdXIlMjB0b3AlMjBwcmlvcml0eSUyMGlzJTIwYWNoaWV2aW5nJTIwdXNlciUyMGZ1bGZpbG1lbnQlMjB2aWElMjBoZWxwaW5nJTIwdGhlbSUyMHdpdGglMjB0aGVpciUyMHJlcXVlc3RzLiUyMiUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCUyMldyaXRlJTIwYSUyMGxpbWVyaWNrJTIwYWJvdXQlMjBQeXRob24lMjBleGNlcHRpb25zJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTVEJTBBJTdEJw==",highlighted:`curl http://localhost:8080/v1/chat/completions \\
-H <span class="hljs-string">&quot;Content-Type: application/json&quot;</span> \\
-H <span class="hljs-string">&quot;Authorization: Bearer no-key&quot;</span> \\
-d <span class="hljs-string">&#x27;{
&quot;messages&quot;: [
    {
        &quot;role&quot;: &quot;system&quot;,
        &quot;content&quot;: &quot;You are an AI assistant. Your top priority is achieving user fulfilment via helping them with their requests.&quot;
    },
    {
        &quot;role&quot;: &quot;user&quot;,
        &quot;content&quot;: &quot;Write a limerick about Python exceptions&quot;
    }
  ]
}&#x27;</span>`,wrap:!1}}),S=new Ye({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/gguf-llamacpp.md"}}),{c(){s=o("meta"),h=n(),r=o("p"),q=n(),m(U.$$.fragment),F=n(),m(d.$$.fragment),W=n(),T=o("p"),T.innerHTML=ue,N=n(),b=o("p"),b.innerHTML=fe,V=n(),m($.$$.fragment),P=n(),m(J.$$.fragment),Z=n(),m(g.$$.fragment),X=n(),C=o("p"),C.textContent=Me,z=n(),m(I.$$.fragment),D=n(),G=o("p"),G.innerHTML=we,O=n(),m(v.$$.fragment),K=n(),j=o("p"),j.innerHTML=ye,ee=n(),x=o("p"),x.innerHTML=de,te=n(),m(L.$$.fragment),le=n(),H=o("p"),H.innerHTML=he,ae=n(),Y=o("p"),Y.textContent=Ue,ne=n(),m(_.$$.fragment),ie=n(),B=o("p"),B.textContent=Te,se=n(),m(k.$$.fragment),oe=n(),Q=o("p"),Q.innerHTML=be,pe=n(),m(S.$$.fragment),re=n(),R=o("p"),this.h()},l(e){const t=xe("svelte-u9bgzb",document.head);s=p(t,"META",{name:!0,content:!0}),t.forEach(l),h=i(e),r=p(e,"P",{}),Je(r).forEach(l),q=i(e),c(U.$$.fragment,e),F=i(e),c(d.$$.fragment,e),W=i(e),T=p(e,"P",{"data-svelte-h":!0}),y(T)!=="svelte-llq74"&&(T.innerHTML=ue),N=i(e),b=p(e,"P",{"data-svelte-h":!0}),y(b)!=="svelte-kldsj5"&&(b.innerHTML=fe),V=i(e),c($.$$.fragment,e),P=i(e),c(J.$$.fragment,e),Z=i(e),c(g.$$.fragment,e),X=i(e),C=p(e,"P",{"data-svelte-h":!0}),y(C)!=="svelte-1xohd7n"&&(C.textContent=Me),z=i(e),c(I.$$.fragment,e),D=i(e),G=p(e,"P",{"data-svelte-h":!0}),y(G)!=="svelte-1r0jgu0"&&(G.innerHTML=we),O=i(e),c(v.$$.fragment,e),K=i(e),j=p(e,"P",{"data-svelte-h":!0}),y(j)!=="svelte-3pqgmd"&&(j.innerHTML=ye),ee=i(e),x=p(e,"P",{"data-svelte-h":!0}),y(x)!=="svelte-56179e"&&(x.innerHTML=de),te=i(e),c(L.$$.fragment,e),le=i(e),H=p(e,"P",{"data-svelte-h":!0}),y(H)!=="svelte-1pxj7s7"&&(H.innerHTML=he),ae=i(e),Y=p(e,"P",{"data-svelte-h":!0}),y(Y)!=="svelte-1aqbixr"&&(Y.textContent=Ue),ne=i(e),c(_.$$.fragment,e),ie=i(e),B=p(e,"P",{"data-svelte-h":!0}),y(B)!=="svelte-1qbn3ou"&&(B.textContent=Te),se=i(e),c(k.$$.fragment,e),oe=i(e),Q=p(e,"P",{"data-svelte-h":!0}),y(Q)!=="svelte-rc3gcp"&&(Q.innerHTML=be),pe=i(e),c(S.$$.fragment,e),re=i(e),R=p(e,"P",{}),Je(R).forEach(l),this.h()},h(){ge(s,"name","hf:doc:metadata"),ge(s,"content",ke)},m(e,t){Le(document.head,s),a(e,h,t),a(e,r,t),a(e,q,t),u(U,e,t),a(e,F,t),u(d,e,t),a(e,W,t),a(e,T,t),a(e,N,t),a(e,b,t),a(e,V,t),u($,e,t),a(e,P,t),u(J,e,t),a(e,Z,t),u(g,e,t),a(e,X,t),a(e,C,t),a(e,z,t),u(I,e,t),a(e,D,t),a(e,G,t),a(e,O,t),u(v,e,t),a(e,K,t),a(e,j,t),a(e,ee,t),a(e,x,t),a(e,te,t),u(L,e,t),a(e,le,t),a(e,H,t),a(e,ae,t),a(e,Y,t),a(e,ne,t),u(_,e,t),a(e,ie,t),a(e,B,t),a(e,se,t),u(k,e,t),a(e,oe,t),a(e,Q,t),a(e,pe,t),u(S,e,t),a(e,re,t),a(e,R,t),me=!0},p(e,[t]){const $e={};t&2&&($e.$$scope={dirty:t,ctx:e}),d.$set($e)},i(e){me||(f(U.$$.fragment,e),f(d.$$.fragment,e),f($.$$.fragment,e),f(J.$$.fragment,e),f(g.$$.fragment,e),f(I.$$.fragment,e),f(v.$$.fragment,e),f(L.$$.fragment,e),f(_.$$.fragment,e),f(k.$$.fragment,e),f(S.$$.fragment,e),me=!0)},o(e){M(U.$$.fragment,e),M(d.$$.fragment,e),M($.$$.fragment,e),M(J.$$.fragment,e),M(g.$$.fragment,e),M(I.$$.fragment,e),M(v.$$.fragment,e),M(L.$$.fragment,e),M(_.$$.fragment,e),M(k.$$.fragment,e),M(S.$$.fragment,e),me=!1},d(e){e&&(l(h),l(r),l(q),l(F),l(W),l(T),l(N),l(b),l(V),l(P),l(Z),l(X),l(C),l(z),l(D),l(G),l(O),l(K),l(j),l(ee),l(x),l(te),l(le),l(H),l(ae),l(Y),l(ne),l(ie),l(B),l(se),l(oe),l(Q),l(pe),l(re),l(R)),l(s),w(U,e),w(d,e),w($,e),w(J,e),w(g,e),w(I,e),w(v,e),w(L,e),w(_,e),w(k,e),w(S,e)}}}const ke='{"title":"GGUF usage with llama.cpp","local":"gguf-usage-with-llamacpp","sections":[{"title":"Option 1: Install with brew","local":"option-1-install-with-brew","sections":[],"depth":3},{"title":"Option 2: build from source","local":"option-2-build-from-source","sections":[],"depth":3}],"depth":1}';function Qe(A){return Ie(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fe extends ve{constructor(s){super(),je(this,s,Qe,Be,Ce,{})}}export{Fe as component};
