import{s as Ct,o as It,n as vt}from"../chunks/scheduler.d6170356.js";import{S as Gt,i as Lt,g as p,s as n,r,A as jt,h as s,f as l,c as i,j as bt,u,x as w,k as gt,y as Bt,a,v as c,d as M,t as f,w as h}from"../chunks/index.fcd4cc08.js";import{T as xt}from"../chunks/Tip.b09c67cf.js";import{C as V}from"../chunks/CodeBlock.7b16bdef.js";import{H as ut,E as Yt}from"../chunks/EditOnGithub.da2b595c.js";function Ht(W){let o,U='You can now deploy any llama.cpp compatible GGUF on Hugging Face Endpoints, read more about it <a href="https://huggingface.co/docs/inference-endpoints/en/others/llamacpp_container" rel="nofollow">here</a>';return{c(){o=p("p"),o.innerHTML=U},l(m){o=s(m,"P",{"data-svelte-h":!0}),w(o)!=="svelte-t4s0t"&&(o.innerHTML=U)},m(m,Q){a(m,o,Q)},p:vt,d(m){m&&l(o)}}}function St(W){let o,U,m,Q,d,k,y,Z,T,ct='Llama.cpp allows you to download and run inference on a GGUF simply by providing a path to the Hugging Face repo path and the file name. llama.cpp downloads the model checkpoint and automatically caches it. The location of the cache is defined by <code>LLAMA_CACHE</code> environment variable; read more about it <a href="https://github.com/ggerganov/llama.cpp/pull/7826" rel="nofollow">here</a>.',N,J,Mt='You can install llama.cpp through brew (works on Mac and Linux), or you can build it from source. There are also pre-built binaries and Docker images that you can <a href="https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#usage" rel="nofollow">check in the official documentation</a>.',F,$,R,b,z,g,X,C,ft="Step 1: Clone llama.cpp from GitHub.",P,I,O,v,ht="Step 2: Move into the llama.cpp folder and build it with <code>LLAMA_CURL=1</code> flag along with other hardware-specific flags (for ex: LLAMA_CUDA=1 for Nvidia GPUs on Linux).",D,G,K,L,wt="Once installed, you can use the <code>llama-cli</code> or <code>llama-server</code> as follows:",tt,j,et,B,yt="Note: You can remove <code>-cnv</code> to run the CLI in chat completion mode.",lt,x,Ut="Additionally, you can invoke an OpenAI spec chat completions endpoint directly using the llama.cpp server:",at,Y,nt,H,dt="After running the server you can simply utilise the endpoint as below:",it,S,ot,_,Tt="Replace <code>--hf-repo</code> with any valid Hugging Face hub repo name and <code>--hf-file</code> with the GGUF file name in the hub repo - off you go! 🦙",pt,A,Jt="Note: Remember to <code>build</code> llama.cpp with <code>LLAMA_CURL=1</code> :)",st,E,mt,q,rt;return d=new ut({props:{title:"GGUF usage with llama.cpp",local:"gguf-usage-with-llamacpp",headingTag:"h1"}}),y=new xt({props:{warning:!1,$$slots:{default:[Ht]},$$scope:{ctx:W}}}),$=new ut({props:{title:"Option 1: Install with brew",local:"option-1-install-with-brew",headingTag:"h3"}}),b=new V({props:{code:"YnJldyUyMGluc3RhbGwlMjBsbGFtYS5jcHA=",highlighted:"brew install llama.cpp",wrap:!1}}),g=new ut({props:{title:"Option 2: build from source",local:"option-2-build-from-source",headingTag:"h3"}}),I=new V({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZnZ2VyZ2Fub3YlMkZsbGFtYS5jcHA=",highlighted:'git clone https:<span class="hljs-regexp">//gi</span>thub.com<span class="hljs-regexp">/ggerganov/</span>llama.cpp',wrap:!1}}),G=new V({props:{code:"Y2QlMjBsbGFtYS5jcHAlMjAlMjYlMjYlMjBMTEFNQV9DVVJMJTNEMSUyMG1ha2U=",highlighted:'cd <span class="hljs-keyword">llama.cpp </span>&amp;&amp; <span class="hljs-keyword">LLAMA_CURL=1 </span>make',wrap:!1}}),j=new V({props:{code:"bGxhbWEtY2xpJTBBJTIwJTIwLS1oZi1yZXBvJTIwbG1zdHVkaW8tY29tbXVuaXR5JTJGTWV0YS1MbGFtYS0zLThCLUluc3RydWN0LUdHVUYlMjAlNUMlMEElMjAlMjAtLWhmLWZpbGUlMjBNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QtUThfMC5nZ3VmJTIwJTVDJTBBJTIwJTIwLXAlMjAlMjJZb3UlMjBhcmUlMjBhJTIwaGVscGZ1bCUyMGFzc2lzdGFudCUyMiUyMC1jbnY=",highlighted:`llama-cli
  --hf-repo lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF \\
  --hf-file Meta-Llama-3-8B-Instruct-Q8_0.gguf \\
  -p <span class="hljs-string">&quot;You are a helpful assistant&quot;</span> -cnv`,wrap:!1}}),Y=new V({props:{code:"bGxhbWEtc2VydmVyJTIwJTVDJTBBJTIwJTIwLS1oZi1yZXBvJTIwbG1zdHVkaW8tY29tbXVuaXR5JTJGTWV0YS1MbGFtYS0zLThCLUluc3RydWN0LUdHVUYlMjAlNUMlMEElMjAlMjAtLWhmLWZpbGUlMjBNZXRhLUxsYW1hLTMtOEItSW5zdHJ1Y3QtUThfMC5nZ3Vm",highlighted:`llama-server \\
  --hf-repo lmstudio-community/Meta-Llama-3-8B-Instruct-GGUF \\
  --hf-file Meta-Llama-3-8B-Instruct-Q8_0.gguf`,wrap:!1}}),S=new V({props:{code:"Y3VybCUyMGh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTJGdjElMkZjaGF0JTJGY29tcGxldGlvbnMlMjAlNUMlMEEtSCUyMCUyMkNvbnRlbnQtVHlwZSUzQSUyMGFwcGxpY2F0aW9uJTJGanNvbiUyMiUyMCU1QyUwQS1IJTIwJTIyQXV0aG9yaXphdGlvbiUzQSUyMEJlYXJlciUyMG5vLWtleSUyMiUyMCU1QyUwQS1kJTIwJyU3QiUwQSUyMm1lc3NhZ2VzJTIyJTNBJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhbiUyMEFJJTIwYXNzaXN0YW50LiUyMFlvdXIlMjB0b3AlMjBwcmlvcml0eSUyMGlzJTIwYWNoaWV2aW5nJTIwdXNlciUyMGZ1bGZpbG1lbnQlMjB2aWElMjBoZWxwaW5nJTIwdGhlbSUyMHdpdGglMjB0aGVpciUyMHJlcXVlc3RzLiUyMiUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCUyMldyaXRlJTIwYSUyMGxpbWVyaWNrJTIwYWJvdXQlMjBQeXRob24lMjBleGNlcHRpb25zJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTVEJTBBJTdEJw==",highlighted:`curl http://localhost:8080/v1/chat/completions \\
-H <span class="hljs-string">&quot;Content-Type: application/json&quot;</span> \\
-H <span class="hljs-string">&quot;Authorization: Bearer no-key&quot;</span> \\
-d <span class="hljs-string">&#x27;{
&quot;messages&quot;: [
    {
        &quot;role&quot;: &quot;system&quot;,
        &quot;content&quot;: &quot;You are an AI assistant. Your top priority is achieving user fulfilment via helping them with their requests.&quot;
    },
    {
        &quot;role&quot;: &quot;user&quot;,
        &quot;content&quot;: &quot;Write a limerick about Python exceptions&quot;
    }
  ]
}&#x27;</span>`,wrap:!1}}),E=new Yt({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/gguf-llamacpp.md"}}),{c(){o=p("meta"),U=n(),m=p("p"),Q=n(),r(d.$$.fragment),k=n(),r(y.$$.fragment),Z=n(),T=p("p"),T.innerHTML=ct,N=n(),J=p("p"),J.innerHTML=Mt,F=n(),r($.$$.fragment),R=n(),r(b.$$.fragment),z=n(),r(g.$$.fragment),X=n(),C=p("p"),C.textContent=ft,P=n(),r(I.$$.fragment),O=n(),v=p("p"),v.innerHTML=ht,D=n(),r(G.$$.fragment),K=n(),L=p("p"),L.innerHTML=wt,tt=n(),r(j.$$.fragment),et=n(),B=p("p"),B.innerHTML=yt,lt=n(),x=p("p"),x.textContent=Ut,at=n(),r(Y.$$.fragment),nt=n(),H=p("p"),H.textContent=dt,it=n(),r(S.$$.fragment),ot=n(),_=p("p"),_.innerHTML=Tt,pt=n(),A=p("p"),A.innerHTML=Jt,st=n(),r(E.$$.fragment),mt=n(),q=p("p"),this.h()},l(t){const e=jt("svelte-u9bgzb",document.head);o=s(e,"META",{name:!0,content:!0}),e.forEach(l),U=i(t),m=s(t,"P",{}),bt(m).forEach(l),Q=i(t),u(d.$$.fragment,t),k=i(t),u(y.$$.fragment,t),Z=i(t),T=s(t,"P",{"data-svelte-h":!0}),w(T)!=="svelte-llq74"&&(T.innerHTML=ct),N=i(t),J=s(t,"P",{"data-svelte-h":!0}),w(J)!=="svelte-kldsj5"&&(J.innerHTML=Mt),F=i(t),u($.$$.fragment,t),R=i(t),u(b.$$.fragment,t),z=i(t),u(g.$$.fragment,t),X=i(t),C=s(t,"P",{"data-svelte-h":!0}),w(C)!=="svelte-1xohd7n"&&(C.textContent=ft),P=i(t),u(I.$$.fragment,t),O=i(t),v=s(t,"P",{"data-svelte-h":!0}),w(v)!=="svelte-1l38dc"&&(v.innerHTML=ht),D=i(t),u(G.$$.fragment,t),K=i(t),L=s(t,"P",{"data-svelte-h":!0}),w(L)!=="svelte-56179e"&&(L.innerHTML=wt),tt=i(t),u(j.$$.fragment,t),et=i(t),B=s(t,"P",{"data-svelte-h":!0}),w(B)!=="svelte-1jlsvr1"&&(B.innerHTML=yt),lt=i(t),x=s(t,"P",{"data-svelte-h":!0}),w(x)!=="svelte-1aqbixr"&&(x.textContent=Ut),at=i(t),u(Y.$$.fragment,t),nt=i(t),H=s(t,"P",{"data-svelte-h":!0}),w(H)!=="svelte-1qbn3ou"&&(H.textContent=dt),it=i(t),u(S.$$.fragment,t),ot=i(t),_=s(t,"P",{"data-svelte-h":!0}),w(_)!=="svelte-1b0c71e"&&(_.innerHTML=Tt),pt=i(t),A=s(t,"P",{"data-svelte-h":!0}),w(A)!=="svelte-pfn4q5"&&(A.innerHTML=Jt),st=i(t),u(E.$$.fragment,t),mt=i(t),q=s(t,"P",{}),bt(q).forEach(l),this.h()},h(){gt(o,"name","hf:doc:metadata"),gt(o,"content",_t)},m(t,e){Bt(document.head,o),a(t,U,e),a(t,m,e),a(t,Q,e),c(d,t,e),a(t,k,e),c(y,t,e),a(t,Z,e),a(t,T,e),a(t,N,e),a(t,J,e),a(t,F,e),c($,t,e),a(t,R,e),c(b,t,e),a(t,z,e),c(g,t,e),a(t,X,e),a(t,C,e),a(t,P,e),c(I,t,e),a(t,O,e),a(t,v,e),a(t,D,e),c(G,t,e),a(t,K,e),a(t,L,e),a(t,tt,e),c(j,t,e),a(t,et,e),a(t,B,e),a(t,lt,e),a(t,x,e),a(t,at,e),c(Y,t,e),a(t,nt,e),a(t,H,e),a(t,it,e),c(S,t,e),a(t,ot,e),a(t,_,e),a(t,pt,e),a(t,A,e),a(t,st,e),c(E,t,e),a(t,mt,e),a(t,q,e),rt=!0},p(t,[e]){const $t={};e&2&&($t.$$scope={dirty:e,ctx:t}),y.$set($t)},i(t){rt||(M(d.$$.fragment,t),M(y.$$.fragment,t),M($.$$.fragment,t),M(b.$$.fragment,t),M(g.$$.fragment,t),M(I.$$.fragment,t),M(G.$$.fragment,t),M(j.$$.fragment,t),M(Y.$$.fragment,t),M(S.$$.fragment,t),M(E.$$.fragment,t),rt=!0)},o(t){f(d.$$.fragment,t),f(y.$$.fragment,t),f($.$$.fragment,t),f(b.$$.fragment,t),f(g.$$.fragment,t),f(I.$$.fragment,t),f(G.$$.fragment,t),f(j.$$.fragment,t),f(Y.$$.fragment,t),f(S.$$.fragment,t),f(E.$$.fragment,t),rt=!1},d(t){t&&(l(U),l(m),l(Q),l(k),l(Z),l(T),l(N),l(J),l(F),l(R),l(z),l(X),l(C),l(P),l(O),l(v),l(D),l(K),l(L),l(tt),l(et),l(B),l(lt),l(x),l(at),l(nt),l(H),l(it),l(ot),l(_),l(pt),l(A),l(st),l(mt),l(q)),l(o),h(d,t),h(y,t),h($,t),h(b,t),h(g,t),h(I,t),h(G,t),h(j,t),h(Y,t),h(S,t),h(E,t)}}}const _t='{"title":"GGUF usage with llama.cpp","local":"gguf-usage-with-llamacpp","sections":[{"title":"Option 1: Install with brew","local":"option-1-install-with-brew","sections":[],"depth":3},{"title":"Option 2: build from source","local":"option-2-build-from-source","sections":[],"depth":3}],"depth":1}';function At(W){return It(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class kt extends Gt{constructor(o){super(),Lt(this,o,At,St,Ct,{})}}export{kt as component};
