import{s as Ce,o as ve,n as Ie}from"../chunks/scheduler.d6170356.js";import{S as Ge,i as Le,g as p,s as n,r,A as xe,h as s,f as l,c as i,j as Je,u as c,x as y,k as ge,y as je,a,v as u,d as f,t as M,w}from"../chunks/index.fcd4cc08.js";import{T as _e}from"../chunks/Tip.b09c67cf.js";import{C as A}from"../chunks/CodeBlock.7b16bdef.js";import{H as ce,E as He}from"../chunks/EditOnGithub.da2b595c.js";function Ye(F){let o,d='You can now deploy any llama.cpp compatible GGUF on Hugging Face Endpoints, read more about it <a href="https://huggingface.co/docs/inference-endpoints/en/others/llamacpp_container" rel="nofollow">here</a>';return{c(){o=p("p"),o.innerHTML=d},l(m){o=s(m,"P",{"data-svelte-h":!0}),y(o)!=="svelte-t4s0t"&&(o.innerHTML=d)},m(m,q){a(m,o,q)},p:Ie,d(m){m&&l(o)}}}function Be(F){let o,d,m,q,U,N,h,R,$,ue='Llama.cpp allows you to download and run inference on a GGUF simply by providing a path to the Hugging Face repo path and the file name. llama.cpp downloads the model checkpoint and automatically caches it. The location of the cache is defined by <code>LLAMA_CACHE</code> environment variable; read more about it <a href="https://github.com/ggerganov/llama.cpp/pull/7826" rel="nofollow">here</a>.',W,T,fe='You can install llama.cpp through brew (works on Mac and Linux), or you can build it from source. There are also pre-built binaries and Docker images that you can <a href="https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#usage" rel="nofollow">check in the official documentation</a>.',V,b,P,J,Z,g,z,C,Me="Step 1: Clone llama.cpp from GitHub.",X,v,O,I,we="Step 2: Move into the llama.cpp folder and build it with <code>LLAMA_CURL=1</code> flag along with other hardware-specific flags (for ex: LLAMA_CUDA=1 for Nvidia GPUs on Linux).",D,G,K,L,ye="Once installed, you can use the <code>llama-cli</code> or <code>llama-server</code> as follows:",ee,x,te,j,he="Note: You can remove <code>-cnv</code> to run the CLI in chat completion mode.",le,_,de="Additionally, you can invoke an OpenAI spec chat completions endpoint directly using the llama.cpp server:",ae,H,ne,Y,Ue="After running the server you can simply utilise the endpoint as below:",ie,B,oe,Q,$e="Replace <code>-hf</code> with any valid Hugging Face hub repo name - off you go! 🦙",pe,S,Te="Note: Remember to <code>build</code> llama.cpp with <code>LLAMA_CURL=1</code> :)",se,k,me,E,re;return U=new ce({props:{title:"GGUF usage with llama.cpp",local:"gguf-usage-with-llamacpp",headingTag:"h1"}}),h=new _e({props:{warning:!1,$$slots:{default:[Ye]},$$scope:{ctx:F}}}),b=new ce({props:{title:"Option 1: Install with brew",local:"option-1-install-with-brew",headingTag:"h3"}}),J=new A({props:{code:"YnJldyUyMGluc3RhbGwlMjBsbGFtYS5jcHA=",highlighted:"brew install llama.cpp",wrap:!1}}),g=new ce({props:{title:"Option 2: build from source",local:"option-2-build-from-source",headingTag:"h3"}}),v=new A({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZnZ2VyZ2Fub3YlMkZsbGFtYS5jcHA=",highlighted:'git clone https:<span class="hljs-regexp">//gi</span>thub.com<span class="hljs-regexp">/ggerganov/</span>llama.cpp',wrap:!1}}),G=new A({props:{code:"Y2QlMjBsbGFtYS5jcHAlMjAlMjYlMjYlMjBMTEFNQV9DVVJMJTNEMSUyMG1ha2U=",highlighted:'cd <span class="hljs-keyword">llama.cpp </span>&amp;&amp; <span class="hljs-keyword">LLAMA_CURL=1 </span>make',wrap:!1}}),x=new A({props:{code:"bGxhbWEtY2xpJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),H=new A({props:{code:"bGxhbWEtc2VydmVyJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),B=new A({props:{code:"Y3VybCUyMGh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTJGdjElMkZjaGF0JTJGY29tcGxldGlvbnMlMjAlNUMlMEEtSCUyMCUyMkNvbnRlbnQtVHlwZSUzQSUyMGFwcGxpY2F0aW9uJTJGanNvbiUyMiUyMCU1QyUwQS1IJTIwJTIyQXV0aG9yaXphdGlvbiUzQSUyMEJlYXJlciUyMG5vLWtleSUyMiUyMCU1QyUwQS1kJTIwJyU3QiUwQSUyMm1lc3NhZ2VzJTIyJTNBJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhbiUyMEFJJTIwYXNzaXN0YW50LiUyMFlvdXIlMjB0b3AlMjBwcmlvcml0eSUyMGlzJTIwYWNoaWV2aW5nJTIwdXNlciUyMGZ1bGZpbG1lbnQlMjB2aWElMjBoZWxwaW5nJTIwdGhlbSUyMHdpdGglMjB0aGVpciUyMHJlcXVlc3RzLiUyMiUwQSUyMCUyMCUyMCUyMCU3RCUyQyUwQSUyMCUyMCUyMCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnJvbGUlMjIlM0ElMjAlMjJ1c2VyJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyY29udGVudCUyMiUzQSUyMCUyMldyaXRlJTIwYSUyMGxpbWVyaWNrJTIwYWJvdXQlMjBQeXRob24lMjBleGNlcHRpb25zJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTVEJTBBJTdEJw==",highlighted:`curl http://localhost:8080/v1/chat/completions \\
-H <span class="hljs-string">&quot;Content-Type: application/json&quot;</span> \\
-H <span class="hljs-string">&quot;Authorization: Bearer no-key&quot;</span> \\
-d <span class="hljs-string">&#x27;{
&quot;messages&quot;: [
    {
        &quot;role&quot;: &quot;system&quot;,
        &quot;content&quot;: &quot;You are an AI assistant. Your top priority is achieving user fulfilment via helping them with their requests.&quot;
    },
    {
        &quot;role&quot;: &quot;user&quot;,
        &quot;content&quot;: &quot;Write a limerick about Python exceptions&quot;
    }
  ]
}&#x27;</span>`,wrap:!1}}),k=new He({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/gguf-llamacpp.md"}}),{c(){o=p("meta"),d=n(),m=p("p"),q=n(),r(U.$$.fragment),N=n(),r(h.$$.fragment),R=n(),$=p("p"),$.innerHTML=ue,W=n(),T=p("p"),T.innerHTML=fe,V=n(),r(b.$$.fragment),P=n(),r(J.$$.fragment),Z=n(),r(g.$$.fragment),z=n(),C=p("p"),C.textContent=Me,X=n(),r(v.$$.fragment),O=n(),I=p("p"),I.innerHTML=we,D=n(),r(G.$$.fragment),K=n(),L=p("p"),L.innerHTML=ye,ee=n(),r(x.$$.fragment),te=n(),j=p("p"),j.innerHTML=he,le=n(),_=p("p"),_.textContent=de,ae=n(),r(H.$$.fragment),ne=n(),Y=p("p"),Y.textContent=Ue,ie=n(),r(B.$$.fragment),oe=n(),Q=p("p"),Q.innerHTML=$e,pe=n(),S=p("p"),S.innerHTML=Te,se=n(),r(k.$$.fragment),me=n(),E=p("p"),this.h()},l(e){const t=xe("svelte-u9bgzb",document.head);o=s(t,"META",{name:!0,content:!0}),t.forEach(l),d=i(e),m=s(e,"P",{}),Je(m).forEach(l),q=i(e),c(U.$$.fragment,e),N=i(e),c(h.$$.fragment,e),R=i(e),$=s(e,"P",{"data-svelte-h":!0}),y($)!=="svelte-llq74"&&($.innerHTML=ue),W=i(e),T=s(e,"P",{"data-svelte-h":!0}),y(T)!=="svelte-kldsj5"&&(T.innerHTML=fe),V=i(e),c(b.$$.fragment,e),P=i(e),c(J.$$.fragment,e),Z=i(e),c(g.$$.fragment,e),z=i(e),C=s(e,"P",{"data-svelte-h":!0}),y(C)!=="svelte-1xohd7n"&&(C.textContent=Me),X=i(e),c(v.$$.fragment,e),O=i(e),I=s(e,"P",{"data-svelte-h":!0}),y(I)!=="svelte-1l38dc"&&(I.innerHTML=we),D=i(e),c(G.$$.fragment,e),K=i(e),L=s(e,"P",{"data-svelte-h":!0}),y(L)!=="svelte-56179e"&&(L.innerHTML=ye),ee=i(e),c(x.$$.fragment,e),te=i(e),j=s(e,"P",{"data-svelte-h":!0}),y(j)!=="svelte-1jlsvr1"&&(j.innerHTML=he),le=i(e),_=s(e,"P",{"data-svelte-h":!0}),y(_)!=="svelte-1aqbixr"&&(_.textContent=de),ae=i(e),c(H.$$.fragment,e),ne=i(e),Y=s(e,"P",{"data-svelte-h":!0}),y(Y)!=="svelte-1qbn3ou"&&(Y.textContent=Ue),ie=i(e),c(B.$$.fragment,e),oe=i(e),Q=s(e,"P",{"data-svelte-h":!0}),y(Q)!=="svelte-rc3gcp"&&(Q.innerHTML=$e),pe=i(e),S=s(e,"P",{"data-svelte-h":!0}),y(S)!=="svelte-pfn4q5"&&(S.innerHTML=Te),se=i(e),c(k.$$.fragment,e),me=i(e),E=s(e,"P",{}),Je(E).forEach(l),this.h()},h(){ge(o,"name","hf:doc:metadata"),ge(o,"content",Qe)},m(e,t){je(document.head,o),a(e,d,t),a(e,m,t),a(e,q,t),u(U,e,t),a(e,N,t),u(h,e,t),a(e,R,t),a(e,$,t),a(e,W,t),a(e,T,t),a(e,V,t),u(b,e,t),a(e,P,t),u(J,e,t),a(e,Z,t),u(g,e,t),a(e,z,t),a(e,C,t),a(e,X,t),u(v,e,t),a(e,O,t),a(e,I,t),a(e,D,t),u(G,e,t),a(e,K,t),a(e,L,t),a(e,ee,t),u(x,e,t),a(e,te,t),a(e,j,t),a(e,le,t),a(e,_,t),a(e,ae,t),u(H,e,t),a(e,ne,t),a(e,Y,t),a(e,ie,t),u(B,e,t),a(e,oe,t),a(e,Q,t),a(e,pe,t),a(e,S,t),a(e,se,t),u(k,e,t),a(e,me,t),a(e,E,t),re=!0},p(e,[t]){const be={};t&2&&(be.$$scope={dirty:t,ctx:e}),h.$set(be)},i(e){re||(f(U.$$.fragment,e),f(h.$$.fragment,e),f(b.$$.fragment,e),f(J.$$.fragment,e),f(g.$$.fragment,e),f(v.$$.fragment,e),f(G.$$.fragment,e),f(x.$$.fragment,e),f(H.$$.fragment,e),f(B.$$.fragment,e),f(k.$$.fragment,e),re=!0)},o(e){M(U.$$.fragment,e),M(h.$$.fragment,e),M(b.$$.fragment,e),M(J.$$.fragment,e),M(g.$$.fragment,e),M(v.$$.fragment,e),M(G.$$.fragment,e),M(x.$$.fragment,e),M(H.$$.fragment,e),M(B.$$.fragment,e),M(k.$$.fragment,e),re=!1},d(e){e&&(l(d),l(m),l(q),l(N),l(R),l($),l(W),l(T),l(V),l(P),l(Z),l(z),l(C),l(X),l(O),l(I),l(D),l(K),l(L),l(ee),l(te),l(j),l(le),l(_),l(ae),l(ne),l(Y),l(ie),l(oe),l(Q),l(pe),l(S),l(se),l(me),l(E)),l(o),w(U,e),w(h,e),w(b,e),w(J,e),w(g,e),w(v,e),w(G,e),w(x,e),w(H,e),w(B,e),w(k,e)}}}const Qe='{"title":"GGUF usage with llama.cpp","local":"gguf-usage-with-llamacpp","sections":[{"title":"Option 1: Install with brew","local":"option-1-install-with-brew","sections":[],"depth":3},{"title":"Option 2: build from source","local":"option-2-build-from-source","sections":[],"depth":3}],"depth":1}';function Se(F){return ve(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ne extends Ge{constructor(o){super(),Le(this,o,Se,Be,Ce,{})}}export{Ne as component};
