import{s as xt,o as At,n as Lt}from"../chunks/scheduler.12fdeddc.js";import{S as Ht,i as _t,g as p,s as n,r as m,A as kt,h as o,f as l,c as i,j as Gt,u as r,x as w,k as It,y as Yt,a,v as c,d as u,t as M,w as f}from"../chunks/index.88dae322.js";import{T as Et}from"../chunks/Tip.64e66512.js";import{C as S}from"../chunks/CodeBlock.8e54a086.js";import{H as wt,E as Ft}from"../chunks/getInferenceSnippets.00466244.js";function Nt(B){let s,d='You can now deploy any llama.cpp compatible GGUF on Hugging Face Endpoints, read more about it <a href="https://huggingface.co/docs/inference-endpoints/en/others/llamacpp_container" rel="nofollow">here</a>';return{c(){s=p("p"),s.innerHTML=d},l(y){s=o(y,"P",{"data-svelte-h":!0}),w(s)!=="svelte-t4s0t"&&(s.innerHTML=d)},m(y,q){a(y,s,q)},p:Lt,d(y){y&&l(s)}}}function Rt(B){let s,d,y,q,$,W,h,Z,U,yt='Llama.cpp allows you to download and run inference on a GGUF simply by providing a path to the Hugging Face repo path and the file name. llama.cpp downloads the model checkpoint and automatically caches it. The location of the cache is defined by <code>LLAMA_CACHE</code> environment variable; read more about it <a href="https://github.com/ggerganov/llama.cpp/pull/7826" rel="nofollow">here</a>.',P,b,ht='You can install llama.cpp through brew (works on Mac and Linux), or you can build it from source. There are also pre-built binaries and Docker images that you can <a href="https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#usage" rel="nofollow">check in the official documentation</a>.',V,g,X,T,O,C,dt="or, on windows via winget",z,j,D,J,K,v,$t="Step 1: Clone llama.cpp from GitHub.",tt,G,et,I,Ut="Step 2: Move into the llama.cpp folder and build it. You can also add hardware-specific flags (for ex: <code>-DGGML_CUDA=1</code> for Nvidia GPUs).",lt,x,at,A,bt='Note: for other hardware support (for ex: AMD ROCm, Intel SYCL), please refer to <a href="https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md" rel="nofollow">llama.cpp’s build guide</a>',nt,L,gt="Once installed, you can use the <code>llama-cli</code> or <code>llama-server</code> as follows:",it,H,st,_,Tt="Note: You can explicitly add <code>-no-cnv</code> to run the CLI in raw completion mode (non-chat mode).",pt,k,Ct="Additionally, you can invoke an OpenAI spec chat completions endpoint directly using the llama.cpp server:",ot,Y,mt,E,jt="After running the server you can simply utilise the endpoint as below:",rt,F,ct,N,Jt="Replace <code>-hf</code> with any valid Hugging Face hub repo name - off you go! 🦙",ut,R,Mt,Q,ft;return $=new wt({props:{title:"GGUF usage with llama.cpp",local:"gguf-usage-with-llamacpp",headingTag:"h1"}}),h=new Et({props:{warning:!1,$$slots:{default:[Nt]},$$scope:{ctx:B}}}),g=new wt({props:{title:"Option 1: Install with brew/ winget",local:"option-1-install-with-brew-winget",headingTag:"h3"}}),T=new S({props:{code:"YnJldyUyMGluc3RhbGwlMjBsbGFtYS5jcHA=",highlighted:"brew install llama.cpp",wrap:!1}}),j=new S({props:{code:"d2luZ2V0JTIwaW5zdGFsbCUyMGxsYW1hLmNwcA==",highlighted:"winget install llama.cpp",wrap:!1}}),J=new wt({props:{title:"Option 2: build from source",local:"option-2-build-from-source",headingTag:"h3"}}),G=new S({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZnZ2VyZ2Fub3YlMkZsbGFtYS5jcHA=",highlighted:'git clone https:<span class="hljs-regexp">//gi</span>thub.com<span class="hljs-regexp">/ggerganov/</span>llama.cpp',wrap:!1}}),x=new S({props:{code:"Y2QlMjBsbGFtYS5jcHAlMEFjbWFrZSUyMC1CJTIwYnVpbGQlMjAlMjAlMjAlMjMlMjBvcHRpb25hbGx5JTJDJTIwYWRkJTIwLURHR01MX0NVREElM0RPTiUyMHRvJTIwYWN0aXZhdGUlMjBDVURBJTBBY21ha2UlMjAtLWJ1aWxkJTIwYnVpbGQlMjAtLWNvbmZpZyUyMFJlbGVhc2U=",highlighted:`cd <span class="hljs-keyword">llama.cpp
</span>cmake -<span class="hljs-keyword">B </span><span class="hljs-keyword">build </span>  <span class="hljs-comment"># optionally, add -DGGML_CUDA=ON to activate CUDA</span>
cmake --<span class="hljs-keyword">build </span><span class="hljs-keyword">build </span>--<span class="hljs-built_in">config</span> Release`,wrap:!1}}),H=new S({props:{code:"bGxhbWEtY2xpJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),Y=new S({props:{code:"bGxhbWEtc2VydmVyJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),F=new S({props:{code:"Y3VybCUyMGh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTJGdjElMkZjaGF0JTJGY29tcGxldGlvbnMlMjAlNUMlMEEtSCUyMCUyMkNvbnRlbnQtVHlwZSUzQSUyMGFwcGxpY2F0aW9uJTJGanNvbiUyMiUyMCU1QyUwQS1IJTIwJTIyQXV0aG9yaXphdGlvbiUzQSUyMEJlYXJlciUyMG5vLWtleSUyMiUyMCU1QyUwQS1kJTIwJyU3QiUwQSUyMm1lc3NhZ2VzJTIyJTNBJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhbiUyMEFJJTIwYXNzaXN0YW50LiUyMFlvdXIlMjB0b3AlMjBwcmlvcml0eSUyMGlzJTIwYWNoaWV2aW5nJTIwdXNlciUyMGZ1bGZpbGxtZW50JTIwdmlhJTIwaGVscGluZyUyMHRoZW0lMjB3aXRoJTIwdGhlaXIlMjByZXF1ZXN0cy4lMjIlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJXcml0ZSUyMGElMjBsaW1lcmljayUyMGFib3V0JTIwUHl0aG9uJTIwZXhjZXB0aW9ucyUyMiUwQSUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCU1RCUwQSU3RCc=",highlighted:`curl http://localhost:8080/v1/chat/completions \\
-H <span class="hljs-string">&quot;Content-Type: application/json&quot;</span> \\
-H <span class="hljs-string">&quot;Authorization: Bearer no-key&quot;</span> \\
-d <span class="hljs-string">&#x27;{
&quot;messages&quot;: [
    {
        &quot;role&quot;: &quot;system&quot;,
        &quot;content&quot;: &quot;You are an AI assistant. Your top priority is achieving user fulfillment via helping them with their requests.&quot;
    },
    {
        &quot;role&quot;: &quot;user&quot;,
        &quot;content&quot;: &quot;Write a limerick about Python exceptions&quot;
    }
  ]
}&#x27;</span>`,wrap:!1}}),R=new Ft({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/gguf-llamacpp.md"}}),{c(){s=p("meta"),d=n(),y=p("p"),q=n(),m($.$$.fragment),W=n(),m(h.$$.fragment),Z=n(),U=p("p"),U.innerHTML=yt,P=n(),b=p("p"),b.innerHTML=ht,V=n(),m(g.$$.fragment),X=n(),m(T.$$.fragment),O=n(),C=p("p"),C.textContent=dt,z=n(),m(j.$$.fragment),D=n(),m(J.$$.fragment),K=n(),v=p("p"),v.textContent=$t,tt=n(),m(G.$$.fragment),et=n(),I=p("p"),I.innerHTML=Ut,lt=n(),m(x.$$.fragment),at=n(),A=p("p"),A.innerHTML=bt,nt=n(),L=p("p"),L.innerHTML=gt,it=n(),m(H.$$.fragment),st=n(),_=p("p"),_.innerHTML=Tt,pt=n(),k=p("p"),k.textContent=Ct,ot=n(),m(Y.$$.fragment),mt=n(),E=p("p"),E.textContent=jt,rt=n(),m(F.$$.fragment),ct=n(),N=p("p"),N.innerHTML=Jt,ut=n(),m(R.$$.fragment),Mt=n(),Q=p("p"),this.h()},l(t){const e=kt("svelte-u9bgzb",document.head);s=o(e,"META",{name:!0,content:!0}),e.forEach(l),d=i(t),y=o(t,"P",{}),Gt(y).forEach(l),q=i(t),r($.$$.fragment,t),W=i(t),r(h.$$.fragment,t),Z=i(t),U=o(t,"P",{"data-svelte-h":!0}),w(U)!=="svelte-llq74"&&(U.innerHTML=yt),P=i(t),b=o(t,"P",{"data-svelte-h":!0}),w(b)!=="svelte-kldsj5"&&(b.innerHTML=ht),V=i(t),r(g.$$.fragment,t),X=i(t),r(T.$$.fragment,t),O=i(t),C=o(t,"P",{"data-svelte-h":!0}),w(C)!=="svelte-1but92h"&&(C.textContent=dt),z=i(t),r(j.$$.fragment,t),D=i(t),r(J.$$.fragment,t),K=i(t),v=o(t,"P",{"data-svelte-h":!0}),w(v)!=="svelte-1xohd7n"&&(v.textContent=$t),tt=i(t),r(G.$$.fragment,t),et=i(t),I=o(t,"P",{"data-svelte-h":!0}),w(I)!=="svelte-1r0jgu0"&&(I.innerHTML=Ut),lt=i(t),r(x.$$.fragment,t),at=i(t),A=o(t,"P",{"data-svelte-h":!0}),w(A)!=="svelte-3pqgmd"&&(A.innerHTML=bt),nt=i(t),L=o(t,"P",{"data-svelte-h":!0}),w(L)!=="svelte-56179e"&&(L.innerHTML=gt),it=i(t),r(H.$$.fragment,t),st=i(t),_=o(t,"P",{"data-svelte-h":!0}),w(_)!=="svelte-1pxj7s7"&&(_.innerHTML=Tt),pt=i(t),k=o(t,"P",{"data-svelte-h":!0}),w(k)!=="svelte-1aqbixr"&&(k.textContent=Ct),ot=i(t),r(Y.$$.fragment,t),mt=i(t),E=o(t,"P",{"data-svelte-h":!0}),w(E)!=="svelte-1qbn3ou"&&(E.textContent=jt),rt=i(t),r(F.$$.fragment,t),ct=i(t),N=o(t,"P",{"data-svelte-h":!0}),w(N)!=="svelte-rc3gcp"&&(N.innerHTML=Jt),ut=i(t),r(R.$$.fragment,t),Mt=i(t),Q=o(t,"P",{}),Gt(Q).forEach(l),this.h()},h(){It(s,"name","hf:doc:metadata"),It(s,"content",St)},m(t,e){Yt(document.head,s),a(t,d,e),a(t,y,e),a(t,q,e),c($,t,e),a(t,W,e),c(h,t,e),a(t,Z,e),a(t,U,e),a(t,P,e),a(t,b,e),a(t,V,e),c(g,t,e),a(t,X,e),c(T,t,e),a(t,O,e),a(t,C,e),a(t,z,e),c(j,t,e),a(t,D,e),c(J,t,e),a(t,K,e),a(t,v,e),a(t,tt,e),c(G,t,e),a(t,et,e),a(t,I,e),a(t,lt,e),c(x,t,e),a(t,at,e),a(t,A,e),a(t,nt,e),a(t,L,e),a(t,it,e),c(H,t,e),a(t,st,e),a(t,_,e),a(t,pt,e),a(t,k,e),a(t,ot,e),c(Y,t,e),a(t,mt,e),a(t,E,e),a(t,rt,e),c(F,t,e),a(t,ct,e),a(t,N,e),a(t,ut,e),c(R,t,e),a(t,Mt,e),a(t,Q,e),ft=!0},p(t,[e]){const vt={};e&2&&(vt.$$scope={dirty:e,ctx:t}),h.$set(vt)},i(t){ft||(u($.$$.fragment,t),u(h.$$.fragment,t),u(g.$$.fragment,t),u(T.$$.fragment,t),u(j.$$.fragment,t),u(J.$$.fragment,t),u(G.$$.fragment,t),u(x.$$.fragment,t),u(H.$$.fragment,t),u(Y.$$.fragment,t),u(F.$$.fragment,t),u(R.$$.fragment,t),ft=!0)},o(t){M($.$$.fragment,t),M(h.$$.fragment,t),M(g.$$.fragment,t),M(T.$$.fragment,t),M(j.$$.fragment,t),M(J.$$.fragment,t),M(G.$$.fragment,t),M(x.$$.fragment,t),M(H.$$.fragment,t),M(Y.$$.fragment,t),M(F.$$.fragment,t),M(R.$$.fragment,t),ft=!1},d(t){t&&(l(d),l(y),l(q),l(W),l(Z),l(U),l(P),l(b),l(V),l(X),l(O),l(C),l(z),l(D),l(K),l(v),l(tt),l(et),l(I),l(lt),l(at),l(A),l(nt),l(L),l(it),l(st),l(_),l(pt),l(k),l(ot),l(mt),l(E),l(rt),l(ct),l(N),l(ut),l(Mt),l(Q)),l(s),f($,t),f(h,t),f(g,t),f(T,t),f(j,t),f(J,t),f(G,t),f(x,t),f(H,t),f(Y,t),f(F,t),f(R,t)}}}const St='{"title":"GGUF usage with llama.cpp","local":"gguf-usage-with-llamacpp","sections":[{"title":"Option 1: Install with brew/ winget","local":"option-1-install-with-brew-winget","sections":[],"depth":3},{"title":"Option 2: build from source","local":"option-2-build-from-source","sections":[],"depth":3}],"depth":1}';function qt(B){return At(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Vt extends Ht{constructor(s){super(),_t(this,s,qt,Rt,xt,{})}}export{Vt as component};
