import{s as ve,a as _e,n as $e,o as ke}from"../chunks/scheduler.53228c21.js";import{S as Ce,i as Ee,e as s,s as a,c as p,h as Ie,a as o,d as l,b as n,f as de,g as f,j as r,k as B,l as We,m as i,n as u,t as c,o as d,p as M}from"../chunks/index.100fac89.js";import{C as Re}from"../chunks/CopyLLMTxtMenu.494046bd.js";import{C as ue}from"../chunks/CodeBlock.d30a6509.js";import{H as ce,E as xe}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.8999c127.js";function Be(Me){let h,X,z,H,g,Q,y,Y,J,he='The GGUF file format is typically used to store models for inference with <a href="https://github.com/ggerganov/ggml" rel="nofollow">GGML</a> and supports a variety of block wise quantization options. Diffusers supports loading checkpoints prequantized and saved in the GGUF format via <code>from_single_file</code> loading with Model classes. Loading GGUF checkpoints via Pipelines is currently not supported.',L,b,ge='The following example will load the <a href="https://huggingface.co/black-forest-labs/FLUX.1-dev" rel="nofollow">FLUX.1 DEV</a> transformer model using the GGUF Q2_K quantization variant.',A,w,ye="Before starting please install gguf in your environment",q,T,S,U,Je='Since GGUF is a single file format, use <code>~FromSingleFileMixin.from_single_file</code> to load the model and pass in the <a href="/docs/diffusers/main/en/api/quantization#diffusers.GGUFQuantizationConfig">GGUFQuantizationConfig</a>.',N,G,be="When using GGUF checkpoints, the quantized weights remain in a low memory <code>dtype</code>(typically <code>torch.uint8</code>) and are dynamically dequantized and cast to the configured <code>compute_dtype</code> during each module’s forward pass through the model. The <code>GGUFQuantizationConfig</code> allows you to set the <code>compute_dtype</code>.",D,Z,we='The functions used for dynamic dequantizatation are based on the great work done by <a href="https://github.com/city96/ComfyUI-GGUF" rel="nofollow">city96</a>, who created the Pytorch ports of the original <a href="https://github.com/ggerganov/llama.cpp/blob/master/gguf-py/gguf/quants.py" rel="nofollow"><code>numpy</code></a> implementation by <a href="https://github.com/compilade" rel="nofollow">compilade</a>.',P,F,K,j,O,v,Te="Optimized CUDA kernels can accelerate GGUF quantized model inference by approximately 10%. This functionality requires a compatible GPU with <code>torch.cuda.get_device_capability</code> greater than 7 and the kernels library:",ee,_,te,$,Ue="Once installed, set <code>DIFFUSERS_GGUF_CUDA_KERNELS=true</code>  to use optimized kernels when available. Note that CUDA kernels may introduce minor numerical differences compared to the original GGUF implementation, potentially causing subtle visual variations in generated images. To disable CUDA kernel usage, set the environment variable <code>DIFFUSERS_GGUF_CUDA_KERNELS=false</code>.",le,k,ie,C,Ge="<li>BF16</li> <li>Q4_0</li> <li>Q4_1</li> <li>Q5_0</li> <li>Q5_1</li> <li>Q8_0</li> <li>Q2_K</li> <li>Q3_K</li> <li>Q4_K</li> <li>Q5_K</li> <li>Q6_K</li>",ae,E,ne,I,Ze=`Use the Space below to convert a Diffusers checkpoint into the GGUF format for inference.
run conversion:`,se,m,Fe,oe,W,re,R,je=`When using Diffusers format GGUF checkpoints, it’s a must to provide the model <code>config</code> path. If the
model config resides in a <code>subfolder</code>, that needs to be specified, too.`,me,x,pe,V,fe;return g=new Re({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),y=new ce({props:{title:"GGUF",local:"gguf",headingTag:"h1"}}),T=new ue({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwZ2d1Zg==",highlighted:"pip install -U gguf",wrap:!1}}),F=new ue({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRmx1eFBpcGVsaW5lJTJDJTIwRmx1eFRyYW5zZm9ybWVyMkRNb2RlbCUyQyUyMEdHVUZRdWFudGl6YXRpb25Db25maWclMEElMEFja3B0X3BhdGglMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmNpdHk5NiUyRkZMVVguMS1kZXYtZ2d1ZiUyRmJsb2IlMkZtYWluJTJGZmx1eDEtZGV2LVEyX0suZ2d1ZiUyMiUwQSklMEF0cmFuc2Zvcm1lciUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9zaW5nbGVfZmlsZSglMEElMjAlMjAlMjAlMjBja3B0X3BhdGglMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNER0dVRlF1YW50aXphdGlvbkNvbmZpZyhjb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYpJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSklMEFwaXBlJTIwJTNEJTIwRmx1eFBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1kZXYlMjIlMkMlMEElMjAlMjAlMjAlMjB0cmFuc2Zvcm1lciUzRHRyYW5zZm9ybWVyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSklMEFwaXBlLmVuYWJsZV9tb2RlbF9jcHVfb2ZmbG9hZCgpJTBBcHJvbXB0JTIwJTNEJTIwJTIyQSUyMGNhdCUyMGhvbGRpbmclMjBhJTIwc2lnbiUyMHRoYXQlMjBzYXlzJTIwaGVsbG8lMjB3b3JsZCUyMiUwQWltYWdlJTIwJTNEJTIwcGlwZShwcm9tcHQlMkMlMjBnZW5lcmF0b3IlM0R0b3JjaC5tYW51YWxfc2VlZCgwKSkuaW1hZ2VzJTVCMCU1RCUwQWltYWdlLnNhdmUoJTIyZmx1eC1nZ3VmLnBuZyUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline, FluxTransformer2DModel, GGUFQuantizationConfig

ckpt_path = (
    <span class="hljs-string">&quot;https://huggingface.co/city96/FLUX.1-dev-gguf/blob/main/flux1-dev-Q2_K.gguf&quot;</span>
)
transformer = FluxTransformer2DModel.from_single_file(
    ckpt_path,
    quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
    torch_dtype=torch.bfloat16,
)
pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer,
    torch_dtype=torch.bfloat16,
)
pipe.enable_model_cpu_offload()
prompt = <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>
image = pipe(prompt, generator=torch.manual_seed(<span class="hljs-number">0</span>)).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;flux-gguf.png&quot;</span>)`,wrap:!1}}),j=new ce({props:{title:"Using Optimized CUDA Kernels with GGUF",local:"using-optimized-cuda-kernels-with-gguf",headingTag:"h2"}}),_=new ue({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwa2VybmVscw==",highlighted:"pip install -U kernels",wrap:!1}}),k=new ce({props:{title:"Supported Quantization Types",local:"supported-quantization-types",headingTag:"h2"}}),E=new ce({props:{title:"Convert to GGUF",local:"convert-to-gguf",headingTag:"h2"}}),W=new ue({props:{code:"aW1wb3J0JTIwdG9yY2glMEElMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRmx1eFBpcGVsaW5lJTJDJTIwRmx1eFRyYW5zZm9ybWVyMkRNb2RlbCUyQyUyMEdHVUZRdWFudGl6YXRpb25Db25maWclMEElMEFja3B0X3BhdGglMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRnNheWFrcGF1bCUyRmRpZmZlcmVudC1sb3JhLWZyb20tY2l2aXRhaSUyRmJsb2IlMkZtYWluJTJGZmx1eF9kZXZfZGlmZnVzZXJzLXE0XzAuZ2d1ZiUyMiUwQSklMEF0cmFuc2Zvcm1lciUyMCUzRCUyMEZsdXhUcmFuc2Zvcm1lcjJETW9kZWwuZnJvbV9zaW5nbGVfZmlsZSglMEElMjAlMjAlMjAlMjBja3B0X3BhdGglMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNER0dVRlF1YW50aXphdGlvbkNvbmZpZyhjb21wdXRlX2R0eXBlJTNEdG9yY2guYmZsb2F0MTYpJTJDJTBBJTIwJTIwJTIwJTIwY29uZmlnJTNEJTIyYmxhY2stZm9yZXN0LWxhYnMlMkZGTFVYLjEtZGV2JTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKSUwQXBpcGUlMjAlM0QlMjBGbHV4UGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYW5zZm9ybWVyJTNEdHJhbnNmb3JtZXIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKSUwQXBpcGUuZW5hYmxlX21vZGVsX2NwdV9vZmZsb2FkKCklMEFwcm9tcHQlMjAlM0QlMjAlMjJBJTIwY2F0JTIwaG9sZGluZyUyMGElMjBzaWduJTIwdGhhdCUyMHNheXMlMjBoZWxsbyUyMHdvcmxkJTIyJTBBaW1hZ2UlMjAlM0QlMjBwaXBlKHByb21wdCUyQyUyMGdlbmVyYXRvciUzRHRvcmNoLm1hbnVhbF9zZWVkKDApKS5pbWFnZXMlNUIwJTVEJTBBaW1hZ2Uuc2F2ZSglMjJmbHV4LWdndWYucG5nJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> torch

<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxPipeline, FluxTransformer2DModel, GGUFQuantizationConfig

ckpt_path = (
    <span class="hljs-string">&quot;https://huggingface.co/sayakpaul/different-lora-from-civitai/blob/main/flux_dev_diffusers-q4_0.gguf&quot;</span>
)
transformer = FluxTransformer2DModel.from_single_file(
    ckpt_path,
    quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
    config=<span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    torch_dtype=torch.bfloat16,
)
pipe = FluxPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>,
    transformer=transformer,
    torch_dtype=torch.bfloat16,
)
pipe.enable_model_cpu_offload()
prompt = <span class="hljs-string">&quot;A cat holding a sign that says hello world&quot;</span>
image = pipe(prompt, generator=torch.manual_seed(<span class="hljs-number">0</span>)).images[<span class="hljs-number">0</span>]
image.save(<span class="hljs-string">&quot;flux-gguf.png&quot;</span>)`,wrap:!1}}),x=new xe({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/quantization/gguf.md"}}),{c(){h=s("meta"),X=a(),z=s("p"),H=a(),p(g.$$.fragment),Q=a(),p(y.$$.fragment),Y=a(),J=s("p"),J.innerHTML=he,L=a(),b=s("p"),b.innerHTML=ge,A=a(),w=s("p"),w.textContent=ye,q=a(),p(T.$$.fragment),S=a(),U=s("p"),U.innerHTML=Je,N=a(),G=s("p"),G.innerHTML=be,D=a(),Z=s("p"),Z.innerHTML=we,P=a(),p(F.$$.fragment),K=a(),p(j.$$.fragment),O=a(),v=s("p"),v.innerHTML=Te,ee=a(),p(_.$$.fragment),te=a(),$=s("p"),$.innerHTML=Ue,le=a(),p(k.$$.fragment),ie=a(),C=s("ul"),C.innerHTML=Ge,ae=a(),p(E.$$.fragment),ne=a(),I=s("p"),I.textContent=Ze,se=a(),m=s("iframe"),oe=a(),p(W.$$.fragment),re=a(),R=s("p"),R.innerHTML=je,me=a(),p(x.$$.fragment),pe=a(),V=s("p"),this.h()},l(e){const t=Ie("svelte-u9bgzb",document.head);h=o(t,"META",{name:!0,content:!0}),t.forEach(l),X=n(e),z=o(e,"P",{}),de(z).forEach(l),H=n(e),f(g.$$.fragment,e),Q=n(e),f(y.$$.fragment,e),Y=n(e),J=o(e,"P",{"data-svelte-h":!0}),r(J)!=="svelte-th49fe"&&(J.innerHTML=he),L=n(e),b=o(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-15l4qp1"&&(b.innerHTML=ge),A=n(e),w=o(e,"P",{"data-svelte-h":!0}),r(w)!=="svelte-1s5awbc"&&(w.textContent=ye),q=n(e),f(T.$$.fragment,e),S=n(e),U=o(e,"P",{"data-svelte-h":!0}),r(U)!=="svelte-v5zdiv"&&(U.innerHTML=Je),N=n(e),G=o(e,"P",{"data-svelte-h":!0}),r(G)!=="svelte-1tiy01a"&&(G.innerHTML=be),D=n(e),Z=o(e,"P",{"data-svelte-h":!0}),r(Z)!=="svelte-2xs1wc"&&(Z.innerHTML=we),P=n(e),f(F.$$.fragment,e),K=n(e),f(j.$$.fragment,e),O=n(e),v=o(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-gm6vu2"&&(v.innerHTML=Te),ee=n(e),f(_.$$.fragment,e),te=n(e),$=o(e,"P",{"data-svelte-h":!0}),r($)!=="svelte-14ty30u"&&($.innerHTML=Ue),le=n(e),f(k.$$.fragment,e),ie=n(e),C=o(e,"UL",{"data-svelte-h":!0}),r(C)!=="svelte-1dde1tf"&&(C.innerHTML=Ge),ae=n(e),f(E.$$.fragment,e),ne=n(e),I=o(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-13q7i79"&&(I.textContent=Ze),se=n(e),m=o(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),de(m).forEach(l),oe=n(e),f(W.$$.fragment,e),re=n(e),R=o(e,"P",{"data-svelte-h":!0}),r(R)!=="svelte-sfexoo"&&(R.innerHTML=je),me=n(e),f(x.$$.fragment,e),pe=n(e),V=o(e,"P",{}),de(V).forEach(l),this.h()},h(){B(h,"name","hf:doc:metadata"),B(h,"content",ze),_e(m.src,Fe="https://diffusers-internal-dev-diffusers-to-gguf.hf.space")||B(m,"src",Fe),B(m,"frameborder","0"),B(m,"width","850"),B(m,"height","450")},m(e,t){We(document.head,h),i(e,X,t),i(e,z,t),i(e,H,t),u(g,e,t),i(e,Q,t),u(y,e,t),i(e,Y,t),i(e,J,t),i(e,L,t),i(e,b,t),i(e,A,t),i(e,w,t),i(e,q,t),u(T,e,t),i(e,S,t),i(e,U,t),i(e,N,t),i(e,G,t),i(e,D,t),i(e,Z,t),i(e,P,t),u(F,e,t),i(e,K,t),u(j,e,t),i(e,O,t),i(e,v,t),i(e,ee,t),u(_,e,t),i(e,te,t),i(e,$,t),i(e,le,t),u(k,e,t),i(e,ie,t),i(e,C,t),i(e,ae,t),u(E,e,t),i(e,ne,t),i(e,I,t),i(e,se,t),i(e,m,t),i(e,oe,t),u(W,e,t),i(e,re,t),i(e,R,t),i(e,me,t),u(x,e,t),i(e,pe,t),i(e,V,t),fe=!0},p:$e,i(e){fe||(c(g.$$.fragment,e),c(y.$$.fragment,e),c(T.$$.fragment,e),c(F.$$.fragment,e),c(j.$$.fragment,e),c(_.$$.fragment,e),c(k.$$.fragment,e),c(E.$$.fragment,e),c(W.$$.fragment,e),c(x.$$.fragment,e),fe=!0)},o(e){d(g.$$.fragment,e),d(y.$$.fragment,e),d(T.$$.fragment,e),d(F.$$.fragment,e),d(j.$$.fragment,e),d(_.$$.fragment,e),d(k.$$.fragment,e),d(E.$$.fragment,e),d(W.$$.fragment,e),d(x.$$.fragment,e),fe=!1},d(e){e&&(l(X),l(z),l(H),l(Q),l(Y),l(J),l(L),l(b),l(A),l(w),l(q),l(S),l(U),l(N),l(G),l(D),l(Z),l(P),l(K),l(O),l(v),l(ee),l(te),l($),l(le),l(ie),l(C),l(ae),l(ne),l(I),l(se),l(m),l(oe),l(re),l(R),l(me),l(pe),l(V)),l(h),M(g,e),M(y,e),M(T,e),M(F,e),M(j,e),M(_,e),M(k,e),M(E,e),M(W,e),M(x,e)}}}const ze='{"title":"GGUF","local":"gguf","sections":[{"title":"Using Optimized CUDA Kernels with GGUF","local":"using-optimized-cuda-kernels-with-gguf","sections":[],"depth":2},{"title":"Supported Quantization Types","local":"supported-quantization-types","sections":[],"depth":2},{"title":"Convert to GGUF","local":"convert-to-gguf","sections":[],"depth":2}],"depth":1}';function Ve(Me){return ke(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ae extends Ce{constructor(h){super(),Ee(this,h,Ve,Be,ve,{})}}export{Ae as component};
