import{s as Za,o as ga,n as v}from"../chunks/scheduler.8c3d61f6.js";import{S as _a,i as Ia,g as c,s as t,r as M,A as xa,h as d,f as l,c as i,j as ja,u,x as f,k as Ta,y as Ga,a as n,v as y,d as J,t as h,w as b}from"../chunks/index.da70eac4.js";import{T as C}from"../chunks/Tip.1d9b8c37.js";import{C as Z}from"../chunks/CodeBlock.a9c4becf.js";import{H as G,E as va}from"../chunks/getInferenceSnippets.cd165132.js";import{H as Ua,a as fn}from"../chunks/HfOption.6c3b4e77.js";function Ca(T){let a,w="Keep in mind these techniques may need to be adjusted depending on the model. For example, a transformer-based diffusion model may not benefit equally from these memory optimizations as a UNet-based model.";return{c(){a=c("p"),a.textContent=w},l(p){a=d(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1wzvfzx"&&(a.textContent=w)},m(p,r){n(p,a,r)},p:v,d(p){p&&l(a)}}}function Wa(T){let a,w="Device placement is an experimental feature and the API may change. Only the <code>balanced</code> strategy is supported at the moment. We plan to support additional mapping strategies in the future.";return{c(){a=c("p"),a.innerHTML=w},l(p){a=d(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1tuq9h4"&&(a.innerHTML=w)},m(p,r){n(p,a,r)},p:v,d(p){p&&l(a)}}}function Ba(T){let a,w="The <code>balanced</code> device placement strategy evenly splits the pipeline across all available devices.",p,r,m,o,U="You can inspect a pipeline’s device map with <code>hf_device_map</code>.",I,g,x;return r=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwU3RhYmxlRGlmZnVzaW9uWExQaXBlbGluZSUwQSUwQXBpcGVsaW5lJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uWExQaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyc3RhYmlsaXR5YWklMkZzdGFibGUtZGlmZnVzaW9uLXhsLWJhc2UtMS4wJTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmJhbGFuY2VkJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, StableDiffusionXLPipeline

pipeline = StableDiffusionXLPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-string">&quot;balanced&quot;</span>
)`,wrap:!1}}),g=new Z({props:{code:"cHJpbnQocGlwZWxpbmUuaGZfZGV2aWNlX21hcCklMEElN0IndW5ldCclM0ElMjAxJTJDJTIwJ3ZhZSclM0ElMjAxJTJDJTIwJ3NhZmV0eV9jaGVja2VyJyUzQSUyMDAlMkMlMjAndGV4dF9lbmNvZGVyJyUzQSUyMDAlN0Q=",highlighted:`<span class="hljs-built_in">print</span>(pipeline.hf_device_map)
{<span class="hljs-string">&#x27;unet&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;vae&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;safety_checker&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;text_encoder&#x27;</span>: <span class="hljs-number">0</span>}`,wrap:!1}}),{c(){a=c("p"),a.innerHTML=w,p=t(),M(r.$$.fragment),m=t(),o=c("p"),o.innerHTML=U,I=t(),M(g.$$.fragment)},l(j){a=d(j,"P",{"data-svelte-h":!0}),f(a)!=="svelte-13intml"&&(a.innerHTML=w),p=i(j),u(r.$$.fragment,j),m=i(j),o=d(j,"P",{"data-svelte-h":!0}),f(o)!=="svelte-cfrzyx"&&(o.innerHTML=U),I=i(j),u(g.$$.fragment,j)},m(j,_){n(j,a,_),n(j,p,_),y(r,j,_),n(j,m,_),n(j,o,_),n(j,I,_),y(g,j,_),x=!0},p:v,i(j){x||(J(r.$$.fragment,j),J(g.$$.fragment,j),x=!0)},o(j){h(r.$$.fragment,j),h(g.$$.fragment,j),x=!1},d(j){j&&(l(a),l(p),l(m),l(o),l(I)),b(r,j),b(g,j)}}}function ka(T){let a,w='The <code>device_map</code> is useful for loading large models, such as the Flux diffusion transformer which has 12.5B parameters. Set it to <code>&quot;auto&quot;</code> to automatically distribute a model across the fastest device first before moving to slower devices. Refer to the <a href="../training/distributed_inference#model-sharding">Model sharding</a> docs for more details.',p,r,m,o,U="You can inspect a model’s device map with <code>hf_device_map</code>.",I,g,x;return r=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTBBJTBBdHJhbnNmb3JtZXIlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel

transformer = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    torch_dtype=torch.bfloat16
)`,wrap:!1}}),g=new Z({props:{code:"cHJpbnQodHJhbnNmb3JtZXIuaGZfZGV2aWNlX21hcCk=",highlighted:'<span class="hljs-built_in">print</span>(transformer.hf_device_map)',wrap:!1}}),{c(){a=c("p"),a.innerHTML=w,p=t(),M(r.$$.fragment),m=t(),o=c("p"),o.innerHTML=U,I=t(),M(g.$$.fragment)},l(j){a=d(j,"P",{"data-svelte-h":!0}),f(a)!=="svelte-djqsxo"&&(a.innerHTML=w),p=i(j),u(r.$$.fragment,j),m=i(j),o=d(j,"P",{"data-svelte-h":!0}),f(o)!=="svelte-af1u4w"&&(o.innerHTML=U),I=i(j),u(g.$$.fragment,j)},m(j,_){n(j,a,_),n(j,p,_),y(r,j,_),n(j,m,_),n(j,o,_),n(j,I,_),y(g,j,_),x=!0},p:v,i(j){x||(J(r.$$.fragment,j),J(g.$$.fragment,j),x=!0)},o(j){h(r.$$.fragment,j),h(g.$$.fragment,j),x=!1},d(j){j&&(l(a),l(p),l(m),l(o),l(I)),b(r,j),b(g,j)}}}function Va(T){let a,w,p,r;return a=new fn({props:{id:"device-map",option:"pipeline level",$$slots:{default:[Ba]},$$scope:{ctx:T}}}),p=new fn({props:{id:"device-map",option:"model level",$$slots:{default:[ka]},$$scope:{ctx:T}}}),{c(){M(a.$$.fragment),w=t(),M(p.$$.fragment)},l(m){u(a.$$.fragment,m),w=i(m),u(p.$$.fragment,m)},m(m,o){y(a,m,o),n(m,w,o),y(p,m,o),r=!0},p(m,o){const U={};o&2&&(U.$$scope={dirty:o,ctx:m}),a.$set(U);const I={};o&2&&(I.$$scope={dirty:o,ctx:m}),p.$set(I)},i(m){r||(J(a.$$.fragment,m),J(p.$$.fragment,m),r=!0)},o(m){h(a.$$.fragment,m),h(p.$$.fragment,m),r=!1},d(m){m&&l(w),b(a,m),b(p,m)}}}function $a(T){let a,w='The <a href="/docs/diffusers/main/en/api/models/autoencoder_kl_wan#diffusers.AutoencoderKLWan">AutoencoderKLWan</a> and <a href="/docs/diffusers/main/en/api/models/asymmetricautoencoderkl#diffusers.AsymmetricAutoencoderKL">AsymmetricAutoencoderKL</a> classes don’t support slicing.';return{c(){a=c("p"),a.innerHTML=w},l(p){a=d(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-ouja0u"&&(a.innerHTML=w)},m(p,r){n(p,a,r)},p:v,d(p){p&&l(a)}}}function Xa(T){let a,w='Don’t move the pipeline to CUDA before calling <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_sequential_cpu_offload">enable_sequential_cpu_offload()</a>, otherwise the amount of memory saved is only minimal (refer to this <a href="https://github.com/huggingface/diffusers/issues/1934" rel="nofollow">issue</a> for more details). This is a stateful operation that installs hooks on the model.';return{c(){a=c("p"),a.innerHTML=w},l(p){a=d(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1d1bxgj"&&(a.innerHTML=w)},m(p,r){n(p,a,r)},p:v,d(p){p&&l(a)}}}function Na(T){let a,w='Keep in mind that if models are reused outside the pipeline after hookes have been installed (see <a href="https://huggingface.co/docs/accelerate/en/package_reference/big_modeling#accelerate.hooks.remove_hook_from_module" rel="nofollow">Removing Hooks</a> for more details), you need to run the entire pipeline and models in the expected order to properly offload them. This is a stateful operation that installs hooks on the model.';return{c(){a=c("p"),a.innerHTML=w},l(p){a=d(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-fx1eup"&&(a.innerHTML=w)},m(p,r){n(p,a,r)},p:v,d(p){p&&l(a)}}}function Ya(T){let a,w="Group offloading may not work with all models if the forward implementation contains weight-dependent device casting of inputs because it may clash with group offloading’s device casting mechanism.";return{c(){a=c("p"),a.textContent=w},l(p){a=d(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-78t6ea"&&(a.textContent=w)},m(p,r){n(p,a,r)},p:v,d(p){p&&l(a)}}}function za(T){let a,w='Call <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_group_offload">enable_group_offload()</a> on a pipeline.',p,r,m;return r=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQ29nVmlkZW9YUGlwZWxpbmUlMEFmcm9tJTIwZGlmZnVzZXJzLmhvb2tzJTIwaW1wb3J0JTIwYXBwbHlfZ3JvdXBfb2ZmbG9hZGluZyUwQWZyb20lMjBkaWZmdXNlcnMudXRpbHMlMjBpbXBvcnQlMjBleHBvcnRfdG9fdmlkZW8lMEElMEFvbmxvYWRfZGV2aWNlJTIwJTNEJTIwdG9yY2guZGV2aWNlKCUyMmN1ZGElMjIpJTBBb2ZmbG9hZF9kZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3B1JTIyKSUwQSUwQXBpcGVsaW5lJTIwJTNEJTIwQ29nVmlkZW9YUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUyMlRIVURNJTJGQ29nVmlkZW9YLTViJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiklMEFwaXBlbGluZS5lbmFibGVfZ3JvdXBfb2ZmbG9hZCglMEElMjAlMjAlMjAlMjBvbmxvYWRfZGV2aWNlJTNEb25sb2FkX2RldmljZSUyQyUwQSUyMCUyMCUyMCUyMG9mZmxvYWRfZGV2aWNlJTNEb2ZmbG9hZF9kZXZpY2UlMkMlMEElMjAlMjAlMjAlMjBvZmZsb2FkX3R5cGUlM0QlMjJsZWFmX2xldmVsJTIyJTJDJTBBJTIwJTIwJTIwJTIwdXNlX3N0cmVhbSUzRFRydWUlMEEpJTBBJTBBcHJvbXB0JTIwJTNEJTIwKCUwQSUyMCUyMCUyMCUyMCUyMkElMjBwYW5kYSUyQyUyMGRyZXNzZWQlMjBpbiUyMGElMjBzbWFsbCUyQyUyMHJlZCUyMGphY2tldCUyMGFuZCUyMGElMjB0aW55JTIwaGF0JTJDJTIwc2l0cyUyMG9uJTIwYSUyMHdvb2RlbiUyMHN0b29sJTIwaW4lMjBhJTIwc2VyZW5lJTIwYmFtYm9vJTIwZm9yZXN0LiUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMlRoZSUyMHBhbmRhJ3MlMjBmbHVmZnklMjBwYXdzJTIwc3RydW0lMjBhJTIwbWluaWF0dXJlJTIwYWNvdXN0aWMlMjBndWl0YXIlMkMlMjBwcm9kdWNpbmclMjBzb2Z0JTJDJTIwbWVsb2RpYyUyMHR1bmVzLiUyME5lYXJieSUyQyUyMGElMjBmZXclMjBvdGhlciUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMnBhbmRhcyUyMGdhdGhlciUyQyUyMHdhdGNoaW5nJTIwY3VyaW91c2x5JTIwYW5kJTIwc29tZSUyMGNsYXBwaW5nJTIwaW4lMjByaHl0aG0uJTIwU3VubGlnaHQlMjBmaWx0ZXJzJTIwdGhyb3VnaCUyMHRoZSUyMHRhbGwlMjBiYW1ib28lMkMlMjAlMjIlMEElMjAlMjAlMjAlMjAlMjJjYXN0aW5nJTIwYSUyMGdlbnRsZSUyMGdsb3clMjBvbiUyMHRoZSUyMHNjZW5lLiUyMFRoZSUyMHBhbmRhJ3MlMjBmYWNlJTIwaXMlMjBleHByZXNzaXZlJTJDJTIwc2hvd2luZyUyMGNvbmNlbnRyYXRpb24lMjBhbmQlMjBqb3klMjBhcyUyMGl0JTIwcGxheXMuJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIyVGhlJTIwYmFja2dyb3VuZCUyMGluY2x1ZGVzJTIwYSUyMHNtYWxsJTJDJTIwZmxvd2luZyUyMHN0cmVhbSUyMGFuZCUyMHZpYnJhbnQlMjBncmVlbiUyMGZvbGlhZ2UlMkMlMjBlbmhhbmNpbmclMjB0aGUlMjBwZWFjZWZ1bCUyMGFuZCUyMG1hZ2ljYWwlMjAlMjIlMEElMjAlMjAlMjAlMjAlMjJhdG1vc3BoZXJlJTIwb2YlMjB0aGlzJTIwdW5pcXVlJTIwbXVzaWNhbCUyMHBlcmZvcm1hbmNlLiUyMiUwQSklMEF2aWRlbyUyMCUzRCUyMHBpcGVsaW5lKHByb21wdCUzRHByb21wdCUyQyUyMGd1aWRhbmNlX3NjYWxlJTNENiUyQyUyMG51bV9pbmZlcmVuY2Vfc3RlcHMlM0Q1MCkuZnJhbWVzJTVCMCU1RCUwQXByaW50KGYlMjJNYXglMjBtZW1vcnklMjByZXNlcnZlZCUzQSUyMCU3QnRvcmNoLmN1ZGEubWF4X21lbW9yeV9hbGxvY2F0ZWQoKSUyMCUyRiUyMDEwMjQqKjMlM0EuMmYlN0QlMjBHQiUyMiklMEFleHBvcnRfdG9fdmlkZW8odmlkZW8lMkMlMjAlMjJvdXRwdXQubXA0JTIyJTJDJTIwZnBzJTNEOCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline
<span class="hljs-keyword">from</span> diffusers.hooks <span class="hljs-keyword">import</span> apply_group_offloading
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

onload_device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
offload_device = torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)

pipeline = CogVideoXPipeline.from_pretrained(<span class="hljs-string">&quot;THUDM/CogVideoX-5b&quot;</span>, torch_dtype=torch.bfloat16)
pipeline.enable_group_offload(
    onload_device=onload_device,
    offload_device=offload_device,
    offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>,
    use_stream=<span class="hljs-literal">True</span>
)

prompt = (
    <span class="hljs-string">&quot;A panda, dressed in a small, red jacket and a tiny hat, sits on a wooden stool in a serene bamboo forest. &quot;</span>
    <span class="hljs-string">&quot;The panda&#x27;s fluffy paws strum a miniature acoustic guitar, producing soft, melodic tunes. Nearby, a few other &quot;</span>
    <span class="hljs-string">&quot;pandas gather, watching curiously and some clapping in rhythm. Sunlight filters through the tall bamboo, &quot;</span>
    <span class="hljs-string">&quot;casting a gentle glow on the scene. The panda&#x27;s face is expressive, showing concentration and joy as it plays. &quot;</span>
    <span class="hljs-string">&quot;The background includes a small, flowing stream and vibrant green foliage, enhancing the peaceful and magical &quot;</span>
    <span class="hljs-string">&quot;atmosphere of this unique musical performance.&quot;</span>
)
video = pipeline(prompt=prompt, guidance_scale=<span class="hljs-number">6</span>, num_inference_steps=<span class="hljs-number">50</span>).frames[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Max memory reserved: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB&quot;</span>)
export_to_video(video, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">8</span>)`,wrap:!1}}),{c(){a=c("p"),a.innerHTML=w,p=t(),M(r.$$.fragment)},l(o){a=d(o,"P",{"data-svelte-h":!0}),f(a)!=="svelte-7or8mg"&&(a.innerHTML=w),p=i(o),u(r.$$.fragment,o)},m(o,U){n(o,a,U),n(o,p,U),y(r,o,U),m=!0},p:v,i(o){m||(J(r.$$.fragment,o),m=!0)},o(o){h(r.$$.fragment,o),m=!1},d(o){o&&(l(a),l(p)),b(r,o)}}}function Ra(T){let a,w='Call <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.enable_group_offload">enable_group_offload()</a> on standard Diffusers model components that inherit from <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin">ModelMixin</a>. For other model components that don’t inherit from <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin">ModelMixin</a>, such as a generic <a href="https://pytorch.org/docs/stable/generated/torch.nn.Module.html" rel="nofollow">torch.nn.Module</a>, use <a href="/docs/diffusers/main/en/api/utilities#diffusers.hooks.apply_group_offloading">apply_group_offloading()</a> instead.',p,r,m;return r=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQ29nVmlkZW9YUGlwZWxpbmUlMEFmcm9tJTIwZGlmZnVzZXJzLmhvb2tzJTIwaW1wb3J0JTIwYXBwbHlfZ3JvdXBfb2ZmbG9hZGluZyUwQWZyb20lMjBkaWZmdXNlcnMudXRpbHMlMjBpbXBvcnQlMjBleHBvcnRfdG9fdmlkZW8lMEElMEFvbmxvYWRfZGV2aWNlJTIwJTNEJTIwdG9yY2guZGV2aWNlKCUyMmN1ZGElMjIpJTBBb2ZmbG9hZF9kZXZpY2UlMjAlM0QlMjB0b3JjaC5kZXZpY2UoJTIyY3B1JTIyKSUwQXBpcGVsaW5lJTIwJTNEJTIwQ29nVmlkZW9YUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUyMlRIVURNJTJGQ29nVmlkZW9YLTViJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiklMEElMEElMjMlMjBVc2UlMjB0aGUlMjBlbmFibGVfZ3JvdXBfb2ZmbG9hZCUyMG1ldGhvZCUyMGZvciUyMERpZmZ1c2VycyUyMG1vZGVsJTIwaW1wbGVtZW50YXRpb25zJTBBcGlwZWxpbmUudHJhbnNmb3JtZXIuZW5hYmxlX2dyb3VwX29mZmxvYWQob25sb2FkX2RldmljZSUzRG9ubG9hZF9kZXZpY2UlMkMlMjBvZmZsb2FkX2RldmljZSUzRG9mZmxvYWRfZGV2aWNlJTJDJTIwb2ZmbG9hZF90eXBlJTNEJTIybGVhZl9sZXZlbCUyMiklMEFwaXBlbGluZS52YWUuZW5hYmxlX2dyb3VwX29mZmxvYWQob25sb2FkX2RldmljZSUzRG9ubG9hZF9kZXZpY2UlMkMlMjBvZmZsb2FkX3R5cGUlM0QlMjJsZWFmX2xldmVsJTIyKSUwQSUwQSUyMyUyMFVzZSUyMHRoZSUyMGFwcGx5X2dyb3VwX29mZmxvYWRpbmclMjBtZXRob2QlMjBmb3IlMjBvdGhlciUyMG1vZGVsJTIwY29tcG9uZW50cyUwQWFwcGx5X2dyb3VwX29mZmxvYWRpbmcocGlwZWxpbmUudGV4dF9lbmNvZGVyJTJDJTIwb25sb2FkX2RldmljZSUzRG9ubG9hZF9kZXZpY2UlMkMlMjBvZmZsb2FkX3R5cGUlM0QlMjJibG9ja19sZXZlbCUyMiUyQyUyMG51bV9ibG9ja3NfcGVyX2dyb3VwJTNEMiklMEElMEFwcm9tcHQlMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwJTIyQSUyMHBhbmRhJTJDJTIwZHJlc3NlZCUyMGluJTIwYSUyMHNtYWxsJTJDJTIwcmVkJTIwamFja2V0JTIwYW5kJTIwYSUyMHRpbnklMjBoYXQlMkMlMjBzaXRzJTIwb24lMjBhJTIwd29vZGVuJTIwc3Rvb2wlMjBpbiUyMGElMjBzZXJlbmUlMjBiYW1ib28lMjBmb3Jlc3QuJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIyVGhlJTIwcGFuZGEncyUyMGZsdWZmeSUyMHBhd3MlMjBzdHJ1bSUyMGElMjBtaW5pYXR1cmUlMjBhY291c3RpYyUyMGd1aXRhciUyQyUyMHByb2R1Y2luZyUyMHNvZnQlMkMlMjBtZWxvZGljJTIwdHVuZXMuJTIwTmVhcmJ5JTJDJTIwYSUyMGZldyUyMG90aGVyJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIycGFuZGFzJTIwZ2F0aGVyJTJDJTIwd2F0Y2hpbmclMjBjdXJpb3VzbHklMjBhbmQlMjBzb21lJTIwY2xhcHBpbmclMjBpbiUyMHJoeXRobS4lMjBTdW5saWdodCUyMGZpbHRlcnMlMjB0aHJvdWdoJTIwdGhlJTIwdGFsbCUyMGJhbWJvbyUyQyUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMmNhc3RpbmclMjBhJTIwZ2VudGxlJTIwZ2xvdyUyMG9uJTIwdGhlJTIwc2NlbmUuJTIwVGhlJTIwcGFuZGEncyUyMGZhY2UlMjBpcyUyMGV4cHJlc3NpdmUlMkMlMjBzaG93aW5nJTIwY29uY2VudHJhdGlvbiUyMGFuZCUyMGpveSUyMGFzJTIwaXQlMjBwbGF5cy4lMjAlMjIlMEElMjAlMjAlMjAlMjAlMjJUaGUlMjBiYWNrZ3JvdW5kJTIwaW5jbHVkZXMlMjBhJTIwc21hbGwlMkMlMjBmbG93aW5nJTIwc3RyZWFtJTIwYW5kJTIwdmlicmFudCUyMGdyZWVuJTIwZm9saWFnZSUyQyUyMGVuaGFuY2luZyUyMHRoZSUyMHBlYWNlZnVsJTIwYW5kJTIwbWFnaWNhbCUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMmF0bW9zcGhlcmUlMjBvZiUyMHRoaXMlMjB1bmlxdWUlMjBtdXNpY2FsJTIwcGVyZm9ybWFuY2UuJTIyJTBBKSUwQXZpZGVvJTIwJTNEJTIwcGlwZWxpbmUocHJvbXB0JTNEcHJvbXB0JTJDJTIwZ3VpZGFuY2Vfc2NhbGUlM0Q2JTJDJTIwbnVtX2luZmVyZW5jZV9zdGVwcyUzRDUwKS5mcmFtZXMlNUIwJTVEJTBBcHJpbnQoZiUyMk1heCUyMG1lbW9yeSUyMHJlc2VydmVkJTNBJTIwJTdCdG9yY2guY3VkYS5tYXhfbWVtb3J5X2FsbG9jYXRlZCgpJTIwJTJGJTIwMTAyNCoqMyUzQS4yZiU3RCUyMEdCJTIyKSUwQWV4cG9ydF90b192aWRlbyh2aWRlbyUyQyUyMCUyMm91dHB1dC5tcDQlMjIlMkMlMjBmcHMlM0Q4KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline
<span class="hljs-keyword">from</span> diffusers.hooks <span class="hljs-keyword">import</span> apply_group_offloading
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

onload_device = torch.device(<span class="hljs-string">&quot;cuda&quot;</span>)
offload_device = torch.device(<span class="hljs-string">&quot;cpu&quot;</span>)
pipeline = CogVideoXPipeline.from_pretrained(<span class="hljs-string">&quot;THUDM/CogVideoX-5b&quot;</span>, torch_dtype=torch.bfloat16)

<span class="hljs-comment"># Use the enable_group_offload method for Diffusers model implementations</span>
pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>)
pipeline.vae.enable_group_offload(onload_device=onload_device, offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>)

<span class="hljs-comment"># Use the apply_group_offloading method for other model components</span>
apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type=<span class="hljs-string">&quot;block_level&quot;</span>, num_blocks_per_group=<span class="hljs-number">2</span>)

prompt = (
    <span class="hljs-string">&quot;A panda, dressed in a small, red jacket and a tiny hat, sits on a wooden stool in a serene bamboo forest. &quot;</span>
    <span class="hljs-string">&quot;The panda&#x27;s fluffy paws strum a miniature acoustic guitar, producing soft, melodic tunes. Nearby, a few other &quot;</span>
    <span class="hljs-string">&quot;pandas gather, watching curiously and some clapping in rhythm. Sunlight filters through the tall bamboo, &quot;</span>
    <span class="hljs-string">&quot;casting a gentle glow on the scene. The panda&#x27;s face is expressive, showing concentration and joy as it plays. &quot;</span>
    <span class="hljs-string">&quot;The background includes a small, flowing stream and vibrant green foliage, enhancing the peaceful and magical &quot;</span>
    <span class="hljs-string">&quot;atmosphere of this unique musical performance.&quot;</span>
)
video = pipeline(prompt=prompt, guidance_scale=<span class="hljs-number">6</span>, num_inference_steps=<span class="hljs-number">50</span>).frames[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Max memory reserved: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB&quot;</span>)
export_to_video(video, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">8</span>)`,wrap:!1}}),{c(){a=c("p"),a.innerHTML=w,p=t(),M(r.$$.fragment)},l(o){a=d(o,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1tfd1v"&&(a.innerHTML=w),p=i(o),u(r.$$.fragment,o)},m(o,U){n(o,a,U),n(o,p,U),y(r,o,U),m=!0},p:v,i(o){m||(J(r.$$.fragment,o),m=!0)},o(o){h(r.$$.fragment,o),m=!1},d(o){o&&(l(a),l(p)),b(r,o)}}}function Qa(T){let a,w,p,r;return a=new fn({props:{id:"group-offloading",option:"pipeline",$$slots:{default:[za]},$$scope:{ctx:T}}}),p=new fn({props:{id:"group-offloading",option:"model",$$slots:{default:[Ra]},$$scope:{ctx:T}}}),{c(){M(a.$$.fragment),w=t(),M(p.$$.fragment)},l(m){u(a.$$.fragment,m),w=i(m),u(p.$$.fragment,m)},m(m,o){y(a,m,o),n(m,w,o),y(p,m,o),r=!0},p(m,o){const U={};o&2&&(U.$$scope={dirty:o,ctx:m}),a.$set(U);const I={};o&2&&(I.$$scope={dirty:o,ctx:m}),p.$set(I)},i(m){r||(J(a.$$.fragment,m),J(p.$$.fragment,m),r=!0)},o(m){h(a.$$.fragment,m),h(p.$$.fragment,m),r=!1},d(m){m&&l(w),b(a,m),b(p,m)}}}function Ha(T){let a,w="When <code>use_stream=True</code> on VAEs with tiling enabled, make sure to do a dummy forward pass (possible with dummy inputs as well) before inference to avoid device mismatch errors. This may not work on all implementations, so feel free to open an issue if you encounter any problems.";return{c(){a=c("p"),a.innerHTML=w},l(p){a=d(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1j8tvtw"&&(a.innerHTML=w)},m(p,r){n(p,a,r)},p:v,d(p){p&&l(a)}}}function Ea(T){let a,w='Combine layerwise casting with <a href="#group-offloading">group offloading</a> for even more memory savings.';return{c(){a=c("p"),a.innerHTML=w},l(p){a=d(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-1u3ymti"&&(a.innerHTML=w)},m(p,r){n(p,a,r)},p:v,d(p){p&&l(a)}}}function Sa(T){let a,w='Layerwise casting may not work with all models if the forward implementation contains internal typecasting of weights. The current implementation of layerwise casting assumes the forward pass is independent of the weight precision and the input datatypes are always specified in <code>compute_dtype</code> (see <a href="https://github.com/huggingface/transformers/blob/7f5077e53682ca855afc826162b204ebf809f1f9/src/transformers/models/t5/modeling_t5.py#L294-L299" rel="nofollow">here</a> for an incompatible implementation).',p,r,m='Layerwise casting may also fail on custom modeling implementations with <a href="https://huggingface.co/docs/peft/index" rel="nofollow">PEFT</a> layers. There are some checks available but they are not extensively tested or guaranteed to work in all cases.';return{c(){a=c("p"),a.innerHTML=w,p=t(),r=c("p"),r.innerHTML=m},l(o){a=d(o,"P",{"data-svelte-h":!0}),f(a)!=="svelte-nfpqdo"&&(a.innerHTML=w),p=i(o),r=d(o,"P",{"data-svelte-h":!0}),f(r)!=="svelte-5074v9"&&(r.innerHTML=m)},m(o,U){n(o,a,U),n(o,p,U),n(o,r,U)},p:v,d(o){o&&(l(a),l(p),l(r))}}}function Fa(T){let a,w='Memory-efficient attention optimizes for memory usage <em>and</em> <a href="./fp16#scaled-dot-product-attention">inference speed</a>!';return{c(){a=c("p"),a.innerHTML=w},l(p){a=d(p,"P",{"data-svelte-h":!0}),f(a)!=="svelte-fs3wzo"&&(a.innerHTML=w)},m(p,r){n(p,a,r)},p:v,d(p){p&&l(a)}}}function Aa(T){let a,w,p,r,m,o,U,I='Modern diffusion models like <a href="../api/pipelines/flux">Flux</a> and <a href="../api/pipelines/wan">Wan</a> have billions of parameters that take up a lot of memory on your hardware for inference. This is challenging because common GPUs often don’t have sufficient memory. To overcome the memory limitations, you can use more than one GPU (if available), offload some of the pipeline components to the CPU, and more.',g,x,j="This guide will show you how to reduce your memory usage.",_,W,Te,E,Ue,S,Mn='If you have access to more than one GPU, there a few options for efficiently loading and distributing a large model across your hardware. These features are supported by the <a href="https://huggingface.co/docs/accelerate/index" rel="nofollow">Accelerate</a> library, so make sure it is installed first.',Ze,F,ge,A,_e,L,un="Loading large checkpoints in several shards in useful because the shards are loaded one at a time. This keeps memory usage low, only requiring enough memory for the model size and the largest shard size. We recommend sharding when the fp32 checkpoint is greater than 5GB. The default shard size is 5GB.",Ie,q,yn='Shard a checkpoint in <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.save_pretrained">save_pretrained()</a> with the <code>max_shard_size</code> parameter.',xe,D,Ge,P,Jn="Now you can use the sharded checkpoint, instead of the regular checkpoint, to save memory.",ve,K,Ce,O,We,B,Be,ss,hn="The <code>device_map</code> parameter controls how the model components in a pipeline or the layers in an individual model are distributed across devices.",ke,k,Ve,es,bn="When designing your own <code>device_map</code>, it should be a dictionary of a model’s specific module name or layer and a device identifier (an integer for GPUs, <code>cpu</code> for CPUs, and <code>disk</code> for disk).",$e,ls,wn="Call <code>hf_device_map</code> on a model to see how model layers are distributed and then design your own.",Xe,ns,Ne,as,jn="For example, the <code>device_map</code> below places <code>single_transformer_blocks.10</code> through <code>single_transformer_blocks.20</code> on a second GPU (<code>1</code>).",Ye,ts,ze,is,Tn="Pass a dictionary mapping maximum memory usage to each device to enforce a limit. If a device is not in <code>max_memory</code>, it is ignored and pipeline components won’t be distributed to it.",Re,ps,Qe,os,Un="Diffusers uses the maxmium memory of all devices by default, but if they don’t fit on the GPUs, then you’ll need to use a single GPU and offload to the CPU with the methods below.",He,rs,Zn='<li><a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> only works on a single GPU but a very large model may not fit on it</li> <li><a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_sequential_cpu_offload">enable_sequential_cpu_offload()</a> may work but it is extremely slow and also limited to a single GPU</li>',Ee,ms,gn='Use the <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.reset_device_map">reset_device_map()</a> method to reset the <code>device_map</code>. This is necessary if you want to use methods like <code>.to()</code>, <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_sequential_cpu_offload">enable_sequential_cpu_offload()</a>, and <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> on a pipeline that was device-mapped.',Se,cs,Fe,ds,Ae,fs,_n="VAE slicing saves memory by splitting large batches of inputs into a single batch of data and separately processing them. This method works best when generating more than one image at a time.",Le,Ms,In="For example, if you’re generating 4 images at once, decoding would increase peak activation memory by 4x. VAE slicing reduces this by only decoding 1 image at a time instead of all 4 images at once.",qe,us,xn='Call <a href="/docs/diffusers/main/en/api/pipelines/controlnet#diffusers.StableDiffusionControlNetPipeline.enable_vae_slicing">enable_vae_slicing()</a> to enable sliced VAE. You can expect a small increase in performance when decoding multi-image batches and no performance impact for single-image batches.',De,ys,Pe,V,Ke,Js,Oe,hs,Gn="VAE tiling saves memory by dividing an image into smaller overlapping tiles instead of processing the entire image at once. This also reduces peak memory usage because the GPU is only processing a tile at a time.",sl,bs,vn='Call <a href="/docs/diffusers/main/en/api/pipelines/latent_consistency_models#diffusers.LatentConsistencyModelPipeline.enable_vae_tiling">enable_vae_tiling()</a> to enable VAE tiling. The generated image may have some tone variation from tile-to-tile because they’re decoded separately, but there shouldn’t be any obvious seams between the tiles. Tiling is disabled for resolutions lower than a pre-specified (but configurable) limit. For example, this limit is 512x512 for the VAE in <a href="/docs/diffusers/main/en/api/pipelines/stable_diffusion/text2img#diffusers.StableDiffusionPipeline">StableDiffusionPipeline</a>.',el,ws,ll,js,Cn='<p>[!WARNING][AutoencoderKLWan](/docs/diffusers/main/en/api/models/autoencoder_kl_wan#diffusers.AutoencoderKLWan) and <a href="/docs/diffusers/main/en/api/models/asymmetricautoencoderkl#diffusers.AsymmetricAutoencoderKL">AsymmetricAutoencoderKL</a> don’t support tiling.</p>',nl,Ts,al,Us,Wn="Offloading strategies move not currently active layers or models to the CPU to avoid increasing GPU memory. These strategies can be combined with quantization and torch.compile to balance inference speed and memory usage.",tl,Zs,Bn='Refer to the <a href="./speed-memory-optims">Compile and offloading quantized models</a> guide for more details.',il,gs,pl,_s,kn="CPU offloading selectively moves weights from the GPU to the CPU. When a component is required, it is transferred to the GPU and when it isn’t required, it is moved to the CPU. This method works on submodules rather than whole models. It saves memory by avoiding storing the entire model on the GPU.",ol,Is,Vn="CPU offloading dramatically reduces memory usage, but it is also <strong>extremely slow</strong> because submodules are passed back and forth multiple times between devices. It can often be impractical due to how slow it is.",rl,$,ml,xs,$n='Call <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_sequential_cpu_offload">enable_sequential_cpu_offload()</a> to enable it on a pipeline.',cl,Gs,dl,vs,fl,Cs,Xn='Model offloading moves entire models to the GPU instead of selectively moving <em>some</em> layers or model components. One of the main pipeline models, usually the text encoder, UNet, and VAE, is placed on the GPU while the other components are held on the CPU. Components like the UNet that run multiple times stays on the GPU until its completely finished and no longer needed. This eliminates the communication overhead of <a href="#cpu-offloading">CPU offloading</a> and makes model offloading a faster alternative. The tradeoff is memory savings won’t be as large.',Ml,X,ul,Ws,Nn='Call <a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> to enable it on a pipeline.',yl,Bs,Jl,ks,Yn='<a href="/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline.enable_model_cpu_offload">enable_model_cpu_offload()</a> also helps when you’re using the <a href="/docs/diffusers/main/en/api/pipelines/stable_diffusion/stable_diffusion_xl#diffusers.StableDiffusionXLPipeline.encode_prompt">encode_prompt()</a> method on its own to generate the text encoders hidden state.',hl,Vs,bl,$s,zn='Group offloading moves groups of internal layers (<a href="https://pytorch.org/docs/stable/generated/torch.nn.ModuleList.html" rel="nofollow">torch.nn.ModuleList</a> or <a href="https://pytorch.org/docs/stable/generated/torch.nn.Sequential.html" rel="nofollow">torch.nn.Sequential</a>) to the CPU. It uses less memory than <a href="#model-offloading">model offloading</a> and it is faster than <a href="#cpu-offloading">CPU offloading</a> because it reduces communication overhead.',wl,N,jl,Xs,Rn="Enable group offloading by configuring the <code>offload_type</code> parameter to <code>block_level</code> or <code>leaf_level</code>.",Tl,Ns,Qn='<li><code>block_level</code> offloads groups of layers based on the <code>num_blocks_per_group</code> parameter. For example, if <code>num_blocks_per_group=2</code> on a model with 40 layers, 2 layers are onloaded and offloaded at a time (20 total onloads/offloads). This drastically reduces memory requirements.</li> <li><code>leaf_level</code> offloads individual layers at the lowest level and is equivalent to <a href="#cpu-offloading">CPU offloading</a>. But it can be made faster if you use streams without giving up inference speed.</li>',Ul,Ys,Hn="Group offloading is supported for entire pipelines or individual models. Applying group offloading to the entire pipeline is the easiest option while selectively applying it to individual models gives users more flexibility to use different offloading techniques for different models.",Zl,Y,gl,zs,_l,Rs,En='The <code>use_stream</code> parameter can be activated for CUDA devices that support asynchronous data transfer streams to reduce overall execution time compared to <a href="#cpu-offloading">CPU offloading</a>. It overlaps data transfer and computation by using layer prefetching. The next layer to be executed is loaded onto the GPU while the current layer is still being executed. It can increase CPU memory significantly so ensure you have 2x the amount of memory as the model size.',Il,Qs,Sn='Set <code>record_stream=True</code> for more of a speedup at the cost of slightly increased memory usage. Refer to the <a href="https://pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html" rel="nofollow">torch.Tensor.record_stream</a> docs to learn more.',xl,z,Gl,Hs,Fn="If you’re using <code>block_level</code> group offloading with <code>use_stream</code> enabled, the <code>num_blocks_per_group</code> parameter should be set to <code>1</code>, otherwise a warning will be raised.",vl,Es,Cl,Ss,An="The <code>low_cpu_mem_usage</code> parameter can be set to <code>True</code> to reduce CPU memory usage when using streams during group offloading. It is best for <code>leaf_level</code> offloading and when CPU memory is bottlenecked. Memory is saved by creating pinned tensors on the fly instead of pre-pinning them. However, this may increase overall execution time.",Wl,Fs,Bl,As,Ln="Group offloading can consume significant system memory depending on the model size. On systems with limited memory, try group offloading onto the disk as a secondary memory.",kl,Ls,qn='Set the <code>offload_to_disk_path</code> argument in either <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.enable_group_offload">enable_group_offload()</a> or <a href="/docs/diffusers/main/en/api/utilities#diffusers.hooks.apply_group_offloading">apply_group_offloading()</a> to offload the model to the disk.',Vl,qs,$l,Ds,Dn='Refer to these <a href="https://github.com/huggingface/diffusers/pull/11682#issue-3129365363" rel="nofollow">two</a> <a href="https://github.com/huggingface/diffusers/pull/11682#issuecomment-2955715126" rel="nofollow">tables</a> to compare the speed and memory trade-offs.',Xl,Ps,Nl,R,Yl,Ks,Pn="Layerwise casting stores weights in a smaller data format (for example, <code>torch.float8_e4m3fn</code> and <code>torch.float8_e5m2</code>) to use less memory and upcasts those weights to a higher precision like <code>torch.float16</code> or <code>torch.bfloat16</code> for computation. Certain layers (normalization and modulation related weights) are skipped because storing them in fp8 can degrade generation quality.",zl,Q,Rl,Os,Kn='Call <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.enable_layerwise_casting">enable_layerwise_casting()</a> to set the storage and computation datatypes.',Ql,se,Hl,ee,On='The <a href="/docs/diffusers/main/en/api/utilities#diffusers.hooks.apply_layerwise_casting">apply_layerwise_casting()</a> method can also be used if you need more control and flexibility. It can be partially applied to model layers by calling it on specific internal modules. Use the <code>skip_modules_pattern</code> or <code>skip_modules_classes</code> parameters to specify modules to avoid, such as the normalization and modulation layers.',El,le,Sl,ne,Fl,ae,sa='<a href="https://pytorch.org/tutorials/intermediate/memory_format_tutorial.html" rel="nofollow">torch.channels_last</a> flips how tensors are stored from <code>(batch size, channels, height, width)</code> to <code>(batch size, heigh, width, channels)</code>. This aligns the tensors with how the hardware sequentially accesses the tensors stored in memory and avoids skipping around in memory to access the pixel values.',Al,te,ea="Not all operators currently support the channels-last format and may result in worst performance, but it is still worth trying.",Ll,ie,ql,pe,Dl,oe,la='<a href="https://pytorch.org/docs/stable/generated/torch.jit.trace.html" rel="nofollow">torch.jit.trace</a> records the operations a model performs on a sample input and creates a new, optimized representation of the model based on the recorded execution path. During tracing, the model is optimized to reduce overhead from Python and dynamic control flows and operations are fused together for more efficiency. The returned executable or <a href="https://pytorch.org/docs/stable/generated/torch.jit.ScriptFunction.html" rel="nofollow">ScriptFunction</a> can be compiled.',Pl,re,Kl,me,na="Replace the pipeline’s UNet with the traced version.",Ol,ce,sn,de,en,H,ln,fe,aa="The Transformers attention mechanism is memory-intensive, especially for long sequences, so you can try using different and more memory-efficient attention types.",nn,Me,ta='By default, if PyTorch &gt;= 2.0 is installed, <a href="https://pytorch.org/docs/stable/generated/torch.nn.functional.scaled_dot_product_attention.html" rel="nofollow">scaled dot-product attention (SDPA)</a> is used. You don’t need to make any additional changes to your code.',an,ue,ia='SDPA supports <a href="https://github.com/Dao-AILab/flash-attention" rel="nofollow">FlashAttention</a> and <a href="https://github.com/facebookresearch/xformers" rel="nofollow">xFormers</a> as well as a native C++ PyTorch implementation. It automatically selects the most optimal implementation based on your input.',tn,ye,pa='You can explicitly use xFormers with the <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.enable_xformers_memory_efficient_attention">enable_xformers_memory_efficient_attention()</a> method.',pn,Je,on,he,oa='Call <a href="/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin.disable_xformers_memory_efficient_attention">disable_xformers_memory_efficient_attention()</a> to disable it.',rn,be,mn,we,cn,je,dn;return m=new G({props:{title:"Reduce memory usage",local:"reduce-memory-usage",headingTag:"h1"}}),W=new C({props:{warning:!1,$$slots:{default:[Ca]},$$scope:{ctx:T}}}),E=new G({props:{title:"Multiple GPUs",local:"multiple-gpus",headingTag:"h2"}}),F=new Z({props:{code:"cGlwJTIwaW5zdGFsbCUyMC1VJTIwYWNjZWxlcmF0ZQ==",highlighted:"pip install -U accelerate",wrap:!1}}),A=new G({props:{title:"Sharded checkpoints",local:"sharded-checkpoints",headingTag:"h3"}}),D=new Z({props:{code:"ZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMEF1dG9Nb2RlbCUwQSUwQXVuZXQlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJpbGl0eWFpJTJGc3RhYmxlLWRpZmZ1c2lvbi14bC1iYXNlLTEuMCUyMiUyQyUyMHN1YmZvbGRlciUzRCUyMnVuZXQlMjIlMEEpJTBBdW5ldC5zYXZlX3ByZXRyYWluZWQoJTIyc2R4bC11bmV0LXNoYXJkZWQlMjIlMkMlMjBtYXhfc2hhcmRfc2l6ZSUzRCUyMjVHQiUyMik=",highlighted:`<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel

unet = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>, subfolder=<span class="hljs-string">&quot;unet&quot;</span>
)
unet.save_pretrained(<span class="hljs-string">&quot;sdxl-unet-sharded&quot;</span>, max_shard_size=<span class="hljs-string">&quot;5GB&quot;</span>)`,wrap:!1}}),K=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwU3RhYmxlRGlmZnVzaW9uWExQaXBlbGluZSUwQSUwQXVuZXQlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnVzZXJuYW1lJTJGc2R4bC11bmV0LXNoYXJkZWQlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYlMEEpJTBBcGlwZWxpbmUlMjAlM0QlMjBTdGFibGVEaWZmdXNpb25YTFBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJzdGFiaWxpdHlhaSUyRnN0YWJsZS1kaWZmdXNpb24teGwtYmFzZS0xLjAlMjIlMkMlMEElMjAlMjAlMjAlMjB1bmV0JTNEdW5ldCUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUwQSkudG8oJTIyY3VkYSUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, StableDiffusionXLPipeline

unet = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;username/sdxl-unet-sharded&quot;</span>, torch_dtype=torch.float16
)
pipeline = StableDiffusionXLPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
    unet=unet,
    torch_dtype=torch.float16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)`,wrap:!1}}),O=new G({props:{title:"Device placement",local:"device-placement",headingTag:"h3"}}),B=new C({props:{warning:!0,$$slots:{default:[Wa]},$$scope:{ctx:T}}}),k=new Ua({props:{id:"device-map",options:["pipeline level","model level"],$$slots:{default:[Va]},$$scope:{ctx:T}}}),ns=new Z({props:{code:"cHJpbnQodHJhbnNmb3JtZXIuaGZfZGV2aWNlX21hcCklMEElN0IncG9zX2VtYmVkJyUzQSUyMDAlMkMlMjAndGltZV90ZXh0X2VtYmVkJyUzQSUyMDAlMkMlMjAnY29udGV4dF9lbWJlZGRlciclM0ElMjAwJTJDJTIwJ3hfZW1iZWRkZXInJTNBJTIwMCUyQyUyMCd0cmFuc2Zvcm1lcl9ibG9ja3MnJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjAnJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjEnJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjInJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjMnJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjQnJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjUnJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjYnJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjcnJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjgnJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjknJTNBJTIwMCUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjEwJyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMTEnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4xMiclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjEzJyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMTQnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4xNSclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjE2JyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMTcnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4xOCclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjE5JyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMjAnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4yMSclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjIyJyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMjMnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4yNCclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjI1JyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMjYnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4yNyclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjI4JyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMjknJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4zMCclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjMxJyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMzInJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4zMyclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjM0JyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMzUnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4zNiclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjM3JyUzQSUyMCdjcHUnJTJDJTIwJ25vcm1fb3V0JyUzQSUyMCdjcHUnJTJDJTIwJ3Byb2pfb3V0JyUzQSUyMCdjcHUnJTdE",highlighted:`<span class="hljs-built_in">print</span>(transformer.hf_device_map)
{<span class="hljs-string">&#x27;pos_embed&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;time_text_embed&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;context_embedder&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;x_embedder&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;transformer_blocks&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.0&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.1&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.2&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.3&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.4&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.5&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.6&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.7&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.8&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.9&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.10&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.11&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.12&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.13&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.14&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.15&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.16&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.17&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.18&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.19&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.20&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.21&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.22&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.23&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.24&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.25&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.26&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.27&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.28&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.29&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.30&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.31&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.32&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.33&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.34&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.35&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.36&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.37&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;norm_out&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;proj_out&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>}`,wrap:!1}}),ts=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTBBJTBBZGV2aWNlX21hcCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCdwb3NfZW1iZWQnJTNBJTIwMCUyQyUyMCd0aW1lX3RleHRfZW1iZWQnJTNBJTIwMCUyQyUyMCdjb250ZXh0X2VtYmVkZGVyJyUzQSUyMDAlMkMlMjAneF9lbWJlZGRlciclM0ElMjAwJTJDJTIwJ3RyYW5zZm9ybWVyX2Jsb2NrcyclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMCclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMSclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMiclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMyclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuNCclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuNSclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuNiclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuNyclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuOCclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuOSclM0ElMjAwJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMTAnJTNBJTIwMSUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjExJyUzQSUyMDElMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4xMiclM0ElMjAxJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMTMnJTNBJTIwMSUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjE0JyUzQSUyMDElMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4xNSclM0ElMjAxJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMTYnJTNBJTIwMSUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjE3JyUzQSUyMDElMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4xOCclM0ElMjAxJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMTknJTNBJTIwMSUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjIwJyUzQSUyMDElMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4yMSclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjIyJyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMjMnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4yNCclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjI1JyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMjYnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4yNyclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjI4JyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMjknJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4zMCclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjMxJyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMzInJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4zMyclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjM0JyUzQSUyMCdjcHUnJTJDJTIwJ3NpbmdsZV90cmFuc2Zvcm1lcl9ibG9ja3MuMzUnJTNBJTIwJ2NwdSclMkMlMjAnc2luZ2xlX3RyYW5zZm9ybWVyX2Jsb2Nrcy4zNiclM0ElMjAnY3B1JyUyQyUyMCdzaW5nbGVfdHJhbnNmb3JtZXJfYmxvY2tzLjM3JyUzQSUyMCdjcHUnJTJDJTIwJ25vcm1fb3V0JyUzQSUyMCdjcHUnJTJDJTIwJ3Byb2pfb3V0JyUzQSUyMCdjcHUnJTBBJTdEJTBBJTBBdHJhbnNmb3JtZXIlMjAlM0QlMjBBdXRvTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmJsYWNrLWZvcmVzdC1sYWJzJTJGRkxVWC4xLWRldiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHN1YmZvbGRlciUzRCUyMnRyYW5zZm9ybWVyJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRGRldmljZV9tYXAlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel

device_map = {
    <span class="hljs-string">&#x27;pos_embed&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;time_text_embed&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;context_embedder&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;x_embedder&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;transformer_blocks&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.0&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.1&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.2&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.3&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.4&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.5&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.6&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.7&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.8&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.9&#x27;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&#x27;single_transformer_blocks.10&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.11&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.12&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.13&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.14&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.15&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.16&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.17&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.18&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.19&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.20&#x27;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&#x27;single_transformer_blocks.21&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.22&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.23&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.24&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.25&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.26&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.27&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.28&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.29&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.30&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.31&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.32&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.33&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.34&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.35&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.36&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;single_transformer_blocks.37&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;norm_out&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>, <span class="hljs-string">&#x27;proj_out&#x27;</span>: <span class="hljs-string">&#x27;cpu&#x27;</span>
}

transformer = AutoModel.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, 
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    device_map=device_map,
    torch_dtype=torch.bfloat16
)`,wrap:!1}}),ps=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwU3RhYmxlRGlmZnVzaW9uWExQaXBlbGluZSUwQSUwQW1heF9tZW1vcnklMjAlM0QlMjAlN0IwJTNBJTIyMUdCJTIyJTJDJTIwMSUzQSUyMjFHQiUyMiU3RCUwQXBpcGVsaW5lJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uWExQaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyc3RhYmlsaXR5YWklMkZzdGFibGUtZGlmZnVzaW9uLXhsLWJhc2UtMS4wJTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmJhbGFuY2VkJTIyJTJDJTBBJTIwJTIwJTIwJTIwbWF4X21lbW9yeSUzRG1heF9tZW1vcnklMEEp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, StableDiffusionXLPipeline

max_memory = {<span class="hljs-number">0</span>:<span class="hljs-string">&quot;1GB&quot;</span>, <span class="hljs-number">1</span>:<span class="hljs-string">&quot;1GB&quot;</span>}
pipeline = StableDiffusionXLPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-string">&quot;balanced&quot;</span>,
    max_memory=max_memory
)`,wrap:!1}}),cs=new Z({props:{code:"cGlwZWxpbmUucmVzZXRfZGV2aWNlX21hcCgp",highlighted:"pipeline.reset_device_map()",wrap:!1}}),ds=new G({props:{title:"VAE slicing",local:"vae-slicing",headingTag:"h2"}}),ys=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsJTJDJTIwU3RhYmxlRGlmZnVzaW9uWExQaXBlbGluZSUwQSUwQXBpcGVsaW5lJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uWExQaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyc3RhYmlsaXR5YWklMkZzdGFibGUtZGlmZnVzaW9uLXhsLWJhc2UtMS4wJTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKS50byglMjJjdWRhJTIyKSUwQXBpcGVsaW5lLmVuYWJsZV92YWVfc2xpY2luZygpJTBBcGlwZWxpbmUoJTVCJTIyQW4lMjBhc3Ryb25hdXQlMjByaWRpbmclMjBhJTIwaG9yc2UlMjBvbiUyME1hcnMlMjIlNUQqMzIpLmltYWdlcyU1QjAlNUQlMEFwcmludChmJTIyTWF4JTIwbWVtb3J5JTIwcmVzZXJ2ZWQlM0ElMjAlN0J0b3JjaC5jdWRhLm1heF9tZW1vcnlfYWxsb2NhdGVkKCklMjAlMkYlMjAxMDI0KiozJTNBLjJmJTdEJTIwR0IlMjIp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoModel, StableDiffusionXLPipeline

pipeline = StableDiffusionXLPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
    torch_dtype=torch.float16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
pipeline.enable_vae_slicing()
pipeline([<span class="hljs-string">&quot;An astronaut riding a horse on Mars&quot;</span>]*<span class="hljs-number">32</span>).images[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Max memory reserved: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB&quot;</span>)`,wrap:!1}}),V=new C({props:{warning:!0,$$slots:{default:[$a]},$$scope:{ctx:T}}}),Js=new G({props:{title:"VAE tiling",local:"vae-tiling",headingTag:"h2"}}),ws=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQXV0b1BpcGVsaW5lRm9ySW1hZ2UySW1hZ2UlMEFmcm9tJTIwZGlmZnVzZXJzLnV0aWxzJTIwaW1wb3J0JTIwbG9hZF9pbWFnZSUwQSUwQXBpcGVsaW5lJTIwJTNEJTIwQXV0b1BpcGVsaW5lRm9ySW1hZ2UySW1hZ2UuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJpbGl0eWFpJTJGc3RhYmxlLWRpZmZ1c2lvbi14bC1iYXNlLTEuMCUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUwQSkudG8oJTIyY3VkYSUyMiklMEFwaXBlbGluZS5lbmFibGVfdmFlX3RpbGluZygpJTBBJTBBaW5pdF9pbWFnZSUyMCUzRCUyMGxvYWRfaW1hZ2UoJTIyaHR0cHMlM0ElMkYlMkZodWdnaW5nZmFjZS5jbyUyRmRhdGFzZXRzJTJGaHVnZ2luZ2ZhY2UlMkZkb2N1bWVudGF0aW9uLWltYWdlcyUyRnJlc29sdmUlMkZtYWluJTJGZGlmZnVzZXJzJTJGaW1nMmltZy1zZHhsLWluaXQucG5nJTIyKSUwQXByb21wdCUyMCUzRCUyMCUyMkFzdHJvbmF1dCUyMGluJTIwYSUyMGp1bmdsZSUyQyUyMGNvbGQlMjBjb2xvciUyMHBhbGV0dGUlMkMlMjBtdXRlZCUyMGNvbG9ycyUyQyUyMGRldGFpbGVkJTJDJTIwOGslMjIlMEFwaXBlbGluZShwcm9tcHQlMkMlMjBpbWFnZSUzRGluaXRfaW1hZ2UlMkMlMjBzdHJlbmd0aCUzRDAuNSkuaW1hZ2VzJTVCMCU1RCUwQXByaW50KGYlMjJNYXglMjBtZW1vcnklMjByZXNlcnZlZCUzQSUyMCU3QnRvcmNoLmN1ZGEubWF4X21lbW9yeV9hbGxvY2F0ZWQoKSUyMCUyRiUyMDEwMjQqKjMlM0EuMmYlN0QlMjBHQiUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AutoPipelineForImage2Image
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image

pipeline = AutoPipelineForImage2Image.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>, torch_dtype=torch.float16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
pipeline.enable_vae_tiling()

init_image = load_image(<span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/img2img-sdxl-init.png&quot;</span>)
prompt = <span class="hljs-string">&quot;Astronaut in a jungle, cold color palette, muted colors, detailed, 8k&quot;</span>
pipeline(prompt, image=init_image, strength=<span class="hljs-number">0.5</span>).images[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Max memory reserved: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB&quot;</span>)`,wrap:!1}}),Ts=new G({props:{title:"Offloading",local:"offloading",headingTag:"h2"}}),gs=new G({props:{title:"CPU offloading",local:"cpu-offloading",headingTag:"h3"}}),$=new C({props:{warning:!0,$$slots:{default:[Xa]},$$scope:{ctx:T}}}),Gs=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlbGluZSUyMCUzRCUyMERpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1zY2huZWxsJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUwQSklMEFwaXBlbGluZS5lbmFibGVfc2VxdWVudGlhbF9jcHVfb2ZmbG9hZCgpJTBBJTBBcGlwZWxpbmUoJTBBJTIwJTIwJTIwJTIwcHJvbXB0JTNEJTIyQW4lMjBhc3Ryb25hdXQlMjByaWRpbmclMjBhJTIwaG9yc2UlMjBvbiUyME1hcnMlMjIlMkMlMEElMjAlMjAlMjAlMjBndWlkYW5jZV9zY2FsZSUzRDAuJTJDJTBBJTIwJTIwJTIwJTIwaGVpZ2h0JTNENzY4JTJDJTBBJTIwJTIwJTIwJTIwd2lkdGglM0QxMzYwJTJDJTBBJTIwJTIwJTIwJTIwbnVtX2luZmVyZW5jZV9zdGVwcyUzRDQlMkMlMEElMjAlMjAlMjAlMjBtYXhfc2VxdWVuY2VfbGVuZ3RoJTNEMjU2JTJDJTBBKS5pbWFnZXMlNUIwJTVEJTBBcHJpbnQoZiUyMk1heCUyMG1lbW9yeSUyMHJlc2VydmVkJTNBJTIwJTdCdG9yY2guY3VkYS5tYXhfbWVtb3J5X2FsbG9jYXRlZCgpJTIwJTJGJTIwMTAyNCoqMyUzQS4yZiU3RCUyMEdCJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline

pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-schnell&quot;</span>, torch_dtype=torch.bfloat16
)
pipeline.enable_sequential_cpu_offload()

pipeline(
    prompt=<span class="hljs-string">&quot;An astronaut riding a horse on Mars&quot;</span>,
    guidance_scale=<span class="hljs-number">0.</span>,
    height=<span class="hljs-number">768</span>,
    width=<span class="hljs-number">1360</span>,
    num_inference_steps=<span class="hljs-number">4</span>,
    max_sequence_length=<span class="hljs-number">256</span>,
).images[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Max memory reserved: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB&quot;</span>)`,wrap:!1}}),vs=new G({props:{title:"Model offloading",local:"model-offloading",headingTag:"h3"}}),X=new C({props:{warning:!0,$$slots:{default:[Na]},$$scope:{ctx:T}}}),Bs=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFwaXBlbGluZSUyMCUzRCUyMERpZmZ1c2lvblBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJibGFjay1mb3Jlc3QtbGFicyUyRkZMVVguMS1zY2huZWxsJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUwQSklMEFwaXBlbGluZS5lbmFibGVfbW9kZWxfY3B1X29mZmxvYWQoKSUwQSUwQXBpcGVsaW5lKCUwQSUyMCUyMCUyMCUyMHByb21wdCUzRCUyMkFuJTIwYXN0cm9uYXV0JTIwcmlkaW5nJTIwYSUyMGhvcnNlJTIwb24lMjBNYXJzJTIyJTJDJTBBJTIwJTIwJTIwJTIwZ3VpZGFuY2Vfc2NhbGUlM0QwLiUyQyUwQSUyMCUyMCUyMCUyMGhlaWdodCUzRDc2OCUyQyUwQSUyMCUyMCUyMCUyMHdpZHRoJTNEMTM2MCUyQyUwQSUyMCUyMCUyMCUyMG51bV9pbmZlcmVuY2Vfc3RlcHMlM0Q0JTJDJTBBJTIwJTIwJTIwJTIwbWF4X3NlcXVlbmNlX2xlbmd0aCUzRDI1NiUyQyUwQSkuaW1hZ2VzJTVCMCU1RCUwQXByaW50KGYlMjJNYXglMjBtZW1vcnklMjByZXNlcnZlZCUzQSUyMCU3QnRvcmNoLmN1ZGEubWF4X21lbW9yeV9hbGxvY2F0ZWQoKSUyMCUyRiUyMDEwMjQqKjMlM0EuMmYlN0QlMjBHQiUyMik=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline

pipeline = DiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-schnell&quot;</span>, torch_dtype=torch.bfloat16
)
pipeline.enable_model_cpu_offload()

pipeline(
    prompt=<span class="hljs-string">&quot;An astronaut riding a horse on Mars&quot;</span>,
    guidance_scale=<span class="hljs-number">0.</span>,
    height=<span class="hljs-number">768</span>,
    width=<span class="hljs-number">1360</span>,
    num_inference_steps=<span class="hljs-number">4</span>,
    max_sequence_length=<span class="hljs-number">256</span>,
).images[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Max memory reserved: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB&quot;</span>)`,wrap:!1}}),Vs=new G({props:{title:"Group offloading",local:"group-offloading",headingTag:"h3"}}),N=new C({props:{warning:!0,$$slots:{default:[Ya]},$$scope:{ctx:T}}}),Y=new Ua({props:{id:"group-offloading",options:["pipeline","model"],$$slots:{default:[Qa]},$$scope:{ctx:T}}}),zs=new G({props:{title:"CUDA stream",local:"cuda-stream",headingTag:"h4"}}),z=new C({props:{warning:!1,$$slots:{default:[Ha]},$$scope:{ctx:T}}}),Es=new Z({props:{code:"cGlwZWxpbmUudHJhbnNmb3JtZXIuZW5hYmxlX2dyb3VwX29mZmxvYWQob25sb2FkX2RldmljZSUzRG9ubG9hZF9kZXZpY2UlMkMlMjBvZmZsb2FkX2RldmljZSUzRG9mZmxvYWRfZGV2aWNlJTJDJTIwb2ZmbG9hZF90eXBlJTNEJTIybGVhZl9sZXZlbCUyMiUyQyUyMHVzZV9zdHJlYW0lM0RUcnVlJTJDJTIwcmVjb3JkX3N0cmVhbSUzRFRydWUp",highlighted:'pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>, use_stream=<span class="hljs-literal">True</span>, record_stream=<span class="hljs-literal">True</span>)',wrap:!1}}),Fs=new G({props:{title:"Offloading to disk",local:"offloading-to-disk",headingTag:"h4"}}),qs=new Z({props:{code:"cGlwZWxpbmUudHJhbnNmb3JtZXIuZW5hYmxlX2dyb3VwX29mZmxvYWQob25sb2FkX2RldmljZSUzRG9ubG9hZF9kZXZpY2UlMkMlMjBvZmZsb2FkX2RldmljZSUzRG9mZmxvYWRfZGV2aWNlJTJDJTIwb2ZmbG9hZF90eXBlJTNEJTIybGVhZl9sZXZlbCUyMiUyQyUyMG9mZmxvYWRfdG9fZGlza19wYXRoJTNEJTIycGF0aCUyRnRvJTJGZGlzayUyMiklMEElMEFhcHBseV9ncm91cF9vZmZsb2FkaW5nKHBpcGVsaW5lLnRleHRfZW5jb2RlciUyQyUyMG9ubG9hZF9kZXZpY2UlM0RvbmxvYWRfZGV2aWNlJTJDJTIwb2ZmbG9hZF90eXBlJTNEJTIyYmxvY2tfbGV2ZWwlMjIlMkMlMjBudW1fYmxvY2tzX3Blcl9ncm91cCUzRDIlMkMlMjBvZmZsb2FkX3RvX2Rpc2tfcGF0aCUzRCUyMnBhdGglMkZ0byUyRmRpc2slMjIp",highlighted:`pipeline.transformer.enable_group_offload(onload_device=onload_device, offload_device=offload_device, offload_type=<span class="hljs-string">&quot;leaf_level&quot;</span>, offload_to_disk_path=<span class="hljs-string">&quot;path/to/disk&quot;</span>)

apply_group_offloading(pipeline.text_encoder, onload_device=onload_device, offload_type=<span class="hljs-string">&quot;block_level&quot;</span>, num_blocks_per_group=<span class="hljs-number">2</span>, offload_to_disk_path=<span class="hljs-string">&quot;path/to/disk&quot;</span>)`,wrap:!1}}),Ps=new G({props:{title:"Layerwise casting",local:"layerwise-casting",headingTag:"h2"}}),R=new C({props:{warning:!1,$$slots:{default:[Ea]},$$scope:{ctx:T}}}),Q=new C({props:{warning:!0,$$slots:{default:[Sa]},$$scope:{ctx:T}}}),se=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQ29nVmlkZW9YUGlwZWxpbmUlMkMlMjBDb2dWaWRlb1hUcmFuc2Zvcm1lcjNETW9kZWwlMEFmcm9tJTIwZGlmZnVzZXJzLnV0aWxzJTIwaW1wb3J0JTIwZXhwb3J0X3RvX3ZpZGVvJTBBJTBBdHJhbnNmb3JtZXIlMjAlM0QlMjBDb2dWaWRlb1hUcmFuc2Zvcm1lcjNETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMlRIVURNJTJGQ29nVmlkZW9YLTViJTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTBBKSUwQXRyYW5zZm9ybWVyLmVuYWJsZV9sYXllcndpc2VfY2FzdGluZyhzdG9yYWdlX2R0eXBlJTNEdG9yY2guZmxvYXQ4X2U0bTNmbiUyQyUyMGNvbXB1dGVfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiklMEElMEFwaXBlbGluZSUyMCUzRCUyMENvZ1ZpZGVvWFBpcGVsaW5lLmZyb21fcHJldHJhaW5lZCglMjJUSFVETSUyRkNvZ1ZpZGVvWC01YiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRyYW5zZm9ybWVyJTNEdHJhbnNmb3JtZXIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTBBKS50byglMjJjdWRhJTIyKSUwQXByb21wdCUyMCUzRCUyMCglMEElMjAlMjAlMjAlMjAlMjJBJTIwcGFuZGElMkMlMjBkcmVzc2VkJTIwaW4lMjBhJTIwc21hbGwlMkMlMjByZWQlMjBqYWNrZXQlMjBhbmQlMjBhJTIwdGlueSUyMGhhdCUyQyUyMHNpdHMlMjBvbiUyMGElMjB3b29kZW4lMjBzdG9vbCUyMGluJTIwYSUyMHNlcmVuZSUyMGJhbWJvbyUyMGZvcmVzdC4lMjAlMjIlMEElMjAlMjAlMjAlMjAlMjJUaGUlMjBwYW5kYSdzJTIwZmx1ZmZ5JTIwcGF3cyUyMHN0cnVtJTIwYSUyMG1pbmlhdHVyZSUyMGFjb3VzdGljJTIwZ3VpdGFyJTJDJTIwcHJvZHVjaW5nJTIwc29mdCUyQyUyMG1lbG9kaWMlMjB0dW5lcy4lMjBOZWFyYnklMkMlMjBhJTIwZmV3JTIwb3RoZXIlMjAlMjIlMEElMjAlMjAlMjAlMjAlMjJwYW5kYXMlMjBnYXRoZXIlMkMlMjB3YXRjaGluZyUyMGN1cmlvdXNseSUyMGFuZCUyMHNvbWUlMjBjbGFwcGluZyUyMGluJTIwcmh5dGhtLiUyMFN1bmxpZ2h0JTIwZmlsdGVycyUyMHRocm91Z2glMjB0aGUlMjB0YWxsJTIwYmFtYm9vJTJDJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIyY2FzdGluZyUyMGElMjBnZW50bGUlMjBnbG93JTIwb24lMjB0aGUlMjBzY2VuZS4lMjBUaGUlMjBwYW5kYSdzJTIwZmFjZSUyMGlzJTIwZXhwcmVzc2l2ZSUyQyUyMHNob3dpbmclMjBjb25jZW50cmF0aW9uJTIwYW5kJTIwam95JTIwYXMlMjBpdCUyMHBsYXlzLiUyMCUyMiUwQSUyMCUyMCUyMCUyMCUyMlRoZSUyMGJhY2tncm91bmQlMjBpbmNsdWRlcyUyMGElMjBzbWFsbCUyQyUyMGZsb3dpbmclMjBzdHJlYW0lMjBhbmQlMjB2aWJyYW50JTIwZ3JlZW4lMjBmb2xpYWdlJTJDJTIwZW5oYW5jaW5nJTIwdGhlJTIwcGVhY2VmdWwlMjBhbmQlMjBtYWdpY2FsJTIwJTIyJTBBJTIwJTIwJTIwJTIwJTIyYXRtb3NwaGVyZSUyMG9mJTIwdGhpcyUyMHVuaXF1ZSUyMG11c2ljYWwlMjBwZXJmb3JtYW5jZS4lMjIlMEEpJTBBdmlkZW8lMjAlM0QlMjBwaXBlbGluZShwcm9tcHQlM0Rwcm9tcHQlMkMlMjBndWlkYW5jZV9zY2FsZSUzRDYlMkMlMjBudW1faW5mZXJlbmNlX3N0ZXBzJTNENTApLmZyYW1lcyU1QjAlNUQlMEFwcmludChmJTIyTWF4JTIwbWVtb3J5JTIwcmVzZXJ2ZWQlM0ElMjAlN0J0b3JjaC5jdWRhLm1heF9tZW1vcnlfYWxsb2NhdGVkKCklMjAlMkYlMjAxMDI0KiozJTNBLjJmJTdEJTIwR0IlMjIpJTBBZXhwb3J0X3RvX3ZpZGVvKHZpZGVvJTJDJTIwJTIyb3V0cHV0Lm1wNCUyMiUyQyUyMGZwcyUzRDgp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline, CogVideoXTransformer3DModel
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> export_to_video

transformer = CogVideoXTransformer3DModel.from_pretrained(
    <span class="hljs-string">&quot;THUDM/CogVideoX-5b&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    torch_dtype=torch.bfloat16
)
transformer.enable_layerwise_casting(storage_dtype=torch.float8_e4m3fn, compute_dtype=torch.bfloat16)

pipeline = CogVideoXPipeline.from_pretrained(<span class="hljs-string">&quot;THUDM/CogVideoX-5b&quot;</span>,
    transformer=transformer,
    torch_dtype=torch.bfloat16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
prompt = (
    <span class="hljs-string">&quot;A panda, dressed in a small, red jacket and a tiny hat, sits on a wooden stool in a serene bamboo forest. &quot;</span>
    <span class="hljs-string">&quot;The panda&#x27;s fluffy paws strum a miniature acoustic guitar, producing soft, melodic tunes. Nearby, a few other &quot;</span>
    <span class="hljs-string">&quot;pandas gather, watching curiously and some clapping in rhythm. Sunlight filters through the tall bamboo, &quot;</span>
    <span class="hljs-string">&quot;casting a gentle glow on the scene. The panda&#x27;s face is expressive, showing concentration and joy as it plays. &quot;</span>
    <span class="hljs-string">&quot;The background includes a small, flowing stream and vibrant green foliage, enhancing the peaceful and magical &quot;</span>
    <span class="hljs-string">&quot;atmosphere of this unique musical performance.&quot;</span>
)
video = pipeline(prompt=prompt, guidance_scale=<span class="hljs-number">6</span>, num_inference_steps=<span class="hljs-number">50</span>).frames[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Max memory reserved: <span class="hljs-subst">{torch.cuda.max_memory_allocated() / <span class="hljs-number">1024</span>**<span class="hljs-number">3</span>:<span class="hljs-number">.2</span>f}</span> GB&quot;</span>)
export_to_video(video, <span class="hljs-string">&quot;output.mp4&quot;</span>, fps=<span class="hljs-number">8</span>)`,wrap:!1}}),le=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQ29nVmlkZW9YVHJhbnNmb3JtZXIzRE1vZGVsJTBBZnJvbSUyMGRpZmZ1c2Vycy5ob29rcyUyMGltcG9ydCUyMGFwcGx5X2xheWVyd2lzZV9jYXN0aW5nJTBBJTBBdHJhbnNmb3JtZXIlMjAlM0QlMjBDb2dWaWRlb1hUcmFuc2Zvcm1lcjNETW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMlRIVURNJTJGQ29nVmlkZW9YLTViJTIyJTJDJTBBJTIwJTIwJTIwJTIwc3ViZm9sZGVyJTNEJTIydHJhbnNmb3JtZXIlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTBBKSUwQSUwQSUyMyUyMHNraXAlMjB0aGUlMjBub3JtYWxpemF0aW9uJTIwbGF5ZXIlMEFhcHBseV9sYXllcndpc2VfY2FzdGluZyglMEElMjAlMjAlMjAlMjB0cmFuc2Zvcm1lciUyQyUwQSUyMCUyMCUyMCUyMHN0b3JhZ2VfZHR5cGUlM0R0b3JjaC5mbG9hdDhfZTRtM2ZuJTJDJTBBJTIwJTIwJTIwJTIwY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwc2tpcF9tb2R1bGVzX2NsYXNzZXMlM0QlNUIlMjJub3JtJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwbm9uX2Jsb2NraW5nJTNEVHJ1ZSUyQyUwQSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXTransformer3DModel
<span class="hljs-keyword">from</span> diffusers.hooks <span class="hljs-keyword">import</span> apply_layerwise_casting

transformer = CogVideoXTransformer3DModel.from_pretrained(
    <span class="hljs-string">&quot;THUDM/CogVideoX-5b&quot;</span>,
    subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
    torch_dtype=torch.bfloat16
)

<span class="hljs-comment"># skip the normalization layer</span>
apply_layerwise_casting(
    transformer,
    storage_dtype=torch.float8_e4m3fn,
    compute_dtype=torch.bfloat16,
    skip_modules_classes=[<span class="hljs-string">&quot;norm&quot;</span>],
    non_blocking=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),ne=new G({props:{title:"torch.channels_last",local:"torchchannelslast",headingTag:"h2"}}),ie=new Z({props:{code:"cHJpbnQocGlwZWxpbmUudW5ldC5jb252X291dC5zdGF0ZV9kaWN0KCklNUIlMjJ3ZWlnaHQlMjIlNUQuc3RyaWRlKCkpJTIwJTIwJTIzJTIwKDI4ODAlMkMlMjA5JTJDJTIwMyUyQyUyMDEpJTBBcGlwZWxpbmUudW5ldC50byhtZW1vcnlfZm9ybWF0JTNEdG9yY2guY2hhbm5lbHNfbGFzdCklMjAlMjAlMjMlMjBpbi1wbGFjZSUyMG9wZXJhdGlvbiUwQXByaW50KCUwQSUyMCUyMCUyMCUyMHBpcGVsaW5lLnVuZXQuY29udl9vdXQuc3RhdGVfZGljdCgpJTVCJTIyd2VpZ2h0JTIyJTVELnN0cmlkZSgpJTBBKSUyMCUyMCUyMyUyMCgyODgwJTJDJTIwMSUyQyUyMDk2MCUyQyUyMDMyMCklMjBoYXZpbmclMjBhJTIwc3RyaWRlJTIwb2YlMjAxJTIwZm9yJTIwdGhlJTIwMm5kJTIwZGltZW5zaW9uJTIwcHJvdmVzJTIwdGhhdCUyMGl0JTIwd29ya3M=",highlighted:`<span class="hljs-built_in">print</span>(pipeline.unet.conv_out.state_dict()[<span class="hljs-string">&quot;weight&quot;</span>].stride())  <span class="hljs-comment"># (2880, 9, 3, 1)</span>
pipeline.unet.to(memory_format=torch.channels_last)  <span class="hljs-comment"># in-place operation</span>
<span class="hljs-built_in">print</span>(
    pipeline.unet.conv_out.state_dict()[<span class="hljs-string">&quot;weight&quot;</span>].stride()
)  <span class="hljs-comment"># (2880, 1, 960, 320) having a stride of 1 for the 2nd dimension proves that it works</span>`,wrap:!1}}),pe=new G({props:{title:"torch.jit.trace",local:"torchjittrace",headingTag:"h2"}}),re=new Z({props:{code:"aW1wb3J0JTIwdGltZSUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMGRpZmZ1c2VycyUyMGltcG9ydCUyMFN0YWJsZURpZmZ1c2lvblBpcGVsaW5lJTBBaW1wb3J0JTIwZnVuY3Rvb2xzJTBBJTBBJTIzJTIwdG9yY2glMjBkaXNhYmxlJTIwZ3JhZCUwQXRvcmNoLnNldF9ncmFkX2VuYWJsZWQoRmFsc2UpJTBBJTBBJTIzJTIwc2V0JTIwdmFyaWFibGVzJTBBbl9leHBlcmltZW50cyUyMCUzRCUyMDIlMEF1bmV0X3J1bnNfcGVyX2V4cGVyaW1lbnQlMjAlM0QlMjA1MCUwQSUwQSUyMyUyMGxvYWQlMjBzYW1wbGUlMjBpbnB1dHMlMEFkZWYlMjBnZW5lcmF0ZV9pbnB1dHMoKSUzQSUwQSUyMCUyMCUyMCUyMHNhbXBsZSUyMCUzRCUyMHRvcmNoLnJhbmRuKCgyJTJDJTIwNCUyQyUyMDY0JTJDJTIwNjQpJTJDJTIwZGV2aWNlJTNEJTIyY3VkYSUyMiUyQyUyMGR0eXBlJTNEdG9yY2guZmxvYXQxNiklMEElMjAlMjAlMjAlMjB0aW1lc3RlcCUyMCUzRCUyMHRvcmNoLnJhbmQoMSUyQyUyMGRldmljZSUzRCUyMmN1ZGElMjIlMkMlMjBkdHlwZSUzRHRvcmNoLmZsb2F0MTYpJTIwKiUyMDk5OSUwQSUyMCUyMCUyMCUyMGVuY29kZXJfaGlkZGVuX3N0YXRlcyUyMCUzRCUyMHRvcmNoLnJhbmRuKCgyJTJDJTIwNzclMkMlMjA3NjgpJTJDJTIwZGV2aWNlJTNEJTIyY3VkYSUyMiUyQyUyMGR0eXBlJTNEdG9yY2guZmxvYXQxNiklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBzYW1wbGUlMkMlMjB0aW1lc3RlcCUyQyUyMGVuY29kZXJfaGlkZGVuX3N0YXRlcyUwQSUwQSUwQXBpcGVsaW5lJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMHVzZV9zYWZldGVuc29ycyUzRFRydWUlMkMlMEEpLnRvKCUyMmN1ZGElMjIpJTBBdW5ldCUyMCUzRCUyMHBpcGVsaW5lLnVuZXQlMEF1bmV0LmV2YWwoKSUwQXVuZXQudG8obWVtb3J5X2Zvcm1hdCUzRHRvcmNoLmNoYW5uZWxzX2xhc3QpJTIwJTIwJTIzJTIwdXNlJTIwY2hhbm5lbHNfbGFzdCUyMG1lbW9yeSUyMGZvcm1hdCUwQXVuZXQuZm9yd2FyZCUyMCUzRCUyMGZ1bmN0b29scy5wYXJ0aWFsKHVuZXQuZm9yd2FyZCUyQyUyMHJldHVybl9kaWN0JTNERmFsc2UpJTIwJTIwJTIzJTIwc2V0JTIwcmV0dXJuX2RpY3QlM0RGYWxzZSUyMGFzJTIwZGVmYXVsdCUwQSUwQSUyMyUyMHdhcm11cCUwQWZvciUyMF8lMjBpbiUyMHJhbmdlKDMpJTNBJTBBJTIwJTIwJTIwJTIwd2l0aCUyMHRvcmNoLmluZmVyZW5jZV9tb2RlKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnB1dHMlMjAlM0QlMjBnZW5lcmF0ZV9pbnB1dHMoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9yaWdfb3V0cHV0JTIwJTNEJTIwdW5ldCgqaW5wdXRzKSUwQSUwQSUyMyUyMHRyYWNlJTBBcHJpbnQoJTIydHJhY2luZy4uJTIyKSUwQXVuZXRfdHJhY2VkJTIwJTNEJTIwdG9yY2guaml0LnRyYWNlKHVuZXQlMkMlMjBpbnB1dHMpJTBBdW5ldF90cmFjZWQuZXZhbCgpJTBBcHJpbnQoJTIyZG9uZSUyMHRyYWNpbmclMjIpJTBBJTBBJTIzJTIwd2FybXVwJTIwYW5kJTIwb3B0aW1pemUlMjBncmFwaCUwQWZvciUyMF8lMjBpbiUyMHJhbmdlKDUpJTNBJTBBJTIwJTIwJTIwJTIwd2l0aCUyMHRvcmNoLmluZmVyZW5jZV9tb2RlKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpbnB1dHMlMjAlM0QlMjBnZW5lcmF0ZV9pbnB1dHMoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9yaWdfb3V0cHV0JTIwJTNEJTIwdW5ldF90cmFjZWQoKmlucHV0cyklMEElMEElMjMlMjBiZW5jaG1hcmtpbmclMEF3aXRoJTIwdG9yY2guaW5mZXJlbmNlX21vZGUoKSUzQSUwQSUyMCUyMCUyMCUyMGZvciUyMF8lMjBpbiUyMHJhbmdlKG5fZXhwZXJpbWVudHMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdG9yY2guY3VkYS5zeW5jaHJvbml6ZSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3RhcnRfdGltZSUyMCUzRCUyMHRpbWUudGltZSgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwXyUyMGluJTIwcmFuZ2UodW5ldF9ydW5zX3Blcl9leHBlcmltZW50KSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9yaWdfb3V0cHV0JTIwJTNEJTIwdW5ldF90cmFjZWQoKmlucHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3JjaC5jdWRhLnN5bmNocm9uaXplKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludChmJTIydW5ldCUyMHRyYWNlZCUyMGluZmVyZW5jZSUyMHRvb2slMjAlN0J0aW1lLnRpbWUoKSUyMC0lMjBzdGFydF90aW1lJTNBLjJmJTdEJTIwc2Vjb25kcyUyMiklMEElMjAlMjAlMjAlMjBmb3IlMjBfJTIwaW4lMjByYW5nZShuX2V4cGVyaW1lbnRzKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcmNoLmN1ZGEuc3luY2hyb25pemUoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0YXJ0X3RpbWUlMjAlM0QlMjB0aW1lLnRpbWUoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZvciUyMF8lMjBpbiUyMHJhbmdlKHVuZXRfcnVuc19wZXJfZXhwZXJpbWVudCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvcmlnX291dHB1dCUyMCUzRCUyMHVuZXQoKmlucHV0cyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB0b3JjaC5jdWRhLnN5bmNocm9uaXplKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludChmJTIydW5ldCUyMGluZmVyZW5jZSUyMHRvb2slMjAlN0J0aW1lLnRpbWUoKSUyMC0lMjBzdGFydF90aW1lJTNBLjJmJTdEJTIwc2Vjb25kcyUyMiklMEElMEElMjMlMjBzYXZlJTIwdGhlJTIwbW9kZWwlMEF1bmV0X3RyYWNlZC5zYXZlKCUyMnVuZXRfdHJhY2VkLnB0JTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> time
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">import</span> functools

<span class="hljs-comment"># torch disable grad</span>
torch.set_grad_enabled(<span class="hljs-literal">False</span>)

<span class="hljs-comment"># set variables</span>
n_experiments = <span class="hljs-number">2</span>
unet_runs_per_experiment = <span class="hljs-number">50</span>

<span class="hljs-comment"># load sample inputs</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">generate_inputs</span>():
    sample = torch.randn((<span class="hljs-number">2</span>, <span class="hljs-number">4</span>, <span class="hljs-number">64</span>, <span class="hljs-number">64</span>), device=<span class="hljs-string">&quot;cuda&quot;</span>, dtype=torch.float16)
    timestep = torch.rand(<span class="hljs-number">1</span>, device=<span class="hljs-string">&quot;cuda&quot;</span>, dtype=torch.float16) * <span class="hljs-number">999</span>
    encoder_hidden_states = torch.randn((<span class="hljs-number">2</span>, <span class="hljs-number">77</span>, <span class="hljs-number">768</span>), device=<span class="hljs-string">&quot;cuda&quot;</span>, dtype=torch.float16)
    <span class="hljs-keyword">return</span> sample, timestep, encoder_hidden_states


pipeline = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;stable-diffusion-v1-5/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
unet = pipeline.unet
unet.<span class="hljs-built_in">eval</span>()
unet.to(memory_format=torch.channels_last)  <span class="hljs-comment"># use channels_last memory format</span>
unet.forward = functools.partial(unet.forward, return_dict=<span class="hljs-literal">False</span>)  <span class="hljs-comment"># set return_dict=False as default</span>

<span class="hljs-comment"># warmup</span>
<span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">3</span>):
    <span class="hljs-keyword">with</span> torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet(*inputs)

<span class="hljs-comment"># trace</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;tracing..&quot;</span>)
unet_traced = torch.jit.trace(unet, inputs)
unet_traced.<span class="hljs-built_in">eval</span>()
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;done tracing&quot;</span>)

<span class="hljs-comment"># warmup and optimize graph</span>
<span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">5</span>):
    <span class="hljs-keyword">with</span> torch.inference_mode():
        inputs = generate_inputs()
        orig_output = unet_traced(*inputs)

<span class="hljs-comment"># benchmarking</span>
<span class="hljs-keyword">with</span> torch.inference_mode():
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(unet_runs_per_experiment):
            orig_output = unet_traced(*inputs)
        torch.cuda.synchronize()
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;unet traced inference took <span class="hljs-subst">{time.time() - start_time:<span class="hljs-number">.2</span>f}</span> seconds&quot;</span>)
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(n_experiments):
        torch.cuda.synchronize()
        start_time = time.time()
        <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(unet_runs_per_experiment):
            orig_output = unet(*inputs)
        torch.cuda.synchronize()
        <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;unet inference took <span class="hljs-subst">{time.time() - start_time:<span class="hljs-number">.2</span>f}</span> seconds&quot;</span>)

<span class="hljs-comment"># save the model</span>
unet_traced.save(<span class="hljs-string">&quot;unet_traced.pt&quot;</span>)`,wrap:!1}}),ce=new Z({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEFmcm9tJTIwZGF0YWNsYXNzZXMlMjBpbXBvcnQlMjBkYXRhY2xhc3MlMEElMEElNDBkYXRhY2xhc3MlMEFjbGFzcyUyMFVOZXQyRENvbmRpdGlvbk91dHB1dCUzQSUwQSUyMCUyMCUyMCUyMHNhbXBsZSUzQSUyMHRvcmNoLlRlbnNvciUwQSUwQXBpcGVsaW5lJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNSUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMHVzZV9zYWZldGVuc29ycyUzRFRydWUlMkMlMEEpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTIzJTIwdXNlJTIwaml0dGVkJTIwdW5ldCUwQXVuZXRfdHJhY2VkJTIwJTNEJTIwdG9yY2guaml0LmxvYWQoJTIydW5ldF90cmFjZWQucHQlMjIpJTBBJTBBJTIzJTIwZGVsJTIwcGlwZWxpbmUudW5ldCUwQWNsYXNzJTIwVHJhY2VkVU5ldCh0b3JjaC5ubi5Nb2R1bGUpJTNBJTBBJTIwJTIwJTIwJTIwZGVmJTIwX19pbml0X18oc2VsZiklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdXBlcigpLl9faW5pdF9fKCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzZWxmLmluX2NoYW5uZWxzJTIwJTNEJTIwcGlwZS51bmV0LmNvbmZpZy5pbl9jaGFubmVscyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbGYuZGV2aWNlJTIwJTNEJTIwcGlwZS51bmV0LmRldmljZSUwQSUwQSUyMCUyMCUyMCUyMGRlZiUyMGZvcndhcmQoc2VsZiUyQyUyMGxhdGVudF9tb2RlbF9pbnB1dCUyQyUyMHQlMkMlMjBlbmNvZGVyX2hpZGRlbl9zdGF0ZXMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc2FtcGxlJTIwJTNEJTIwdW5ldF90cmFjZWQobGF0ZW50X21vZGVsX2lucHV0JTJDJTIwdCUyQyUyMGVuY29kZXJfaGlkZGVuX3N0YXRlcyklNUIwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwVU5ldDJEQ29uZGl0aW9uT3V0cHV0KHNhbXBsZSUzRHNhbXBsZSklMEElMEFwaXBlbGluZS51bmV0JTIwJTNEJTIwVHJhY2VkVU5ldCgpJTBBJTBBd2l0aCUyMHRvcmNoLmluZmVyZW5jZV9tb2RlKCklM0ElMEElMjAlMjAlMjAlMjBpbWFnZSUyMCUzRCUyMHBpcGUoJTVCcHJvbXB0JTVEJTIwKiUyMDElMkMlMjBudW1faW5mZXJlbmNlX3N0ZXBzJTNENTApLmltYWdlcyU1QjAlNUQ=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionPipeline
<span class="hljs-keyword">from</span> dataclasses <span class="hljs-keyword">import</span> dataclass

<span class="hljs-meta">@dataclass</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">UNet2DConditionOutput</span>:
    sample: torch.Tensor

pipeline = StableDiffusionPipeline.from_pretrained(
    <span class="hljs-string">&quot;stable-diffusion-v1-5/stable-diffusion-v1-5&quot;</span>,
    torch_dtype=torch.float16,
    use_safetensors=<span class="hljs-literal">True</span>,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># use jitted unet</span>
unet_traced = torch.jit.load(<span class="hljs-string">&quot;unet_traced.pt&quot;</span>)

<span class="hljs-comment"># del pipeline.unet</span>
<span class="hljs-keyword">class</span> <span class="hljs-title class_">TracedUNet</span>(torch.nn.Module):
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self</span>):
        <span class="hljs-built_in">super</span>().__init__()
        self.in_channels = pipe.unet.config.in_channels
        self.device = pipe.unet.device

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">forward</span>(<span class="hljs-params">self, latent_model_input, t, encoder_hidden_states</span>):
        sample = unet_traced(latent_model_input, t, encoder_hidden_states)[<span class="hljs-number">0</span>]
        <span class="hljs-keyword">return</span> UNet2DConditionOutput(sample=sample)

pipeline.unet = TracedUNet()

<span class="hljs-keyword">with</span> torch.inference_mode():
    image = pipe([prompt] * <span class="hljs-number">1</span>, num_inference_steps=<span class="hljs-number">50</span>).images[<span class="hljs-number">0</span>]`,wrap:!1}}),de=new G({props:{title:"Memory-efficient attention",local:"memory-efficient-attention",headingTag:"h2"}}),H=new C({props:{warning:!1,$$slots:{default:[Fa]},$$scope:{ctx:T}}}),Je=new Z({props:{code:"JTIzJTIwcGlwJTIwaW5zdGFsbCUyMHhmb3JtZXJzJTBBaW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwU3RhYmxlRGlmZnVzaW9uWExQaXBlbGluZSUwQSUwQXBpcGVsaW5lJTIwJTNEJTIwU3RhYmxlRGlmZnVzaW9uWExQaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyc3RhYmlsaXR5YWklMkZzdGFibGUtZGlmZnVzaW9uLXhsLWJhc2UtMS4wJTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBKS50byglMjJjdWRhJTIyKSUwQXBpcGVsaW5lLmVuYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbigp",highlighted:`<span class="hljs-comment"># pip install xformers</span>
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLPipeline

pipeline = StableDiffusionXLPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
    torch_dtype=torch.float16,
).to(<span class="hljs-string">&quot;cuda&quot;</span>)
pipeline.enable_xformers_memory_efficient_attention()`,wrap:!1}}),be=new Z({props:{code:"cGlwZWxpbmUuZGlzYWJsZV94Zm9ybWVyc19tZW1vcnlfZWZmaWNpZW50X2F0dGVudGlvbigp",highlighted:"pipeline.disable_xformers_memory_efficient_attention()",wrap:!1}}),we=new va({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/memory.md"}}),{c(){a=c("meta"),w=t(),p=c("p"),r=t(),M(m.$$.fragment),o=t(),U=c("p"),U.innerHTML=I,g=t(),x=c("p"),x.textContent=j,_=t(),M(W.$$.fragment),Te=t(),M(E.$$.fragment),Ue=t(),S=c("p"),S.innerHTML=Mn,Ze=t(),M(F.$$.fragment),ge=t(),M(A.$$.fragment),_e=t(),L=c("p"),L.textContent=un,Ie=t(),q=c("p"),q.innerHTML=yn,xe=t(),M(D.$$.fragment),Ge=t(),P=c("p"),P.textContent=Jn,ve=t(),M(K.$$.fragment),Ce=t(),M(O.$$.fragment),We=t(),M(B.$$.fragment),Be=t(),ss=c("p"),ss.innerHTML=hn,ke=t(),M(k.$$.fragment),Ve=t(),es=c("p"),es.innerHTML=bn,$e=t(),ls=c("p"),ls.innerHTML=wn,Xe=t(),M(ns.$$.fragment),Ne=t(),as=c("p"),as.innerHTML=jn,Ye=t(),M(ts.$$.fragment),ze=t(),is=c("p"),is.innerHTML=Tn,Re=t(),M(ps.$$.fragment),Qe=t(),os=c("p"),os.textContent=Un,He=t(),rs=c("ul"),rs.innerHTML=Zn,Ee=t(),ms=c("p"),ms.innerHTML=gn,Se=t(),M(cs.$$.fragment),Fe=t(),M(ds.$$.fragment),Ae=t(),fs=c("p"),fs.textContent=_n,Le=t(),Ms=c("p"),Ms.textContent=In,qe=t(),us=c("p"),us.innerHTML=xn,De=t(),M(ys.$$.fragment),Pe=t(),M(V.$$.fragment),Ke=t(),M(Js.$$.fragment),Oe=t(),hs=c("p"),hs.textContent=Gn,sl=t(),bs=c("p"),bs.innerHTML=vn,el=t(),M(ws.$$.fragment),ll=t(),js=c("blockquote"),js.innerHTML=Cn,nl=t(),M(Ts.$$.fragment),al=t(),Us=c("p"),Us.textContent=Wn,tl=t(),Zs=c("p"),Zs.innerHTML=Bn,il=t(),M(gs.$$.fragment),pl=t(),_s=c("p"),_s.textContent=kn,ol=t(),Is=c("p"),Is.innerHTML=Vn,rl=t(),M($.$$.fragment),ml=t(),xs=c("p"),xs.innerHTML=$n,cl=t(),M(Gs.$$.fragment),dl=t(),M(vs.$$.fragment),fl=t(),Cs=c("p"),Cs.innerHTML=Xn,Ml=t(),M(X.$$.fragment),ul=t(),Ws=c("p"),Ws.innerHTML=Nn,yl=t(),M(Bs.$$.fragment),Jl=t(),ks=c("p"),ks.innerHTML=Yn,hl=t(),M(Vs.$$.fragment),bl=t(),$s=c("p"),$s.innerHTML=zn,wl=t(),M(N.$$.fragment),jl=t(),Xs=c("p"),Xs.innerHTML=Rn,Tl=t(),Ns=c("ul"),Ns.innerHTML=Qn,Ul=t(),Ys=c("p"),Ys.textContent=Hn,Zl=t(),M(Y.$$.fragment),gl=t(),M(zs.$$.fragment),_l=t(),Rs=c("p"),Rs.innerHTML=En,Il=t(),Qs=c("p"),Qs.innerHTML=Sn,xl=t(),M(z.$$.fragment),Gl=t(),Hs=c("p"),Hs.innerHTML=Fn,vl=t(),M(Es.$$.fragment),Cl=t(),Ss=c("p"),Ss.innerHTML=An,Wl=t(),M(Fs.$$.fragment),Bl=t(),As=c("p"),As.textContent=Ln,kl=t(),Ls=c("p"),Ls.innerHTML=qn,Vl=t(),M(qs.$$.fragment),$l=t(),Ds=c("p"),Ds.innerHTML=Dn,Xl=t(),M(Ps.$$.fragment),Nl=t(),M(R.$$.fragment),Yl=t(),Ks=c("p"),Ks.innerHTML=Pn,zl=t(),M(Q.$$.fragment),Rl=t(),Os=c("p"),Os.innerHTML=Kn,Ql=t(),M(se.$$.fragment),Hl=t(),ee=c("p"),ee.innerHTML=On,El=t(),M(le.$$.fragment),Sl=t(),M(ne.$$.fragment),Fl=t(),ae=c("p"),ae.innerHTML=sa,Al=t(),te=c("p"),te.textContent=ea,Ll=t(),M(ie.$$.fragment),ql=t(),M(pe.$$.fragment),Dl=t(),oe=c("p"),oe.innerHTML=la,Pl=t(),M(re.$$.fragment),Kl=t(),me=c("p"),me.textContent=na,Ol=t(),M(ce.$$.fragment),sn=t(),M(de.$$.fragment),en=t(),M(H.$$.fragment),ln=t(),fe=c("p"),fe.textContent=aa,nn=t(),Me=c("p"),Me.innerHTML=ta,an=t(),ue=c("p"),ue.innerHTML=ia,tn=t(),ye=c("p"),ye.innerHTML=pa,pn=t(),M(Je.$$.fragment),on=t(),he=c("p"),he.innerHTML=oa,rn=t(),M(be.$$.fragment),mn=t(),M(we.$$.fragment),cn=t(),je=c("p"),this.h()},l(s){const e=xa("svelte-u9bgzb",document.head);a=d(e,"META",{name:!0,content:!0}),e.forEach(l),w=i(s),p=d(s,"P",{}),ja(p).forEach(l),r=i(s),u(m.$$.fragment,s),o=i(s),U=d(s,"P",{"data-svelte-h":!0}),f(U)!=="svelte-11rtgo9"&&(U.innerHTML=I),g=i(s),x=d(s,"P",{"data-svelte-h":!0}),f(x)!=="svelte-vxpcfq"&&(x.textContent=j),_=i(s),u(W.$$.fragment,s),Te=i(s),u(E.$$.fragment,s),Ue=i(s),S=d(s,"P",{"data-svelte-h":!0}),f(S)!=="svelte-646d08"&&(S.innerHTML=Mn),Ze=i(s),u(F.$$.fragment,s),ge=i(s),u(A.$$.fragment,s),_e=i(s),L=d(s,"P",{"data-svelte-h":!0}),f(L)!=="svelte-z6beln"&&(L.textContent=un),Ie=i(s),q=d(s,"P",{"data-svelte-h":!0}),f(q)!=="svelte-115ttl4"&&(q.innerHTML=yn),xe=i(s),u(D.$$.fragment,s),Ge=i(s),P=d(s,"P",{"data-svelte-h":!0}),f(P)!=="svelte-1f9wrc7"&&(P.textContent=Jn),ve=i(s),u(K.$$.fragment,s),Ce=i(s),u(O.$$.fragment,s),We=i(s),u(B.$$.fragment,s),Be=i(s),ss=d(s,"P",{"data-svelte-h":!0}),f(ss)!=="svelte-1n4q4np"&&(ss.innerHTML=hn),ke=i(s),u(k.$$.fragment,s),Ve=i(s),es=d(s,"P",{"data-svelte-h":!0}),f(es)!=="svelte-2a94z6"&&(es.innerHTML=bn),$e=i(s),ls=d(s,"P",{"data-svelte-h":!0}),f(ls)!=="svelte-c7w8iz"&&(ls.innerHTML=wn),Xe=i(s),u(ns.$$.fragment,s),Ne=i(s),as=d(s,"P",{"data-svelte-h":!0}),f(as)!=="svelte-1ut2jly"&&(as.innerHTML=jn),Ye=i(s),u(ts.$$.fragment,s),ze=i(s),is=d(s,"P",{"data-svelte-h":!0}),f(is)!=="svelte-1oxgib7"&&(is.innerHTML=Tn),Re=i(s),u(ps.$$.fragment,s),Qe=i(s),os=d(s,"P",{"data-svelte-h":!0}),f(os)!=="svelte-whoqae"&&(os.textContent=Un),He=i(s),rs=d(s,"UL",{"data-svelte-h":!0}),f(rs)!=="svelte-5cwvqn"&&(rs.innerHTML=Zn),Ee=i(s),ms=d(s,"P",{"data-svelte-h":!0}),f(ms)!=="svelte-6e1s7a"&&(ms.innerHTML=gn),Se=i(s),u(cs.$$.fragment,s),Fe=i(s),u(ds.$$.fragment,s),Ae=i(s),fs=d(s,"P",{"data-svelte-h":!0}),f(fs)!=="svelte-d36gg7"&&(fs.textContent=_n),Le=i(s),Ms=d(s,"P",{"data-svelte-h":!0}),f(Ms)!=="svelte-5ay0zr"&&(Ms.textContent=In),qe=i(s),us=d(s,"P",{"data-svelte-h":!0}),f(us)!=="svelte-jm20t4"&&(us.innerHTML=xn),De=i(s),u(ys.$$.fragment,s),Pe=i(s),u(V.$$.fragment,s),Ke=i(s),u(Js.$$.fragment,s),Oe=i(s),hs=d(s,"P",{"data-svelte-h":!0}),f(hs)!=="svelte-vsq5zh"&&(hs.textContent=Gn),sl=i(s),bs=d(s,"P",{"data-svelte-h":!0}),f(bs)!=="svelte-a79yk8"&&(bs.innerHTML=vn),el=i(s),u(ws.$$.fragment,s),ll=i(s),js=d(s,"BLOCKQUOTE",{"data-svelte-h":!0}),f(js)!=="svelte-1ch13xh"&&(js.innerHTML=Cn),nl=i(s),u(Ts.$$.fragment,s),al=i(s),Us=d(s,"P",{"data-svelte-h":!0}),f(Us)!=="svelte-1lghmz8"&&(Us.textContent=Wn),tl=i(s),Zs=d(s,"P",{"data-svelte-h":!0}),f(Zs)!=="svelte-sey8xy"&&(Zs.innerHTML=Bn),il=i(s),u(gs.$$.fragment,s),pl=i(s),_s=d(s,"P",{"data-svelte-h":!0}),f(_s)!=="svelte-nwk8qi"&&(_s.textContent=kn),ol=i(s),Is=d(s,"P",{"data-svelte-h":!0}),f(Is)!=="svelte-1jssif2"&&(Is.innerHTML=Vn),rl=i(s),u($.$$.fragment,s),ml=i(s),xs=d(s,"P",{"data-svelte-h":!0}),f(xs)!=="svelte-vdp76n"&&(xs.innerHTML=$n),cl=i(s),u(Gs.$$.fragment,s),dl=i(s),u(vs.$$.fragment,s),fl=i(s),Cs=d(s,"P",{"data-svelte-h":!0}),f(Cs)!=="svelte-lwbzkf"&&(Cs.innerHTML=Xn),Ml=i(s),u(X.$$.fragment,s),ul=i(s),Ws=d(s,"P",{"data-svelte-h":!0}),f(Ws)!=="svelte-t552fb"&&(Ws.innerHTML=Nn),yl=i(s),u(Bs.$$.fragment,s),Jl=i(s),ks=d(s,"P",{"data-svelte-h":!0}),f(ks)!=="svelte-1ic7r11"&&(ks.innerHTML=Yn),hl=i(s),u(Vs.$$.fragment,s),bl=i(s),$s=d(s,"P",{"data-svelte-h":!0}),f($s)!=="svelte-6x5rs4"&&($s.innerHTML=zn),wl=i(s),u(N.$$.fragment,s),jl=i(s),Xs=d(s,"P",{"data-svelte-h":!0}),f(Xs)!=="svelte-pp820p"&&(Xs.innerHTML=Rn),Tl=i(s),Ns=d(s,"UL",{"data-svelte-h":!0}),f(Ns)!=="svelte-8dkla6"&&(Ns.innerHTML=Qn),Ul=i(s),Ys=d(s,"P",{"data-svelte-h":!0}),f(Ys)!=="svelte-axmnvs"&&(Ys.textContent=Hn),Zl=i(s),u(Y.$$.fragment,s),gl=i(s),u(zs.$$.fragment,s),_l=i(s),Rs=d(s,"P",{"data-svelte-h":!0}),f(Rs)!=="svelte-1ik3m1u"&&(Rs.innerHTML=En),Il=i(s),Qs=d(s,"P",{"data-svelte-h":!0}),f(Qs)!=="svelte-f4wc5s"&&(Qs.innerHTML=Sn),xl=i(s),u(z.$$.fragment,s),Gl=i(s),Hs=d(s,"P",{"data-svelte-h":!0}),f(Hs)!=="svelte-17oamma"&&(Hs.innerHTML=Fn),vl=i(s),u(Es.$$.fragment,s),Cl=i(s),Ss=d(s,"P",{"data-svelte-h":!0}),f(Ss)!=="svelte-1ixqgvf"&&(Ss.innerHTML=An),Wl=i(s),u(Fs.$$.fragment,s),Bl=i(s),As=d(s,"P",{"data-svelte-h":!0}),f(As)!=="svelte-mq1z2c"&&(As.textContent=Ln),kl=i(s),Ls=d(s,"P",{"data-svelte-h":!0}),f(Ls)!=="svelte-1p1tymo"&&(Ls.innerHTML=qn),Vl=i(s),u(qs.$$.fragment,s),$l=i(s),Ds=d(s,"P",{"data-svelte-h":!0}),f(Ds)!=="svelte-1ugmkgt"&&(Ds.innerHTML=Dn),Xl=i(s),u(Ps.$$.fragment,s),Nl=i(s),u(R.$$.fragment,s),Yl=i(s),Ks=d(s,"P",{"data-svelte-h":!0}),f(Ks)!=="svelte-1boiobn"&&(Ks.innerHTML=Pn),zl=i(s),u(Q.$$.fragment,s),Rl=i(s),Os=d(s,"P",{"data-svelte-h":!0}),f(Os)!=="svelte-19ytv0"&&(Os.innerHTML=Kn),Ql=i(s),u(se.$$.fragment,s),Hl=i(s),ee=d(s,"P",{"data-svelte-h":!0}),f(ee)!=="svelte-1hmx68j"&&(ee.innerHTML=On),El=i(s),u(le.$$.fragment,s),Sl=i(s),u(ne.$$.fragment,s),Fl=i(s),ae=d(s,"P",{"data-svelte-h":!0}),f(ae)!=="svelte-1tu261v"&&(ae.innerHTML=sa),Al=i(s),te=d(s,"P",{"data-svelte-h":!0}),f(te)!=="svelte-dfwok5"&&(te.textContent=ea),Ll=i(s),u(ie.$$.fragment,s),ql=i(s),u(pe.$$.fragment,s),Dl=i(s),oe=d(s,"P",{"data-svelte-h":!0}),f(oe)!=="svelte-15jhfav"&&(oe.innerHTML=la),Pl=i(s),u(re.$$.fragment,s),Kl=i(s),me=d(s,"P",{"data-svelte-h":!0}),f(me)!=="svelte-16t9h37"&&(me.textContent=na),Ol=i(s),u(ce.$$.fragment,s),sn=i(s),u(de.$$.fragment,s),en=i(s),u(H.$$.fragment,s),ln=i(s),fe=d(s,"P",{"data-svelte-h":!0}),f(fe)!=="svelte-8q1sue"&&(fe.textContent=aa),nn=i(s),Me=d(s,"P",{"data-svelte-h":!0}),f(Me)!=="svelte-1idpc42"&&(Me.innerHTML=ta),an=i(s),ue=d(s,"P",{"data-svelte-h":!0}),f(ue)!=="svelte-4p44md"&&(ue.innerHTML=ia),tn=i(s),ye=d(s,"P",{"data-svelte-h":!0}),f(ye)!=="svelte-u62ci7"&&(ye.innerHTML=pa),pn=i(s),u(Je.$$.fragment,s),on=i(s),he=d(s,"P",{"data-svelte-h":!0}),f(he)!=="svelte-p2p9rw"&&(he.innerHTML=oa),rn=i(s),u(be.$$.fragment,s),mn=i(s),u(we.$$.fragment,s),cn=i(s),je=d(s,"P",{}),ja(je).forEach(l),this.h()},h(){Ta(a,"name","hf:doc:metadata"),Ta(a,"content",La)},m(s,e){Ga(document.head,a),n(s,w,e),n(s,p,e),n(s,r,e),y(m,s,e),n(s,o,e),n(s,U,e),n(s,g,e),n(s,x,e),n(s,_,e),y(W,s,e),n(s,Te,e),y(E,s,e),n(s,Ue,e),n(s,S,e),n(s,Ze,e),y(F,s,e),n(s,ge,e),y(A,s,e),n(s,_e,e),n(s,L,e),n(s,Ie,e),n(s,q,e),n(s,xe,e),y(D,s,e),n(s,Ge,e),n(s,P,e),n(s,ve,e),y(K,s,e),n(s,Ce,e),y(O,s,e),n(s,We,e),y(B,s,e),n(s,Be,e),n(s,ss,e),n(s,ke,e),y(k,s,e),n(s,Ve,e),n(s,es,e),n(s,$e,e),n(s,ls,e),n(s,Xe,e),y(ns,s,e),n(s,Ne,e),n(s,as,e),n(s,Ye,e),y(ts,s,e),n(s,ze,e),n(s,is,e),n(s,Re,e),y(ps,s,e),n(s,Qe,e),n(s,os,e),n(s,He,e),n(s,rs,e),n(s,Ee,e),n(s,ms,e),n(s,Se,e),y(cs,s,e),n(s,Fe,e),y(ds,s,e),n(s,Ae,e),n(s,fs,e),n(s,Le,e),n(s,Ms,e),n(s,qe,e),n(s,us,e),n(s,De,e),y(ys,s,e),n(s,Pe,e),y(V,s,e),n(s,Ke,e),y(Js,s,e),n(s,Oe,e),n(s,hs,e),n(s,sl,e),n(s,bs,e),n(s,el,e),y(ws,s,e),n(s,ll,e),n(s,js,e),n(s,nl,e),y(Ts,s,e),n(s,al,e),n(s,Us,e),n(s,tl,e),n(s,Zs,e),n(s,il,e),y(gs,s,e),n(s,pl,e),n(s,_s,e),n(s,ol,e),n(s,Is,e),n(s,rl,e),y($,s,e),n(s,ml,e),n(s,xs,e),n(s,cl,e),y(Gs,s,e),n(s,dl,e),y(vs,s,e),n(s,fl,e),n(s,Cs,e),n(s,Ml,e),y(X,s,e),n(s,ul,e),n(s,Ws,e),n(s,yl,e),y(Bs,s,e),n(s,Jl,e),n(s,ks,e),n(s,hl,e),y(Vs,s,e),n(s,bl,e),n(s,$s,e),n(s,wl,e),y(N,s,e),n(s,jl,e),n(s,Xs,e),n(s,Tl,e),n(s,Ns,e),n(s,Ul,e),n(s,Ys,e),n(s,Zl,e),y(Y,s,e),n(s,gl,e),y(zs,s,e),n(s,_l,e),n(s,Rs,e),n(s,Il,e),n(s,Qs,e),n(s,xl,e),y(z,s,e),n(s,Gl,e),n(s,Hs,e),n(s,vl,e),y(Es,s,e),n(s,Cl,e),n(s,Ss,e),n(s,Wl,e),y(Fs,s,e),n(s,Bl,e),n(s,As,e),n(s,kl,e),n(s,Ls,e),n(s,Vl,e),y(qs,s,e),n(s,$l,e),n(s,Ds,e),n(s,Xl,e),y(Ps,s,e),n(s,Nl,e),y(R,s,e),n(s,Yl,e),n(s,Ks,e),n(s,zl,e),y(Q,s,e),n(s,Rl,e),n(s,Os,e),n(s,Ql,e),y(se,s,e),n(s,Hl,e),n(s,ee,e),n(s,El,e),y(le,s,e),n(s,Sl,e),y(ne,s,e),n(s,Fl,e),n(s,ae,e),n(s,Al,e),n(s,te,e),n(s,Ll,e),y(ie,s,e),n(s,ql,e),y(pe,s,e),n(s,Dl,e),n(s,oe,e),n(s,Pl,e),y(re,s,e),n(s,Kl,e),n(s,me,e),n(s,Ol,e),y(ce,s,e),n(s,sn,e),y(de,s,e),n(s,en,e),y(H,s,e),n(s,ln,e),n(s,fe,e),n(s,nn,e),n(s,Me,e),n(s,an,e),n(s,ue,e),n(s,tn,e),n(s,ye,e),n(s,pn,e),y(Je,s,e),n(s,on,e),n(s,he,e),n(s,rn,e),y(be,s,e),n(s,mn,e),y(we,s,e),n(s,cn,e),n(s,je,e),dn=!0},p(s,[e]){const ra={};e&2&&(ra.$$scope={dirty:e,ctx:s}),W.$set(ra);const ma={};e&2&&(ma.$$scope={dirty:e,ctx:s}),B.$set(ma);const ca={};e&2&&(ca.$$scope={dirty:e,ctx:s}),k.$set(ca);const da={};e&2&&(da.$$scope={dirty:e,ctx:s}),V.$set(da);const fa={};e&2&&(fa.$$scope={dirty:e,ctx:s}),$.$set(fa);const Ma={};e&2&&(Ma.$$scope={dirty:e,ctx:s}),X.$set(Ma);const ua={};e&2&&(ua.$$scope={dirty:e,ctx:s}),N.$set(ua);const ya={};e&2&&(ya.$$scope={dirty:e,ctx:s}),Y.$set(ya);const Ja={};e&2&&(Ja.$$scope={dirty:e,ctx:s}),z.$set(Ja);const ha={};e&2&&(ha.$$scope={dirty:e,ctx:s}),R.$set(ha);const ba={};e&2&&(ba.$$scope={dirty:e,ctx:s}),Q.$set(ba);const wa={};e&2&&(wa.$$scope={dirty:e,ctx:s}),H.$set(wa)},i(s){dn||(J(m.$$.fragment,s),J(W.$$.fragment,s),J(E.$$.fragment,s),J(F.$$.fragment,s),J(A.$$.fragment,s),J(D.$$.fragment,s),J(K.$$.fragment,s),J(O.$$.fragment,s),J(B.$$.fragment,s),J(k.$$.fragment,s),J(ns.$$.fragment,s),J(ts.$$.fragment,s),J(ps.$$.fragment,s),J(cs.$$.fragment,s),J(ds.$$.fragment,s),J(ys.$$.fragment,s),J(V.$$.fragment,s),J(Js.$$.fragment,s),J(ws.$$.fragment,s),J(Ts.$$.fragment,s),J(gs.$$.fragment,s),J($.$$.fragment,s),J(Gs.$$.fragment,s),J(vs.$$.fragment,s),J(X.$$.fragment,s),J(Bs.$$.fragment,s),J(Vs.$$.fragment,s),J(N.$$.fragment,s),J(Y.$$.fragment,s),J(zs.$$.fragment,s),J(z.$$.fragment,s),J(Es.$$.fragment,s),J(Fs.$$.fragment,s),J(qs.$$.fragment,s),J(Ps.$$.fragment,s),J(R.$$.fragment,s),J(Q.$$.fragment,s),J(se.$$.fragment,s),J(le.$$.fragment,s),J(ne.$$.fragment,s),J(ie.$$.fragment,s),J(pe.$$.fragment,s),J(re.$$.fragment,s),J(ce.$$.fragment,s),J(de.$$.fragment,s),J(H.$$.fragment,s),J(Je.$$.fragment,s),J(be.$$.fragment,s),J(we.$$.fragment,s),dn=!0)},o(s){h(m.$$.fragment,s),h(W.$$.fragment,s),h(E.$$.fragment,s),h(F.$$.fragment,s),h(A.$$.fragment,s),h(D.$$.fragment,s),h(K.$$.fragment,s),h(O.$$.fragment,s),h(B.$$.fragment,s),h(k.$$.fragment,s),h(ns.$$.fragment,s),h(ts.$$.fragment,s),h(ps.$$.fragment,s),h(cs.$$.fragment,s),h(ds.$$.fragment,s),h(ys.$$.fragment,s),h(V.$$.fragment,s),h(Js.$$.fragment,s),h(ws.$$.fragment,s),h(Ts.$$.fragment,s),h(gs.$$.fragment,s),h($.$$.fragment,s),h(Gs.$$.fragment,s),h(vs.$$.fragment,s),h(X.$$.fragment,s),h(Bs.$$.fragment,s),h(Vs.$$.fragment,s),h(N.$$.fragment,s),h(Y.$$.fragment,s),h(zs.$$.fragment,s),h(z.$$.fragment,s),h(Es.$$.fragment,s),h(Fs.$$.fragment,s),h(qs.$$.fragment,s),h(Ps.$$.fragment,s),h(R.$$.fragment,s),h(Q.$$.fragment,s),h(se.$$.fragment,s),h(le.$$.fragment,s),h(ne.$$.fragment,s),h(ie.$$.fragment,s),h(pe.$$.fragment,s),h(re.$$.fragment,s),h(ce.$$.fragment,s),h(de.$$.fragment,s),h(H.$$.fragment,s),h(Je.$$.fragment,s),h(be.$$.fragment,s),h(we.$$.fragment,s),dn=!1},d(s){s&&(l(w),l(p),l(r),l(o),l(U),l(g),l(x),l(_),l(Te),l(Ue),l(S),l(Ze),l(ge),l(_e),l(L),l(Ie),l(q),l(xe),l(Ge),l(P),l(ve),l(Ce),l(We),l(Be),l(ss),l(ke),l(Ve),l(es),l($e),l(ls),l(Xe),l(Ne),l(as),l(Ye),l(ze),l(is),l(Re),l(Qe),l(os),l(He),l(rs),l(Ee),l(ms),l(Se),l(Fe),l(Ae),l(fs),l(Le),l(Ms),l(qe),l(us),l(De),l(Pe),l(Ke),l(Oe),l(hs),l(sl),l(bs),l(el),l(ll),l(js),l(nl),l(al),l(Us),l(tl),l(Zs),l(il),l(pl),l(_s),l(ol),l(Is),l(rl),l(ml),l(xs),l(cl),l(dl),l(fl),l(Cs),l(Ml),l(ul),l(Ws),l(yl),l(Jl),l(ks),l(hl),l(bl),l($s),l(wl),l(jl),l(Xs),l(Tl),l(Ns),l(Ul),l(Ys),l(Zl),l(gl),l(_l),l(Rs),l(Il),l(Qs),l(xl),l(Gl),l(Hs),l(vl),l(Cl),l(Ss),l(Wl),l(Bl),l(As),l(kl),l(Ls),l(Vl),l($l),l(Ds),l(Xl),l(Nl),l(Yl),l(Ks),l(zl),l(Rl),l(Os),l(Ql),l(Hl),l(ee),l(El),l(Sl),l(Fl),l(ae),l(Al),l(te),l(Ll),l(ql),l(Dl),l(oe),l(Pl),l(Kl),l(me),l(Ol),l(sn),l(en),l(ln),l(fe),l(nn),l(Me),l(an),l(ue),l(tn),l(ye),l(pn),l(on),l(he),l(rn),l(mn),l(cn),l(je)),l(a),b(m,s),b(W,s),b(E,s),b(F,s),b(A,s),b(D,s),b(K,s),b(O,s),b(B,s),b(k,s),b(ns,s),b(ts,s),b(ps,s),b(cs,s),b(ds,s),b(ys,s),b(V,s),b(Js,s),b(ws,s),b(Ts,s),b(gs,s),b($,s),b(Gs,s),b(vs,s),b(X,s),b(Bs,s),b(Vs,s),b(N,s),b(Y,s),b(zs,s),b(z,s),b(Es,s),b(Fs,s),b(qs,s),b(Ps,s),b(R,s),b(Q,s),b(se,s),b(le,s),b(ne,s),b(ie,s),b(pe,s),b(re,s),b(ce,s),b(de,s),b(H,s),b(Je,s),b(be,s),b(we,s)}}}const La='{"title":"Reduce memory usage","local":"reduce-memory-usage","sections":[{"title":"Multiple GPUs","local":"multiple-gpus","sections":[{"title":"Sharded checkpoints","local":"sharded-checkpoints","sections":[],"depth":3},{"title":"Device placement","local":"device-placement","sections":[],"depth":3}],"depth":2},{"title":"VAE slicing","local":"vae-slicing","sections":[],"depth":2},{"title":"VAE tiling","local":"vae-tiling","sections":[],"depth":2},{"title":"Offloading","local":"offloading","sections":[{"title":"CPU offloading","local":"cpu-offloading","sections":[],"depth":3},{"title":"Model offloading","local":"model-offloading","sections":[],"depth":3},{"title":"Group offloading","local":"group-offloading","sections":[{"title":"CUDA stream","local":"cuda-stream","sections":[],"depth":4},{"title":"Offloading to disk","local":"offloading-to-disk","sections":[],"depth":4}],"depth":3}],"depth":2},{"title":"Layerwise casting","local":"layerwise-casting","sections":[],"depth":2},{"title":"torch.channels_last","local":"torchchannelslast","sections":[],"depth":2},{"title":"torch.jit.trace","local":"torchjittrace","sections":[],"depth":2},{"title":"Memory-efficient attention","local":"memory-efficient-attention","sections":[],"depth":2}],"depth":1}';function qa(T){return ga(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class lt extends _a{constructor(a){super(),Ia(this,a,qa,Aa,Za,{})}}export{lt as component};
