import{s as de,n as he,o as ue}from"../chunks/scheduler.53228c21.js";import{S as be,i as ye,e as i,s,c as U,h as we,a as o,d as a,b as l,f as me,g as Z,j as r,k as O,l as Me,m as n,n as k,t as $,o as X,p as W}from"../chunks/index.100fac89.js";import{C as ge}from"../chunks/CopyLLMTxtMenu.b0ff86f3.js";import{C as fe}from"../chunks/CodeBlock.d30a6509.js";import{H as ee,E as Je}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.136ef3fd.js";function Te(te){let p,G,v,x,m,N,f,Y,d,ae="Caching accelerates inference by storing and reusing intermediate outputs of different layers, such as attention and feedforward layers, instead of performing the entire computation at each inference step. It significantly improves generation speed at the expense of more memory and doesn’t require additional training.",R,h,ne="This guide shows you how to use the caching methods supported in Diffusers.",F,u,I,b,se='<a href="https://huggingface.co/papers/2408.12588" rel="nofollow">Pyramid Attention Broadcast (PAB)</a> is based on the observation that attention outputs aren’t that different between successive timesteps of the generation process. The attention differences are smallest in the cross attention layers and are generally cached over a longer timestep range. This is followed by temporal attention and spatial attention layers.',Q,c,le="<p>Not all video models have three types of attention (cross, temporal, and spatial)!</p>",E,y,ie="PAB can be combined with other techniques like sequence parallelism and classifier-free guidance parallelism (data parallelism) for near real-time video generation.",V,w,oe='Set up and pass a <a href="/docs/diffusers/main/en/api/cache#diffusers.PyramidAttentionBroadcastConfig">PyramidAttentionBroadcastConfig</a> to a pipeline’s transformer to enable it. The <code>spatial_attention_block_skip_range</code> controls how often to skip attention calculations in the spatial attention blocks and the <code>spatial_attention_timestep_skip_range</code> is the range of timesteps to skip. Take care to choose an appropriate range because a smaller interval can lead to slower inference speeds and a larger interval can result in lower generation quality.',P,M,S,g,A,J,pe='<a href="https://huggingface.co/papers/2410.19355" rel="nofollow">FasterCache</a> caches and reuses attention features similar to <a href="#pyramid-attention-broadcast">PAB</a> since output differences are small for each successive timestep.',H,T,re="This method may also choose to skip the unconditional branch prediction, when using classifier-free guidance for sampling (common in most base models), and estimate it from the conditional branch prediction if there is significant redundancy in the predicted latent outputs between successive timesteps.",q,_,ce='Set up and pass a <a href="/docs/diffusers/main/en/api/cache#diffusers.FasterCacheConfig">FasterCacheConfig</a> to a pipeline’s transformer to enable it.',z,C,L,j,D,B,K;return m=new ge({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),f=new ee({props:{title:"Caching",local:"caching",headingTag:"h1"}}),u=new ee({props:{title:"Pyramid Attention Broadcast",local:"pyramid-attention-broadcast",headingTag:"h2"}}),M=new fe({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQ29nVmlkZW9YUGlwZWxpbmUlMkMlMjBQeXJhbWlkQXR0ZW50aW9uQnJvYWRjYXN0Q29uZmlnJTBBJTBBcGlwZWxpbmUlMjAlM0QlMjBDb2dWaWRlb1hQaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoJTIyVEhVRE0lMkZDb2dWaWRlb1gtNWIlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2KSUwQXBpcGVsaW5lLnRvKCUyMmN1ZGElMjIpJTBBJTBBY29uZmlnJTIwJTNEJTIwUHlyYW1pZEF0dGVudGlvbkJyb2FkY2FzdENvbmZpZyglMEElMjAlMjAlMjAlMjBzcGF0aWFsX2F0dGVudGlvbl9ibG9ja19za2lwX3JhbmdlJTNEMiUyQyUwQSUyMCUyMCUyMCUyMHNwYXRpYWxfYXR0ZW50aW9uX3RpbWVzdGVwX3NraXBfcmFuZ2UlM0QoMTAwJTJDJTIwODAwKSUyQyUwQSUyMCUyMCUyMCUyMGN1cnJlbnRfdGltZXN0ZXBfY2FsbGJhY2slM0RsYW1iZGElM0ElMjBwaXBlLmN1cnJlbnRfdGltZXN0ZXAlMkMlMEEpJTBBcGlwZWxpbmUudHJhbnNmb3JtZXIuZW5hYmxlX2NhY2hlKGNvbmZpZyk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline, PyramidAttentionBroadcastConfig

pipeline = CogVideoXPipeline.from_pretrained(<span class="hljs-string">&quot;THUDM/CogVideoX-5b&quot;</span>, torch_dtype=torch.bfloat16)
pipeline.to(<span class="hljs-string">&quot;cuda&quot;</span>)

config = PyramidAttentionBroadcastConfig(
    spatial_attention_block_skip_range=<span class="hljs-number">2</span>,
    spatial_attention_timestep_skip_range=(<span class="hljs-number">100</span>, <span class="hljs-number">800</span>),
    current_timestep_callback=<span class="hljs-keyword">lambda</span>: pipe.current_timestep,
)
pipeline.transformer.enable_cache(config)`,wrap:!1}}),g=new ee({props:{title:"FasterCache",local:"fastercache",headingTag:"h2"}}),C=new fe({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwZGlmZnVzZXJzJTIwaW1wb3J0JTIwQ29nVmlkZW9YUGlwZWxpbmUlMkMlMjBGYXN0ZXJDYWNoZUNvbmZpZyUwQSUwQXBpcGUlMjBsaW5lJTNEJTIwQ29nVmlkZW9YUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUyMlRIVURNJTJGQ29nVmlkZW9YLTViJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiklMEFwaXBlbGluZS50byglMjJjdWRhJTIyKSUwQSUwQWNvbmZpZyUyMCUzRCUyMEZhc3RlckNhY2hlQ29uZmlnKCUwQSUyMCUyMCUyMCUyMHNwYXRpYWxfYXR0ZW50aW9uX2Jsb2NrX3NraXBfcmFuZ2UlM0QyJTJDJTBBJTIwJTIwJTIwJTIwc3BhdGlhbF9hdHRlbnRpb25fdGltZXN0ZXBfc2tpcF9yYW5nZSUzRCgtMSUyQyUyMDY4MSklMkMlMEElMjAlMjAlMjAlMjBjdXJyZW50X3RpbWVzdGVwX2NhbGxiYWNrJTNEbGFtYmRhJTNBJTIwcGlwZS5jdXJyZW50X3RpbWVzdGVwJTJDJTBBJTIwJTIwJTIwJTIwYXR0ZW50aW9uX3dlaWdodF9jYWxsYmFjayUzRGxhbWJkYSUyMF8lM0ElMjAwLjMlMkMlMEElMjAlMjAlMjAlMjB1bmNvbmRpdGlvbmFsX2JhdGNoX3NraXBfcmFuZ2UlM0Q1JTJDJTBBJTIwJTIwJTIwJTIwdW5jb25kaXRpb25hbF9iYXRjaF90aW1lc3RlcF9za2lwX3JhbmdlJTNEKC0xJTJDJTIwNzgxKSUyQyUwQSUyMCUyMCUyMCUyMHRlbnNvcl9mb3JtYXQlM0QlMjJCRkNIVyUyMiUyQyUwQSklMEFwaXBlbGluZS50cmFuc2Zvcm1lci5lbmFibGVfY2FjaGUoY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> CogVideoXPipeline, FasterCacheConfig

pipe line= CogVideoXPipeline.from_pretrained(<span class="hljs-string">&quot;THUDM/CogVideoX-5b&quot;</span>, torch_dtype=torch.bfloat16)
pipeline.to(<span class="hljs-string">&quot;cuda&quot;</span>)

config = FasterCacheConfig(
    spatial_attention_block_skip_range=<span class="hljs-number">2</span>,
    spatial_attention_timestep_skip_range=(-<span class="hljs-number">1</span>, <span class="hljs-number">681</span>),
    current_timestep_callback=<span class="hljs-keyword">lambda</span>: pipe.current_timestep,
    attention_weight_callback=<span class="hljs-keyword">lambda</span> _: <span class="hljs-number">0.3</span>,
    unconditional_batch_skip_range=<span class="hljs-number">5</span>,
    unconditional_batch_timestep_skip_range=(-<span class="hljs-number">1</span>, <span class="hljs-number">781</span>),
    tensor_format=<span class="hljs-string">&quot;BFCHW&quot;</span>,
)
pipeline.transformer.enable_cache(config)`,wrap:!1}}),j=new Je({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/cache.md"}}),{c(){p=i("meta"),G=s(),v=i("p"),x=s(),U(m.$$.fragment),N=s(),U(f.$$.fragment),Y=s(),d=i("p"),d.textContent=ae,R=s(),h=i("p"),h.textContent=ne,F=s(),U(u.$$.fragment),I=s(),b=i("p"),b.innerHTML=se,Q=s(),c=i("blockquote"),c.innerHTML=le,E=s(),y=i("p"),y.textContent=ie,V=s(),w=i("p"),w.innerHTML=oe,P=s(),U(M.$$.fragment),S=s(),U(g.$$.fragment),A=s(),J=i("p"),J.innerHTML=pe,H=s(),T=i("p"),T.textContent=re,q=s(),_=i("p"),_.innerHTML=ce,z=s(),U(C.$$.fragment),L=s(),U(j.$$.fragment),D=s(),B=i("p"),this.h()},l(e){const t=we("svelte-u9bgzb",document.head);p=o(t,"META",{name:!0,content:!0}),t.forEach(a),G=l(e),v=o(e,"P",{}),me(v).forEach(a),x=l(e),Z(m.$$.fragment,e),N=l(e),Z(f.$$.fragment,e),Y=l(e),d=o(e,"P",{"data-svelte-h":!0}),r(d)!=="svelte-y872v2"&&(d.textContent=ae),R=l(e),h=o(e,"P",{"data-svelte-h":!0}),r(h)!=="svelte-67viyl"&&(h.textContent=ne),F=l(e),Z(u.$$.fragment,e),I=l(e),b=o(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-175y28v"&&(b.innerHTML=se),Q=l(e),c=o(e,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(c)!=="svelte-jobxuf"&&(c.innerHTML=le),E=l(e),y=o(e,"P",{"data-svelte-h":!0}),r(y)!=="svelte-rlr2qi"&&(y.textContent=ie),V=l(e),w=o(e,"P",{"data-svelte-h":!0}),r(w)!=="svelte-1c6hyt4"&&(w.innerHTML=oe),P=l(e),Z(M.$$.fragment,e),S=l(e),Z(g.$$.fragment,e),A=l(e),J=o(e,"P",{"data-svelte-h":!0}),r(J)!=="svelte-1hv75yq"&&(J.innerHTML=pe),H=l(e),T=o(e,"P",{"data-svelte-h":!0}),r(T)!=="svelte-1ekw8kg"&&(T.textContent=re),q=l(e),_=o(e,"P",{"data-svelte-h":!0}),r(_)!=="svelte-11scqgp"&&(_.innerHTML=ce),z=l(e),Z(C.$$.fragment,e),L=l(e),Z(j.$$.fragment,e),D=l(e),B=o(e,"P",{}),me(B).forEach(a),this.h()},h(){O(p,"name","hf:doc:metadata"),O(p,"content",_e),O(c,"class","tip")},m(e,t){Me(document.head,p),n(e,G,t),n(e,v,t),n(e,x,t),k(m,e,t),n(e,N,t),k(f,e,t),n(e,Y,t),n(e,d,t),n(e,R,t),n(e,h,t),n(e,F,t),k(u,e,t),n(e,I,t),n(e,b,t),n(e,Q,t),n(e,c,t),n(e,E,t),n(e,y,t),n(e,V,t),n(e,w,t),n(e,P,t),k(M,e,t),n(e,S,t),k(g,e,t),n(e,A,t),n(e,J,t),n(e,H,t),n(e,T,t),n(e,q,t),n(e,_,t),n(e,z,t),k(C,e,t),n(e,L,t),k(j,e,t),n(e,D,t),n(e,B,t),K=!0},p:he,i(e){K||($(m.$$.fragment,e),$(f.$$.fragment,e),$(u.$$.fragment,e),$(M.$$.fragment,e),$(g.$$.fragment,e),$(C.$$.fragment,e),$(j.$$.fragment,e),K=!0)},o(e){X(m.$$.fragment,e),X(f.$$.fragment,e),X(u.$$.fragment,e),X(M.$$.fragment,e),X(g.$$.fragment,e),X(C.$$.fragment,e),X(j.$$.fragment,e),K=!1},d(e){e&&(a(G),a(v),a(x),a(N),a(Y),a(d),a(R),a(h),a(F),a(I),a(b),a(Q),a(c),a(E),a(y),a(V),a(w),a(P),a(S),a(A),a(J),a(H),a(T),a(q),a(_),a(z),a(L),a(D),a(B)),a(p),W(m,e),W(f,e),W(u,e),W(M,e),W(g,e),W(C,e),W(j,e)}}}const _e='{"title":"Caching","local":"caching","sections":[{"title":"Pyramid Attention Broadcast","local":"pyramid-attention-broadcast","sections":[],"depth":2},{"title":"FasterCache","local":"fastercache","sections":[],"depth":2}],"depth":1}';function Ce(te){return ue(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Xe extends be{constructor(p){super(),ye(this,p,Ce,Te,de,{})}}export{Xe as component};
