import{S as rl,i as dl,s as ll,e as n,k as d,w as g,t as i,M as cl,c as a,d as o,m as l,a as s,x as _,h as r,b as c,G as e,g as m,y as v,q as b,o as M,B as w,v as pl,L as sl}from"../../../chunks/vendor-hf-doc-builder.js";import{T as ml}from"../../../chunks/Tip-hf-doc-builder.js";import{D as I}from"../../../chunks/Docstring-hf-doc-builder.js";import{C as il}from"../../../chunks/CodeBlock-hf-doc-builder.js";import{I as V}from"../../../chunks/IconCopyLink-hf-doc-builder.js";import{E as al}from"../../../chunks/ExampleCodeBlock-hf-doc-builder.js";function ul(me){let f,L,y,h,x,u,A,H;return{c(){f=n("p"),L=i("Make sure to check out the Schedulers "),y=n("a"),h=i("guide"),x=i(` to learn how to explore the tradeoff between
scheduler speed and quality, and see the `),u=n("a"),A=i("reuse components across pipelines"),H=i(`
section to learn how to efficiently load the same components into multiple pipelines.`),this.h()},l(Y){f=a(Y,"P",{});var $=s(f);L=r($,"Make sure to check out the Schedulers "),y=a($,"A",{href:!0});var J=s(y);h=r(J,"guide"),J.forEach(o),x=r($,` to learn how to explore the tradeoff between
scheduler speed and quality, and see the `),u=a($,"A",{href:!0});var Tt=s(u);A=r(Tt,"reuse components across pipelines"),Tt.forEach(o),H=r($,`
section to learn how to efficiently load the same components into multiple pipelines.`),$.forEach(o),this.h()},h(){c(y,"href","/using-diffusers/schedulers"),c(u,"href","/using-diffusers/loading#reuse-components-across-pipelines")},m(Y,$){m(Y,f,$),e(f,L),e(f,y),e(y,h),e(f,x),e(f,u),e(u,A),e(f,H)},d(Y){Y&&o(f)}}}function fl(me){let f,L,y,h,x;return h=new il({props:{code:"aW1wb3J0JTIwc2NpcHklMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjBkaWZmdXNlcnMlMjBpbXBvcnQlMjBBdWRpb0xETTJQaXBlbGluZSUwQSUwQXJlcG9faWQlMjAlM0QlMjAlMjJjdnNzcCUyRmF1ZGlvbGRtMiUyMiUwQXBpcGUlMjAlM0QlMjBBdWRpb0xETTJQaXBlbGluZS5mcm9tX3ByZXRyYWluZWQocmVwb19pZCUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiklMEFwaXBlJTIwJTNEJTIwcGlwZS50byglMjJjdWRhJTIyKSUwQSUwQSUyMyUyMGRlZmluZSUyMHRoZSUyMHByb21wdHMlMEFwcm9tcHQlMjAlM0QlMjAlMjJUaGUlMjBzb3VuZCUyMG9mJTIwYSUyMGhhbW1lciUyMGhpdHRpbmclMjBhJTIwd29vZGVuJTIwc3VyZmFjZS4lMjIlMEFuZWdhdGl2ZV9wcm9tcHQlMjAlM0QlMjAlMjJMb3clMjBxdWFsaXR5LiUyMiUwQSUwQSUyMyUyMHNldCUyMHRoZSUyMHNlZWQlMjBmb3IlMjBnZW5lcmF0b3IlMEFnZW5lcmF0b3IlMjAlM0QlMjB0b3JjaC5HZW5lcmF0b3IoJTIyY3VkYSUyMikubWFudWFsX3NlZWQoMCklMEElMEElMjMlMjBydW4lMjB0aGUlMjBnZW5lcmF0aW9uJTBBYXVkaW8lMjAlM0QlMjBwaXBlKCUwQSUyMCUyMCUyMCUyMHByb21wdCUyQyUwQSUyMCUyMCUyMCUyMG5lZ2F0aXZlX3Byb21wdCUzRG5lZ2F0aXZlX3Byb21wdCUyQyUwQSUyMCUyMCUyMCUyMG51bV9pbmZlcmVuY2Vfc3RlcHMlM0QyMDAlMkMlMEElMjAlMjAlMjAlMjBhdWRpb19sZW5ndGhfaW5fcyUzRDEwLjAlMkMlMEElMjAlMjAlMjAlMjBudW1fd2F2ZWZvcm1zX3Blcl9wcm9tcHQlM0QzJTJDJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdG9yJTNEZ2VuZXJhdG9yJTJDJTBBKS5hdWRpb3MlMEElMEElMjMlMjBzYXZlJTIwdGhlJTIwYmVzdCUyMGF1ZGlvJTIwc2FtcGxlJTIwKGluZGV4JTIwMCklMjBhcyUyMGElMjAud2F2JTIwZmlsZSUwQXNjaXB5LmlvLndhdmZpbGUud3JpdGUoJTIydGVjaG5vLndhdiUyMiUyQyUyMHJhdGUlM0QxNjAwMCUyQyUyMGRhdGElM0RhdWRpbyU1QjAlNUQp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> scipy
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AudioLDM2Pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>repo_id = <span class="hljs-string">&quot;cvssp/audioldm2&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>pipe = AudioLDM2Pipeline.from_pretrained(repo_id, torch_dtype=torch.float16)
<span class="hljs-meta">&gt;&gt;&gt; </span>pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># define the prompts</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt = <span class="hljs-string">&quot;The sound of a hammer hitting a wooden surface.&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>negative_prompt = <span class="hljs-string">&quot;Low quality.&quot;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># set the seed for generator</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generator = torch.Generator(<span class="hljs-string">&quot;cuda&quot;</span>).manual_seed(<span class="hljs-number">0</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># run the generation</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = pipe(
<span class="hljs-meta">... </span>    prompt,
<span class="hljs-meta">... </span>    negative_prompt=negative_prompt,
<span class="hljs-meta">... </span>    num_inference_steps=<span class="hljs-number">200</span>,
<span class="hljs-meta">... </span>    audio_length_in_s=<span class="hljs-number">10.0</span>,
<span class="hljs-meta">... </span>    num_waveforms_per_prompt=<span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    generator=generator,
<span class="hljs-meta">... </span>).audios

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># save the best audio sample (index 0) as a .wav file</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>scipy.io.wavfile.write(<span class="hljs-string">&quot;techno.wav&quot;</span>, rate=<span class="hljs-number">16000</span>, data=audio[<span class="hljs-number">0</span>])`}}),{c(){f=n("p"),L=i("Examples:"),y=d(),g(h.$$.fragment)},l(u){f=a(u,"P",{});var A=s(f);L=r(A,"Examples:"),A.forEach(o),y=l(u),_(h.$$.fragment,u)},m(u,A){m(u,f,A),e(f,L),m(u,y,A),v(h,u,A),x=!0},p:sl,i(u){x||(b(h.$$.fragment,u),x=!0)},o(u){M(h.$$.fragment,u),x=!1},d(u){u&&o(f),u&&o(y),w(h,u)}}}function hl(me){let f,L,y,h,x;return h=new il({props:{code:"aW1wb3J0JTIwc2NpcHklMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjBkaWZmdXNlcnMlMjBpbXBvcnQlMjBBdWRpb0xETTJQaXBlbGluZSUwQSUwQXJlcG9faWQlMjAlM0QlMjAlMjJjdnNzcCUyRmF1ZGlvbGRtMiUyMiUwQXBpcGUlMjAlM0QlMjBBdWRpb0xETTJQaXBlbGluZS5mcm9tX3ByZXRyYWluZWQocmVwb19pZCUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiklMEFwaXBlJTIwJTNEJTIwcGlwZS50byglMjJjdWRhJTIyKSUwQSUwQSUyMyUyMEdldCUyMHRleHQlMjBlbWJlZGRpbmclMjB2ZWN0b3JzJTBBcHJvbXB0X2VtYmVkcyUyQyUyMGF0dGVudGlvbl9tYXNrJTJDJTIwZ2VuZXJhdGVkX3Byb21wdF9lbWJlZHMlMjAlM0QlMjBwaXBlLmVuY29kZV9wcm9tcHQoJTBBJTIwJTIwJTIwJTIwcHJvbXB0JTNEJTIyVGVjaG5vJTIwbXVzaWMlMjB3aXRoJTIwYSUyMHN0cm9uZyUyQyUyMHVwYmVhdCUyMHRlbXBvJTIwYW5kJTIwaGlnaCUyMG1lbG9kaWMlMjByaWZmcyUyMiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZSUzRCUyMmN1ZGElMjIlMkMlMEElMjAlMjAlMjAlMjBkb19jbGFzc2lmaWVyX2ZyZWVfZ3VpZGFuY2UlM0RUcnVlJTJDJTBBKSUwQSUwQSUyMyUyMFBhc3MlMjB0ZXh0JTIwZW1iZWRkaW5ncyUyMHRvJTIwcGlwZWxpbmUlMjBmb3IlMjB0ZXh0LWNvbmRpdGlvbmFsJTIwYXVkaW8lMjBnZW5lcmF0aW9uJTBBYXVkaW8lMjAlM0QlMjBwaXBlKCUwQSUyMCUyMCUyMCUyMHByb21wdF9lbWJlZHMlM0Rwcm9tcHRfZW1iZWRzJTJDJTBBJTIwJTIwJTIwJTIwYXR0ZW50aW9uX21hc2slM0RhdHRlbnRpb25fbWFzayUyQyUwQSUyMCUyMCUyMCUyMGdlbmVyYXRlZF9wcm9tcHRfZW1iZWRzJTNEZ2VuZXJhdGVkX3Byb21wdF9lbWJlZHMlMkMlMEElMjAlMjAlMjAlMjBudW1faW5mZXJlbmNlX3N0ZXBzJTNEMjAwJTJDJTBBJTIwJTIwJTIwJTIwYXVkaW9fbGVuZ3RoX2luX3MlM0QxMC4wJTJDJTBBKS5hdWRpb3MlNUIwJTVEJTBBJTBBJTIzJTIwc2F2ZSUyMGdlbmVyYXRlZCUyMGF1ZGlvJTIwc2FtcGxlJTBBc2NpcHkuaW8ud2F2ZmlsZS53cml0ZSglMjJ0ZWNobm8ud2F2JTIyJTJDJTIwcmF0ZSUzRDE2MDAwJTJDJTIwZGF0YSUzRGF1ZGlvKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> scipy
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> AudioLDM2Pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>repo_id = <span class="hljs-string">&quot;cvssp/audioldm2&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>pipe = AudioLDM2Pipeline.from_pretrained(repo_id, torch_dtype=torch.float16)
<span class="hljs-meta">&gt;&gt;&gt; </span>pipe = pipe.to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Get text embedding vectors</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>prompt_embeds, attention_mask, generated_prompt_embeds = pipe.encode_prompt(
<span class="hljs-meta">... </span>    prompt=<span class="hljs-string">&quot;Techno music with a strong, upbeat tempo and high melodic riffs&quot;</span>,
<span class="hljs-meta">... </span>    device=<span class="hljs-string">&quot;cuda&quot;</span>,
<span class="hljs-meta">... </span>    do_classifier_free_guidance=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Pass text embeddings to pipeline for text-conditional audio generation</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = pipe(
<span class="hljs-meta">... </span>    prompt_embeds=prompt_embeds,
<span class="hljs-meta">... </span>    attention_mask=attention_mask,
<span class="hljs-meta">... </span>    generated_prompt_embeds=generated_prompt_embeds,
<span class="hljs-meta">... </span>    num_inference_steps=<span class="hljs-number">200</span>,
<span class="hljs-meta">... </span>    audio_length_in_s=<span class="hljs-number">10.0</span>,
<span class="hljs-meta">... </span>).audios[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># save generated audio sample</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>scipy.io.wavfile.write(<span class="hljs-string">&quot;techno.wav&quot;</span>, rate=<span class="hljs-number">16000</span>, data=audio)`}}),{c(){f=n("p"),L=i("Example:"),y=d(),g(h.$$.fragment)},l(u){f=a(u,"P",{});var A=s(f);L=r(A,"Example:"),A.forEach(o),y=l(u),_(h.$$.fragment,u)},m(u,A){m(u,f,A),e(f,L),m(u,y,A),v(h,u,A),x=!0},p:sl,i(u){x||(b(h.$$.fragment,u),x=!0)},o(u){M(h.$$.fragment,u),x=!1},d(u){u&&o(f),u&&o(y),w(h,u)}}}function gl(me){let f,L,y,h,x,u,A,H,Y,$,J,Tt,Be,na,aa,nn,D,sa,Ie,ia,ra,Ft,da,la,Je,ca,pa,qe,ma,ua,Ge,fa,ha,Se,ga,_a,Zt,va,ba,Fe,Ma,wa,Ot,ya,Da,an,At,Ta,sn,kt,Wt,Aa,rn,S,ka,Ze,xa,Ea,Oe,La,$a,dn,K,ue,zt,We,ja,Qt,Ua,ln,ee,fe,Rt,ze,Na,Xt,Pa,cn,xt,Ca,pn,Et,Ba,mn,he,Vt,j,Ht,Ia,Ja,Yt,qa,Ga,Kt,Sa,Fa,eo,Za,Oa,to,Wa,za,te,U,oo,Qe,Qa,Ra,no,Xa,Va,ao,Ha,Ya,so,Ka,es,io,ts,os,N,ro,Re,ns,as,lo,ss,is,co,rs,ds,po,ls,cs,mo,ps,ms,P,uo,Xe,us,fs,fo,hs,gs,ho,_s,vs,go,bs,Ms,_o,ws,un,oe,ge,vo,Ve,ys,bo,Ds,fn,F,Mo,Ts,As,wo,ks,xs,He,Es,yo,Ls,$s,hn,ne,_e,Do,Ye,js,To,Us,gn,ve,ae,Ns,Ao,Ps,Cs,ko,Bs,Is,Js,se,qs,xo,Gs,Ss,Eo,Fs,Zs,_n,ie,be,Lo,Ke,Os,$o,Ws,vn,Me,jo,zs,Qs,et,Rs,Uo,Xs,Vs,bn,we,Hs,tt,Ys,Ks,Mn,ye,wn,re,De,No,ot,ei,Po,ti,yn,T,nt,oi,Co,ni,ai,at,si,Lt,ii,ri,di,Z,st,li,Bo,ci,pi,Te,mi,Ae,it,ui,rt,fi,Io,hi,gi,_i,ke,dt,vi,C,bi,Jo,Mi,wi,qo,yi,Di,Go,Ti,Ai,So,ki,xi,Ei,xe,lt,Li,Fo,$i,ji,O,ct,Ui,Zo,Ni,Pi,Ee,Ci,Le,pt,Bi,Oo,Ii,Dn,de,$e,Wo,mt,Ji,zo,qi,Tn,q,ut,Gi,ft,Si,Qo,Fi,Zi,Oi,$t,ht,An,le,je,Ro,gt,Wi,Xo,zi,kn,B,_t,Qi,G,Ri,jt,Xi,Vi,Vo,Hi,Yi,Ho,Ki,er,tr,vt,or,Ut,nr,ar,sr,Ue,bt,ir,Mt,rr,Nt,dr,lr,xn,ce,Ne,Yo,wt,cr,Ko,pr,En,pe,yt,mr,en,ur,Ln;return u=new V({}),We=new V({}),ze=new V({}),Ve=new V({}),Ye=new V({}),Ke=new V({}),ye=new ml({props:{$$slots:{default:[ul]},$$scope:{ctx:me}}}),ot=new V({}),nt=new I({props:{name:"class diffusers.AudioLDM2Pipeline",anchor:"diffusers.AudioLDM2Pipeline",parameters:[{name:"vae",val:": AutoencoderKL"},{name:"text_encoder",val:": ClapModel"},{name:"text_encoder_2",val:": T5EncoderModel"},{name:"projection_model",val:": AudioLDM2ProjectionModel"},{name:"language_model",val:": GPT2Model"},{name:"tokenizer",val:": typing.Union[transformers.models.roberta.tokenization_roberta.RobertaTokenizer, transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast]"},{name:"tokenizer_2",val:": typing.Union[transformers.models.t5.tokenization_t5.T5Tokenizer, transformers.models.t5.tokenization_t5_fast.T5TokenizerFast]"},{name:"feature_extractor",val:": ClapFeatureExtractor"},{name:"unet",val:": AudioLDM2UNet2DConditionModel"},{name:"scheduler",val:": KarrasDiffusionSchedulers"},{name:"vocoder",val:": SpeechT5HifiGan"}],parametersDescription:[{anchor:"diffusers.AudioLDM2Pipeline.vae",description:`<strong>vae</strong> (<a href="/docs/diffusers/main/en/api/models/autoencoderkl#diffusers.AutoencoderKL">AutoencoderKL</a>) &#x2014;
Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations.`,name:"vae"},{anchor:"diffusers.AudioLDM2Pipeline.text_encoder",description:`<strong>text_encoder</strong> (<a href="https://huggingface.co/docs/transformers/main/en/model_doc/clap#transformers.ClapModel" rel="nofollow">ClapModel</a>) &#x2014;
First frozen text-encoder. AudioLDM2 uses the joint audio-text embedding model
<a href="https://huggingface.co/docs/transformers/model_doc/clap#transformers.CLAPTextModelWithProjection" rel="nofollow">CLAP</a>,
specifically the <a href="https://huggingface.co/laion/clap-htsat-unfused" rel="nofollow">laion/clap-htsat-unfused</a> variant. The
text branch is used to encode the text prompt to a prompt embedding. The full audio-text model is used to
rank generated waveforms against the text prompt by computing similarity scores.`,name:"text_encoder"},{anchor:"diffusers.AudioLDM2Pipeline.text_encoder_2",description:`<strong>text_encoder_2</strong> (<a href="https://huggingface.co/docs/transformers/main/en/model_doc/t5#transformers.T5EncoderModel" rel="nofollow">T5EncoderModel</a>) &#x2014;
Second frozen text-encoder. AudioLDM2 uses the encoder of
<a href="https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5EncoderModel" rel="nofollow">T5</a>, specifically the
<a href="https://huggingface.co/google/flan-t5-large" rel="nofollow">google/flan-t5-large</a> variant.`,name:"text_encoder_2"},{anchor:"diffusers.AudioLDM2Pipeline.projection_model",description:`<strong>projection_model</strong> (<a href="/docs/diffusers/main/en/api/pipelines/audioldm2#diffusers.AudioLDM2ProjectionModel">AudioLDM2ProjectionModel</a>) &#x2014;
A trained model used to linearly project the hidden-states from the first and second text encoder models
and insert learned SOS and EOS token embeddings. The projected hidden-states from the two text encoders are
concatenated to give the input to the language model.`,name:"projection_model"},{anchor:"diffusers.AudioLDM2Pipeline.language_model",description:`<strong>language_model</strong> (<a href="https://huggingface.co/docs/transformers/main/en/model_doc/gpt2#transformers.GPT2Model" rel="nofollow">GPT2Model</a>) &#x2014;
An auto-regressive language model used to generate a sequence of hidden-states conditioned on the projected
outputs from the two text encoders.`,name:"language_model"},{anchor:"diffusers.AudioLDM2Pipeline.tokenizer",description:`<strong>tokenizer</strong> (<a href="https://huggingface.co/docs/transformers/main/en/model_doc/roberta#transformers.RobertaTokenizer" rel="nofollow">RobertaTokenizer</a>) &#x2014;
Tokenizer to tokenize text for the first frozen text-encoder.`,name:"tokenizer"},{anchor:"diffusers.AudioLDM2Pipeline.tokenizer_2",description:`<strong>tokenizer_2</strong> (<a href="https://huggingface.co/docs/transformers/main/en/model_doc/t5#transformers.T5Tokenizer" rel="nofollow">T5Tokenizer</a>) &#x2014;
Tokenizer to tokenize text for the second frozen text-encoder.`,name:"tokenizer_2"},{anchor:"diffusers.AudioLDM2Pipeline.feature_extractor",description:`<strong>feature_extractor</strong> (<a href="https://huggingface.co/docs/transformers/main/en/model_doc/clap#transformers.ClapFeatureExtractor" rel="nofollow">ClapFeatureExtractor</a>) &#x2014;
Feature extractor to pre-process generated audio waveforms to log-mel spectrograms for automatic scoring.`,name:"feature_extractor"},{anchor:"diffusers.AudioLDM2Pipeline.unet",description:`<strong>unet</strong> (<a href="/docs/diffusers/main/en/api/models/unet2d-cond#diffusers.UNet2DConditionModel">UNet2DConditionModel</a>) &#x2014;
A <code>UNet2DConditionModel</code> to denoise the encoded audio latents.`,name:"unet"},{anchor:"diffusers.AudioLDM2Pipeline.scheduler",description:`<strong>scheduler</strong> (<a href="/docs/diffusers/main/en/api/schedulers/overview#diffusers.SchedulerMixin">SchedulerMixin</a>) &#x2014;
A scheduler to be used in combination with <code>unet</code> to denoise the encoded audio latents. Can be one of
<a href="/docs/diffusers/main/en/api/schedulers/ddim#diffusers.DDIMScheduler">DDIMScheduler</a>, <a href="/docs/diffusers/main/en/api/schedulers/lms_discrete#diffusers.LMSDiscreteScheduler">LMSDiscreteScheduler</a>, or <a href="/docs/diffusers/main/en/api/schedulers/pndm#diffusers.PNDMScheduler">PNDMScheduler</a>.`,name:"scheduler"},{anchor:"diffusers.AudioLDM2Pipeline.vocoder",description:`<strong>vocoder</strong> (<a href="https://huggingface.co/docs/transformers/main/en/model_doc/speecht5#transformers.SpeechT5HifiGan" rel="nofollow">SpeechT5HifiGan</a>) &#x2014;
Vocoder of class <code>SpeechT5HifiGan</code> to convert the mel-spectrogram latents to the final audio waveform.`,name:"vocoder"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py#L103"}}),st=new I({props:{name:"__call__",anchor:"diffusers.AudioLDM2Pipeline.__call__",parameters:[{name:"prompt",val:": typing.Union[str, typing.List[str]] = None"},{name:"audio_length_in_s",val:": typing.Optional[float] = None"},{name:"num_inference_steps",val:": int = 200"},{name:"guidance_scale",val:": float = 3.5"},{name:"negative_prompt",val:": typing.Union[str, typing.List[str], NoneType] = None"},{name:"num_waveforms_per_prompt",val:": typing.Optional[int] = 1"},{name:"eta",val:": float = 0.0"},{name:"generator",val:": typing.Union[torch._C.Generator, typing.List[torch._C.Generator], NoneType] = None"},{name:"latents",val:": typing.Optional[torch.FloatTensor] = None"},{name:"prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"generated_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_generated_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"negative_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"max_new_tokens",val:": typing.Optional[int] = None"},{name:"return_dict",val:": bool = True"},{name:"callback",val:": typing.Union[typing.Callable[[int, int, torch.FloatTensor], NoneType], NoneType] = None"},{name:"callback_steps",val:": typing.Optional[int] = 1"},{name:"cross_attention_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"output_type",val:": typing.Optional[str] = 'np'"}],parametersDescription:[{anchor:"diffusers.AudioLDM2Pipeline.__call__.prompt",description:`<strong>prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts to guide audio generation. If not defined, you need to pass <code>prompt_embeds</code>.`,name:"prompt"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.audio_length_in_s",description:`<strong>audio_length_in_s</strong> (<code>int</code>, <em>optional</em>, defaults to 10.24) &#x2014;
The length of the generated audio sample in seconds.`,name:"audio_length_in_s"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.num_inference_steps",description:`<strong>num_inference_steps</strong> (<code>int</code>, <em>optional</em>, defaults to 200) &#x2014;
The number of denoising steps. More denoising steps usually lead to a higher quality audio at the
expense of slower inference.`,name:"num_inference_steps"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.guidance_scale",description:`<strong>guidance_scale</strong> (<code>float</code>, <em>optional</em>, defaults to 3.5) &#x2014;
A higher guidance scale value encourages the model to generate audio that is closely linked to the text
<code>prompt</code> at the expense of lower sound quality. Guidance scale is enabled when <code>guidance_scale &gt; 1</code>.`,name:"guidance_scale"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.negative_prompt",description:`<strong>negative_prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts to guide what to not include in audio generation. If not defined, you need to
pass <code>negative_prompt_embeds</code> instead. Ignored when not using guidance (<code>guidance_scale &lt; 1</code>).`,name:"negative_prompt"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.num_waveforms_per_prompt",description:`<strong>num_waveforms_per_prompt</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of waveforms to generate per prompt. If <code>num_waveforms_per_prompt &gt; 1</code>, then automatic
scoring is performed between the generated outputs and the text prompt. This scoring ranks the
generated waveforms based on their cosine similarity with the text input in the joint text-audio
embedding space.`,name:"num_waveforms_per_prompt"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.eta",description:`<strong>eta</strong> (<code>float</code>, <em>optional</em>, defaults to 0.0) &#x2014;
Corresponds to parameter eta (&#x3B7;) from the <a href="https://arxiv.org/abs/2010.02502" rel="nofollow">DDIM</a> paper. Only applies
to the <a href="/docs/diffusers/main/en/api/schedulers/ddim#diffusers.DDIMScheduler">DDIMScheduler</a>, and is ignored in other schedulers.`,name:"eta"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.generator",description:`<strong>generator</strong> (<code>torch.Generator</code> or <code>List[torch.Generator]</code>, <em>optional</em>) &#x2014;
A <a href="https://pytorch.org/docs/stable/generated/torch.Generator.html" rel="nofollow"><code>torch.Generator</code></a> to make
generation deterministic.`,name:"generator"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.latents",description:`<strong>latents</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for spectrogram
generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
tensor is generated by sampling using the supplied random <code>generator</code>.`,name:"latents"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.prompt_embeds",description:`<strong>prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
provided, text embeddings are generated from the <code>prompt</code> input argument.`,name:"prompt_embeds"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.negative_prompt_embeds",description:`<strong>negative_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
not provided, <code>negative_prompt_embeds</code> are generated from the <code>negative_prompt</code> input argument.`,name:"negative_prompt_embeds"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.generated_prompt_embeds",description:`<strong>generated_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated text embeddings from the GPT2 langauge model. Can be used to easily tweak text inputs,
<em>e.g.</em> prompt weighting. If not provided, text embeddings will be generated from <code>prompt</code> input
argument.`,name:"generated_prompt_embeds"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.negative_generated_prompt_embeds",description:`<strong>negative_generated_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative text embeddings from the GPT2 language model. Can be used to easily tweak text
inputs, <em>e.g.</em> prompt weighting. If not provided, negative_prompt_embeds will be computed from
<code>negative_prompt</code> input argument.`,name:"negative_generated_prompt_embeds"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.LongTensor</code>, <em>optional</em>) &#x2014;
Pre-computed attention mask to be applied to the <code>prompt_embeds</code>. If not provided, attention mask will
be computed from <code>prompt</code> input argument.`,name:"attention_mask"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.negative_attention_mask",description:`<strong>negative_attention_mask</strong> (<code>torch.LongTensor</code>, <em>optional</em>) &#x2014;
Pre-computed attention mask to be applied to the <code>negative_prompt_embeds</code>. If not provided, attention
mask will be computed from <code>negative_prompt</code> input argument.`,name:"negative_attention_mask"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.max_new_tokens",description:`<strong>max_new_tokens</strong> (<code>int</code>, <em>optional</em>, defaults to None) &#x2014;
Number of new tokens to generate with the GPT2 language model. If not provided, number of tokens will
be taken from the config of the model.`,name:"max_new_tokens"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to return a <a href="/docs/diffusers/main/en/api/pipelines/stable_diffusion/img2img#diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput">StableDiffusionPipelineOutput</a> instead of a
plain tuple.`,name:"return_dict"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.callback",description:`<strong>callback</strong> (<code>Callable</code>, <em>optional</em>) &#x2014;
A function that calls every <code>callback_steps</code> steps during inference. The function is called with the
following arguments: <code>callback(step: int, timestep: int, latents: torch.FloatTensor)</code>.`,name:"callback"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.callback_steps",description:`<strong>callback_steps</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014;
The frequency at which the <code>callback</code> function is called. If not specified, the callback is called at
every step.`,name:"callback_steps"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.cross_attention_kwargs",description:`<strong>cross_attention_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
A kwargs dictionary that if specified is passed along to the <code>AttentionProcessor</code> as defined in
<a href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py" rel="nofollow"><code>self.processor</code></a>.`,name:"cross_attention_kwargs"},{anchor:"diffusers.AudioLDM2Pipeline.__call__.output_type",description:`<strong>output_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;np&quot;</code>) &#x2014;
The output format of the generated audio. Choose between <code>&quot;np&quot;</code> to return a NumPy <code>np.ndarray</code> or
<code>&quot;pt&quot;</code> to return a PyTorch <code>torch.Tensor</code> object. Set to <code>&quot;latent&quot;</code> to return the latent diffusion
model (LDM) output.`,name:"output_type"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py#L732",returnDescription:`
<p>If <code>return_dict</code> is <code>True</code>, <a
  href="/docs/diffusers/main/en/api/pipelines/stable_diffusion/img2img#diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput"
>StableDiffusionPipelineOutput</a> is returned,
otherwise a <code>tuple</code> is returned where the first element is a list with the generated audio.</p>
`,returnType:`
<p><a
  href="/docs/diffusers/main/en/api/pipelines/stable_diffusion/img2img#diffusers.pipelines.stable_diffusion.StableDiffusionPipelineOutput"
>StableDiffusionPipelineOutput</a> or <code>tuple</code></p>
`}}),Te=new al({props:{anchor:"diffusers.AudioLDM2Pipeline.__call__.example",$$slots:{default:[fl]},$$scope:{ctx:me}}}),it=new I({props:{name:"disable_vae_slicing",anchor:"diffusers.AudioLDM2Pipeline.disable_vae_slicing",parameters:[],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py#L185"}}),dt=new I({props:{name:"enable_model_cpu_offload",anchor:"diffusers.AudioLDM2Pipeline.enable_model_cpu_offload",parameters:[{name:"gpu_id",val:" = 0"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py#L192"}}),lt=new I({props:{name:"enable_vae_slicing",anchor:"diffusers.AudioLDM2Pipeline.enable_vae_slicing",parameters:[],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py#L177"}}),ct=new I({props:{name:"encode_prompt",anchor:"diffusers.AudioLDM2Pipeline.encode_prompt",parameters:[{name:"prompt",val:""},{name:"device",val:""},{name:"num_waveforms_per_prompt",val:""},{name:"do_classifier_free_guidance",val:""},{name:"negative_prompt",val:" = None"},{name:"prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"generated_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"negative_generated_prompt_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"negative_attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"max_new_tokens",val:": typing.Optional[int] = None"}],parametersDescription:[{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.prompt",description:`<strong>prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
prompt to be encoded`,name:"prompt"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.device",description:`<strong>device</strong> (<code>torch.device</code>) &#x2014;
torch device`,name:"device"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.num_waveforms_per_prompt",description:`<strong>num_waveforms_per_prompt</strong> (<code>int</code>) &#x2014;
number of waveforms that should be generated per prompt`,name:"num_waveforms_per_prompt"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.do_classifier_free_guidance",description:`<strong>do_classifier_free_guidance</strong> (<code>bool</code>) &#x2014;
whether to use classifier free guidance or not`,name:"do_classifier_free_guidance"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.negative_prompt",description:`<strong>negative_prompt</strong> (<code>str</code> or <code>List[str]</code>, <em>optional</em>) &#x2014;
The prompt or prompts not to guide the audio generation. If not defined, one has to pass
<code>negative_prompt_embeds</code> instead. Ignored when not using guidance (i.e., ignored if <code>guidance_scale</code> is
less than <code>1</code>).`,name:"negative_prompt"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.prompt_embeds",description:`<strong>prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-computed text embeddings from the Flan T5 model. Can be used to easily tweak text inputs, <em>e.g.</em>
prompt weighting. If not provided, text embeddings will be computed from <code>prompt</code> input argument.`,name:"prompt_embeds"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.negative_prompt_embeds",description:`<strong>negative_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-computed negative text embeddings from the Flan T5 model. Can be used to easily tweak text inputs,
<em>e.g.</em> prompt weighting. If not provided, negative_prompt_embeds will be computed from
<code>negative_prompt</code> input argument.`,name:"negative_prompt_embeds"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.generated_prompt_embeds",description:`<strong>generated_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated text embeddings from the GPT2 langauge model. Can be used to easily tweak text inputs,
<em>e.g.</em> prompt weighting. If not provided, text embeddings will be generated from <code>prompt</code> input
argument.`,name:"generated_prompt_embeds"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.negative_generated_prompt_embeds",description:`<strong>negative_generated_prompt_embeds</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
Pre-generated negative text embeddings from the GPT2 language model. Can be used to easily tweak text
inputs, <em>e.g.</em> prompt weighting. If not provided, negative_prompt_embeds will be computed from
<code>negative_prompt</code> input argument.`,name:"negative_generated_prompt_embeds"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.LongTensor</code>, <em>optional</em>) &#x2014;
Pre-computed attention mask to be applied to the <code>prompt_embeds</code>. If not provided, attention mask will
be computed from <code>prompt</code> input argument.`,name:"attention_mask"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.negative_attention_mask",description:`<strong>negative_attention_mask</strong> (<code>torch.LongTensor</code>, <em>optional</em>) &#x2014;
Pre-computed attention mask to be applied to the <code>negative_prompt_embeds</code>. If not provided, attention
mask will be computed from <code>negative_prompt</code> input argument.`,name:"negative_attention_mask"},{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.max_new_tokens",description:`<strong>max_new_tokens</strong> (<code>int</code>, <em>optional</em>, defaults to None) &#x2014;
The number of new tokens to generate with the GPT2 language model.`,name:"max_new_tokens"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py#L270",returnDescription:`
<p>Text embeddings from the Flan T5 model.
attention_mask (<code>torch.LongTensor</code>):
Attention mask to be applied to the <code>prompt_embeds</code>.
generated_prompt_embeds (<code>torch.FloatTensor</code>):
Text embeddings generated from the GPT2 langauge model.</p>
`,returnType:`
<p>prompt_embeds (<code>torch.FloatTensor</code>)</p>
`}}),Ee=new al({props:{anchor:"diffusers.AudioLDM2Pipeline.encode_prompt.example",$$slots:{default:[hl]},$$scope:{ctx:me}}}),pt=new I({props:{name:"generate_language_model",anchor:"diffusers.AudioLDM2Pipeline.generate_language_model",parameters:[{name:"inputs_embeds",val:": Tensor = None"},{name:"max_new_tokens",val:": int = 8"},{name:"**model_kwargs",val:""}],parametersDescription:[{anchor:"diffusers.AudioLDM2Pipeline.generate_language_model.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) &#x2014;
The sequence used as a prompt for the generation.`,name:"inputs_embeds"},{anchor:"diffusers.AudioLDM2Pipeline.generate_language_model.max_new_tokens",description:`<strong>max_new_tokens</strong> (<code>int</code>) &#x2014;
Number of new tokens to generate.`,name:"max_new_tokens"},{anchor:"diffusers.AudioLDM2Pipeline.generate_language_model.model_kwargs",description:`<strong>model_kwargs</strong> (<code>Dict[str, Any]</code>, <em>optional</em>) &#x2014;
Ad hoc parametrization of additional model-specific kwargs that will be forwarded to the <code>forward</code>
function of the model.`,name:"model_kwargs"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py#L229",returnDescription:`
<p>The sequence of generated hidden-states.</p>
`,returnType:`
<p><code>inputs_embeds (</code>torch.FloatTensor<code>of shape</code>(batch_size, sequence_length, hidden_size)\`)</p>
`}}),mt=new V({}),ut=new I({props:{name:"class diffusers.AudioLDM2ProjectionModel",anchor:"diffusers.AudioLDM2ProjectionModel",parameters:[{name:"text_encoder_dim",val:""},{name:"text_encoder_1_dim",val:""},{name:"langauge_model_dim",val:""}],parametersDescription:[{anchor:"diffusers.AudioLDM2ProjectionModel.text_encoder_dim",description:`<strong>text_encoder_dim</strong> (<code>int</code>) &#x2014;
Dimensionality of the text embeddings from the first text encoder (CLAP).`,name:"text_encoder_dim"},{anchor:"diffusers.AudioLDM2ProjectionModel.text_encoder_1_dim",description:`<strong>text_encoder_1_dim</strong> (<code>int</code>) &#x2014;
Dimensionality of the text embeddings from the second text encoder (T5 or VITS).`,name:"text_encoder_1_dim"},{anchor:"diffusers.AudioLDM2ProjectionModel.langauge_model_dim",description:`<strong>langauge_model_dim</strong> (<code>int</code>) &#x2014;
Dimensionality of the text embeddings from the language model (GPT2).`,name:"langauge_model_dim"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py#L76"}}),ht=new I({props:{name:"forward",anchor:"diffusers.AudioLDM2ProjectionModel.forward",parameters:[{name:"hidden_states",val:": typing.Optional[torch.FloatTensor] = None"},{name:"hidden_states_1",val:": typing.Optional[torch.FloatTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.LongTensor] = None"},{name:"attention_mask_1",val:": typing.Optional[torch.LongTensor] = None"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py#L105"}}),gt=new V({}),_t=new I({props:{name:"class diffusers.AudioLDM2UNet2DConditionModel",anchor:"diffusers.AudioLDM2UNet2DConditionModel",parameters:[{name:"sample_size",val:": typing.Optional[int] = None"},{name:"in_channels",val:": int = 4"},{name:"out_channels",val:": int = 4"},{name:"flip_sin_to_cos",val:": bool = True"},{name:"freq_shift",val:": int = 0"},{name:"down_block_types",val:": typing.Tuple[str] = ('CrossAttnDownBlock2D', 'CrossAttnDownBlock2D', 'CrossAttnDownBlock2D', 'DownBlock2D')"},{name:"mid_block_type",val:": typing.Optional[str] = 'UNetMidBlock2DCrossAttn'"},{name:"up_block_types",val:": typing.Tuple[str] = ('UpBlock2D', 'CrossAttnUpBlock2D', 'CrossAttnUpBlock2D', 'CrossAttnUpBlock2D')"},{name:"only_cross_attention",val:": typing.Union[bool, typing.Tuple[bool]] = False"},{name:"block_out_channels",val:": typing.Tuple[int] = (320, 640, 1280, 1280)"},{name:"layers_per_block",val:": typing.Union[int, typing.Tuple[int]] = 2"},{name:"downsample_padding",val:": int = 1"},{name:"mid_block_scale_factor",val:": float = 1"},{name:"act_fn",val:": str = 'silu'"},{name:"norm_num_groups",val:": typing.Optional[int] = 32"},{name:"norm_eps",val:": float = 1e-05"},{name:"cross_attention_dim",val:": typing.Union[int, typing.Tuple[int]] = 1280"},{name:"transformer_layers_per_block",val:": typing.Union[int, typing.Tuple[int]] = 1"},{name:"attention_head_dim",val:": typing.Union[int, typing.Tuple[int]] = 8"},{name:"num_attention_heads",val:": typing.Union[int, typing.Tuple[int], NoneType] = None"},{name:"use_linear_projection",val:": bool = False"},{name:"class_embed_type",val:": typing.Optional[str] = None"},{name:"num_class_embeds",val:": typing.Optional[int] = None"},{name:"upcast_attention",val:": bool = False"},{name:"resnet_time_scale_shift",val:": str = 'default'"},{name:"time_embedding_type",val:": str = 'positional'"},{name:"time_embedding_dim",val:": typing.Optional[int] = None"},{name:"time_embedding_act_fn",val:": typing.Optional[str] = None"},{name:"timestep_post_act",val:": typing.Optional[str] = None"},{name:"time_cond_proj_dim",val:": typing.Optional[int] = None"},{name:"conv_in_kernel",val:": int = 3"},{name:"conv_out_kernel",val:": int = 3"},{name:"projection_class_embeddings_input_dim",val:": typing.Optional[int] = None"},{name:"class_embeddings_concat",val:": bool = False"}],parametersDescription:[{anchor:"diffusers.AudioLDM2UNet2DConditionModel.sample_size",description:`<strong>sample_size</strong> (<code>int</code> or <code>Tuple[int, int]</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Height and width of input/output sample.`,name:"sample_size"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.in_channels",description:"<strong>in_channels</strong> (<code>int</code>, <em>optional</em>, defaults to 4) &#x2014; Number of channels in the input sample.",name:"in_channels"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.out_channels",description:"<strong>out_channels</strong> (<code>int</code>, <em>optional</em>, defaults to 4) &#x2014; Number of channels in the output.",name:"out_channels"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.flip_sin_to_cos",description:`<strong>flip_sin_to_cos</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to flip the sin to cos in the time embedding.`,name:"flip_sin_to_cos"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.freq_shift",description:"<strong>freq_shift</strong> (<code>int</code>, <em>optional</em>, defaults to 0) &#x2014; The frequency shift to apply to the time embedding.",name:"freq_shift"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.down_block_types",description:`<strong>down_block_types</strong> (<code>Tuple[str]</code>, <em>optional</em>, defaults to <code>(&quot;CrossAttnDownBlock2D&quot;, &quot;CrossAttnDownBlock2D&quot;, &quot;CrossAttnDownBlock2D&quot;, &quot;DownBlock2D&quot;)</code>) &#x2014;
The tuple of downsample blocks to use.`,name:"down_block_types"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.mid_block_type",description:`<strong>mid_block_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;UNetMidBlock2DCrossAttn&quot;</code>) &#x2014;
Block type for middle of UNet, it can only be <code>UNetMidBlock2DCrossAttn</code> for AudioLDM2.`,name:"mid_block_type"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.up_block_types",description:`<strong>up_block_types</strong> (<code>Tuple[str]</code>, <em>optional</em>, defaults to <code>(&quot;UpBlock2D&quot;, &quot;CrossAttnUpBlock2D&quot;, &quot;CrossAttnUpBlock2D&quot;, &quot;CrossAttnUpBlock2D&quot;)</code>) &#x2014;
The tuple of upsample blocks to use.`,name:"up_block_types"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.only_cross_attention",description:`<strong>only_cross_attention</strong> (<code>bool</code> or <code>Tuple[bool]</code>, <em>optional</em>, default to <code>False</code>) &#x2014;
Whether to include self-attention in the basic transformer blocks, see
<code>BasicTransformerBlock</code>.`,name:"only_cross_attention"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.block_out_channels",description:`<strong>block_out_channels</strong> (<code>Tuple[int]</code>, <em>optional</em>, defaults to <code>(320, 640, 1280, 1280)</code>) &#x2014;
The tuple of output channels for each block.`,name:"block_out_channels"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.layers_per_block",description:"<strong>layers_per_block</strong> (<code>int</code>, <em>optional</em>, defaults to 2) &#x2014; The number of layers per block.",name:"layers_per_block"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.downsample_padding",description:"<strong>downsample_padding</strong> (<code>int</code>, <em>optional</em>, defaults to 1) &#x2014; The padding to use for the downsampling convolution.",name:"downsample_padding"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.mid_block_scale_factor",description:"<strong>mid_block_scale_factor</strong> (<code>float</code>, <em>optional</em>, defaults to 1.0) &#x2014; The scale factor to use for the mid block.",name:"mid_block_scale_factor"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.act_fn",description:"<strong>act_fn</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;silu&quot;</code>) &#x2014; The activation function to use.",name:"act_fn"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.norm_num_groups",description:`<strong>norm_num_groups</strong> (<code>int</code>, <em>optional</em>, defaults to 32) &#x2014; The number of groups to use for the normalization.
If <code>None</code>, normalization and activation layers is skipped in post-processing.`,name:"norm_num_groups"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.norm_eps",description:"<strong>norm_eps</strong> (<code>float</code>, <em>optional</em>, defaults to 1e-5) &#x2014; The epsilon to use for the normalization.",name:"norm_eps"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.cross_attention_dim",description:`<strong>cross_attention_dim</strong> (<code>int</code> or <code>Tuple[int]</code>, <em>optional</em>, defaults to 1280) &#x2014;
The dimension of the cross attention features.`,name:"cross_attention_dim"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.transformer_layers_per_block",description:`<strong>transformer_layers_per_block</strong> (<code>int</code> or <code>Tuple[int]</code>, <em>optional</em>, defaults to 1) &#x2014;
The number of transformer blocks of type <code>BasicTransformerBlock</code>. Only relevant for
<code>CrossAttnDownBlock2D</code>, <code>CrossAttnUpBlock2D</code>,
<code>UNetMidBlock2DCrossAttn</code>.`,name:"transformer_layers_per_block"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.attention_head_dim",description:"<strong>attention_head_dim</strong> (<code>int</code>, <em>optional</em>, defaults to 8) &#x2014; The dimension of the attention heads.",name:"attention_head_dim"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.num_attention_heads",description:`<strong>num_attention_heads</strong> (<code>int</code>, <em>optional</em>) &#x2014;
The number of attention heads. If not defined, defaults to <code>attention_head_dim</code>`,name:"num_attention_heads"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.resnet_time_scale_shift",description:`<strong>resnet_time_scale_shift</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;default&quot;</code>) &#x2014; Time scale shift config
for ResNet blocks (see <code>ResnetBlock2D</code>). Choose from <code>default</code> or <code>scale_shift</code>.`,name:"resnet_time_scale_shift"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.class_embed_type",description:`<strong>class_embed_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The type of class embedding to use which is ultimately summed with the time embeddings. Choose from <code>None</code>,
<code>&quot;timestep&quot;</code>, <code>&quot;identity&quot;</code>, <code>&quot;projection&quot;</code>, or <code>&quot;simple_projection&quot;</code>.`,name:"class_embed_type"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.num_class_embeds",description:`<strong>num_class_embeds</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Input dimension of the learnable embedding matrix to be projected to <code>time_embed_dim</code>, when performing
class conditioning with <code>class_embed_type</code> equal to <code>None</code>.`,name:"num_class_embeds"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.time_embedding_type",description:`<strong>time_embedding_type</strong> (<code>str</code>, <em>optional</em>, defaults to <code>positional</code>) &#x2014;
The type of position embedding to use for timesteps. Choose from <code>positional</code> or <code>fourier</code>.`,name:"time_embedding_type"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.time_embedding_dim",description:`<strong>time_embedding_dim</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
An optional override for the dimension of the projected time embedding.`,name:"time_embedding_dim"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.time_embedding_act_fn",description:`<strong>time_embedding_act_fn</strong> (<code>str</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Optional activation function to use only once on the time embeddings before they are passed to the rest of
the UNet. Choose from <code>silu</code>, <code>mish</code>, <code>gelu</code>, and <code>swish</code>.`,name:"time_embedding_act_fn"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.timestep_post_act",description:`<strong>timestep_post_act</strong> (<code>str</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The second activation function to use in timestep embedding. Choose from <code>silu</code>, <code>mish</code> and <code>gelu</code>.`,name:"timestep_post_act"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.time_cond_proj_dim",description:`<strong>time_cond_proj_dim</strong> (<code>int</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
The dimension of <code>cond_proj</code> layer in the timestep embedding.`,name:"time_cond_proj_dim"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.conv_in_kernel",description:"<strong>conv_in_kernel</strong> (<code>int</code>, <em>optional</em>, default to <code>3</code>) &#x2014; The kernel size of <code>conv_in</code> layer.",name:"conv_in_kernel"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.conv_out_kernel",description:"<strong>conv_out_kernel</strong> (<code>int</code>, <em>optional</em>, default to <code>3</code>) &#x2014; The kernel size of <code>conv_out</code> layer.",name:"conv_out_kernel"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.projection_class_embeddings_input_dim",description:`<strong>projection_class_embeddings_input_dim</strong> (<code>int</code>, <em>optional</em>) &#x2014; The dimension of the <code>class_labels</code> input when
<code>class_embed_type=&quot;projection&quot;</code>. Required when <code>class_embed_type=&quot;projection&quot;</code>.`,name:"projection_class_embeddings_input_dim"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.class_embeddings_concat",description:`<strong>class_embeddings_concat</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014; Whether to concatenate the time
embeddings with the class embeddings.`,name:"class_embeddings_concat"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py#L142"}}),bt=new I({props:{name:"forward",anchor:"diffusers.AudioLDM2UNet2DConditionModel.forward",parameters:[{name:"sample",val:": FloatTensor"},{name:"timestep",val:": typing.Union[torch.Tensor, float, int]"},{name:"encoder_hidden_states",val:": Tensor"},{name:"class_labels",val:": typing.Optional[torch.Tensor] = None"},{name:"timestep_cond",val:": typing.Optional[torch.Tensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"cross_attention_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"encoder_attention_mask",val:": typing.Optional[torch.Tensor] = None"},{name:"return_dict",val:": bool = True"},{name:"encoder_hidden_states_1",val:": typing.Optional[torch.Tensor] = None"},{name:"encoder_attention_mask_1",val:": typing.Optional[torch.Tensor] = None"}],parametersDescription:[{anchor:"diffusers.AudioLDM2UNet2DConditionModel.forward.sample",description:`<strong>sample</strong> (<code>torch.FloatTensor</code>) &#x2014;
The noisy input tensor with the following shape <code>(batch, channel, height, width)</code>.`,name:"sample"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.forward.timestep",description:"<strong>timestep</strong> (<code>torch.FloatTensor</code> or <code>float</code> or <code>int</code>) &#x2014; The number of timesteps to denoise an input.",name:"timestep"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.forward.encoder_hidden_states",description:`<strong>encoder_hidden_states</strong> (<code>torch.FloatTensor</code>) &#x2014;
The encoder hidden states with shape <code>(batch, sequence_length, feature_dim)</code>.`,name:"encoder_hidden_states"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.forward.encoder_attention_mask",description:`<strong>encoder_attention_mask</strong> (<code>torch.Tensor</code>) &#x2014;
A cross-attention mask of shape <code>(batch, sequence_length)</code> is applied to <code>encoder_hidden_states</code>. If
<code>True</code> the mask is kept, otherwise if <code>False</code> it is discarded. Mask will be converted into a bias,
which adds large negative values to the attention scores corresponding to &#x201C;discard&#x201D; tokens.`,name:"encoder_attention_mask"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not to return a <a href="/docs/diffusers/main/en/api/models/unet2d-cond#diffusers.models.unet_2d_condition.UNet2DConditionOutput">UNet2DConditionOutput</a> instead of a plain
tuple.`,name:"return_dict"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.forward.cross_attention_kwargs",description:`<strong>cross_attention_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
A kwargs dictionary that if specified is passed along to the <code>AttnProcessor</code>.`,name:"cross_attention_kwargs"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.forward.encoder_hidden_states_1",description:`<strong>encoder_hidden_states_1</strong> (<code>torch.FloatTensor</code>, <em>optional</em>) &#x2014;
A second set of encoder hidden states with shape <code>(batch, sequence_length_2, feature_dim_2)</code>. Can be
used to condition the model on a different set of embeddings to <code>encoder_hidden_states</code>.`,name:"encoder_hidden_states_1"},{anchor:"diffusers.AudioLDM2UNet2DConditionModel.forward.encoder_attention_mask_1",description:`<strong>encoder_attention_mask_1</strong> (<code>torch.Tensor</code>, <em>optional</em>) &#x2014;
A cross-attention mask of shape <code>(batch, sequence_length_2)</code> is applied to <code>encoder_hidden_states_1</code>.
If <code>True</code> the mask is kept, otherwise if <code>False</code> it is discarded. Mask will be converted into a bias,
which adds large negative values to the attention scores corresponding to &#x201C;discard&#x201D; tokens.`,name:"encoder_attention_mask_1"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py#L647",returnDescription:`
<p>If <code>return_dict</code> is True, an <a
  href="/docs/diffusers/main/en/api/models/unet2d-cond#diffusers.models.unet_2d_condition.UNet2DConditionOutput"
>UNet2DConditionOutput</a> is returned, otherwise
a <code>tuple</code> is returned where the first element is the sample tensor.</p>
`,returnType:`
<p><a
  href="/docs/diffusers/main/en/api/models/unet2d-cond#diffusers.models.unet_2d_condition.UNet2DConditionOutput"
>UNet2DConditionOutput</a> or <code>tuple</code></p>
`}}),wt=new V({}),yt=new I({props:{name:"class diffusers.AudioPipelineOutput",anchor:"diffusers.AudioPipelineOutput",parameters:[{name:"audios",val:": ndarray"}],parametersDescription:[{anchor:"diffusers.AudioPipelineOutput.audios",description:`<strong>audios</strong> (<code>np.ndarray</code>) &#x2014;
List of denoised audio samples of a NumPy array of shape <code>(batch_size, num_channels, sample_rate)</code>.`,name:"audios"}],source:"https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/pipeline_utils.py#L126"}}),{c(){f=n("meta"),L=d(),y=n("h1"),h=n("a"),x=n("span"),g(u.$$.fragment),A=d(),H=n("span"),Y=i("AudioLDM 2"),$=d(),J=n("p"),Tt=i("AudioLDM 2 was proposed in "),Be=n("a"),na=i("AudioLDM 2: Learning Holistic Audio Generation with Self-supervised Pretraining"),aa=i(`
by Haohe Liu et al. AudioLDM 2 takes a text prompt as input and predicts the corresponding audio. It can generate
text-conditional sound effects, human speech and music.`),nn=d(),D=n("p"),sa=i("Inspired by "),Ie=n("a"),ia=i("Stable Diffusion"),ra=i(`, AudioLDM 2
is a text-to-audio `),Ft=n("em"),da=i("latent diffusion model (LDM)"),la=i(` that learns continuous audio representations from text embeddings. Two
text encoder models are used to compute the text embeddings from a prompt input: the text-branch of `),Je=n("a"),ca=i("CLAP"),pa=i(`
and the encoder of `),qe=n("a"),ma=i("Flan-T5"),ua=i(`. These text embeddings
are then projected to a shared embedding space by an `),Ge=n("a"),fa=i("AudioLDM2ProjectionModel"),ha=i(`.
A `),Se=n("a"),ga=i("GPT2"),_a=d(),Zt=n("em"),va=i("language model (LM)"),ba=i(` is used to auto-regressively
predict eight new embedding vectors, conditional on the projected CLAP and Flan-T5 embeddings. The generated embedding
vectors and Flan-T5 text embeddings are used as cross-attention conditioning in the LDM. The `),Fe=n("a"),Ma=i("UNet"),wa=i(`
of AudioLDM 2 is unique in the sense that it takes `),Ot=n("strong"),ya=i("two"),Da=i(` cross-attention embeddings, as opposed to one cross-attention
conditioning, as in most other LDMs.`),an=d(),At=n("p"),Ta=i("The abstract of the paper is the following:"),sn=d(),kt=n("p"),Wt=n("em"),Aa=i("Although audio generation shares commonalities across different types of audio, such as speech, music, and sound effects, designing models for each type requires careful consideration of specific objectives and biases that can significantly differ from those of other types. To bring us closer to a unified perspective of audio generation, this paper proposes a framework that utilizes the same learning method for speech, music, and sound effect generation. Our framework introduces a general representation of audio, called language of audio (LOA). Any audio can be translated into LOA based on AudioMAE, a self-supervised pre-trained representation learning model. In the generation process, we translate any modalities into LOA by using a GPT-2 model, and we perform self-supervised audio generation learning with a latent diffusion model conditioned on LOA. The proposed framework naturally brings advantages such as in-context learning abilities and reusable self-supervised pretrained AudioMAE and latent diffusion models. Experiments on the major benchmarks of text-to-audio, text-to-music, and text-to-speech demonstrate new state-of-the-art or competitive performance to previous approaches."),rn=d(),S=n("p"),ka=i("This pipeline was contributed by "),Ze=n("a"),xa=i("sanchit-gandhi"),Ea=i(`. The original codebase can be
found at `),Oe=n("a"),La=i("haoheliu/audioldm2"),$a=i("."),dn=d(),K=n("h2"),ue=n("a"),zt=n("span"),g(We.$$.fragment),ja=d(),Qt=n("span"),Ua=i("Tips"),ln=d(),ee=n("h3"),fe=n("a"),Rt=n("span"),g(ze.$$.fragment),Na=d(),Xt=n("span"),Pa=i("Choosing a checkpoint"),cn=d(),xt=n("p"),Ca=i(`AudioLDM2 comes in three variants. Two of these checkpoints are applicable to the general task of text-to-audio
generation. The third checkpoint is trained exclusively on text-to-music generation.`),pn=d(),Et=n("p"),Ba=i(`All checkpoints share the same model size for the text encoders and VAE. They differ in the size and depth of the UNet.
See table below for details on the three checkpoints:`),mn=d(),he=n("table"),Vt=n("thead"),j=n("tr"),Ht=n("th"),Ia=i("Checkpoint"),Ja=d(),Yt=n("th"),qa=i("Task"),Ga=d(),Kt=n("th"),Sa=i("UNet Model Size"),Fa=d(),eo=n("th"),Za=i("Total Model Size"),Oa=d(),to=n("th"),Wa=i("Training Data / h"),za=d(),te=n("tbody"),U=n("tr"),oo=n("td"),Qe=n("a"),Qa=i("audioldm2"),Ra=d(),no=n("td"),Xa=i("Text-to-audio"),Va=d(),ao=n("td"),Ha=i("350M"),Ya=d(),so=n("td"),Ka=i("1.1B"),es=d(),io=n("td"),ts=i("1150k"),os=d(),N=n("tr"),ro=n("td"),Re=n("a"),ns=i("audioldm2-large"),as=d(),lo=n("td"),ss=i("Text-to-audio"),is=d(),co=n("td"),rs=i("750M"),ds=d(),po=n("td"),ls=i("1.5B"),cs=d(),mo=n("td"),ps=i("1150k"),ms=d(),P=n("tr"),uo=n("td"),Xe=n("a"),us=i("audioldm2-music"),fs=d(),fo=n("td"),hs=i("Text-to-music"),gs=d(),ho=n("td"),_s=i("350M"),vs=d(),go=n("td"),bs=i("1.1B"),Ms=d(),_o=n("td"),ws=i("665k"),un=d(),oe=n("h3"),ge=n("a"),vo=n("span"),g(Ve.$$.fragment),ys=d(),bo=n("span"),Ds=i("Constructing a prompt"),fn=d(),F=n("ul"),Mo=n("li"),Ts=i("Descriptive prompt inputs work best: use adjectives to describe the sound (e.g. \u201Chigh quality\u201D or \u201Cclear\u201D) and make the prompt context specific (e.g. \u201Cwater stream in a forest\u201D instead of \u201Cstream\u201D)."),As=d(),wo=n("li"),ks=i("It\u2019s best to use general terms like \u201Ccat\u201D or \u201Cdog\u201D instead of specific names or abstract objects the model may not be familiar with."),xs=d(),He=n("li"),Es=i("Using a "),yo=n("strong"),Ls=i("negative prompt"),$s=i(" can significantly improve the quality of the generated waveform, by guiding the generation away from terms that correspond to poor quality audio. Try using a negative prompt of \u201CLow quality.\u201D"),hn=d(),ne=n("h3"),_e=n("a"),Do=n("span"),g(Ye.$$.fragment),js=d(),To=n("span"),Us=i("Controlling inference"),gn=d(),ve=n("ul"),ae=n("li"),Ns=i("The "),Ao=n("em"),Ps=i("quality"),Cs=i(" of the predicted audio sample can be controlled by the "),ko=n("code"),Bs=i("num_inference_steps"),Is=i(" argument; higher steps give higher quality audio at the expense of slower inference."),Js=d(),se=n("li"),qs=i("The "),xo=n("em"),Gs=i("length"),Ss=i(" of the predicted audio sample can be controlled by varying the "),Eo=n("code"),Fs=i("audio_length_in_s"),Zs=i(" argument."),_n=d(),ie=n("h3"),be=n("a"),Lo=n("span"),g(Ke.$$.fragment),Os=d(),$o=n("span"),Ws=i("Evaluating generated waveforms:"),vn=d(),Me=n("ul"),jo=n("li"),zs=i("The quality of the generated waveforms can vary significantly based on the seed. Try generating with different seeds until you find a satisfactory generation"),Qs=d(),et=n("li"),Rs=i("Multiple waveforms can be generated in one go: set "),Uo=n("code"),Xs=i("num_waveforms_per_prompt"),Vs=i(" to a value greater than 1. Automatic scoring will be performed between the generated waveforms and prompt text, and the audios ranked from best to worst accordingly."),bn=d(),we=n("p"),Hs=i("The following example demonstrates how to construct good music generation using the aforementioned tips: "),tt=n("a"),Ys=i("example"),Ks=i("."),Mn=d(),g(ye.$$.fragment),wn=d(),re=n("h2"),De=n("a"),No=n("span"),g(ot.$$.fragment),ei=d(),Po=n("span"),ti=i("AudioLDM2Pipeline"),yn=d(),T=n("div"),g(nt.$$.fragment),oi=d(),Co=n("p"),ni=i("Pipeline for text-to-audio generation using AudioLDM2."),ai=d(),at=n("p"),si=i("This model inherits from "),Lt=n("a"),ii=i("DiffusionPipeline"),ri=i(`. Check the superclass documentation for the generic methods
implemented for all pipelines (downloading, saving, running on a particular device, etc.).`),di=d(),Z=n("div"),g(st.$$.fragment),li=d(),Bo=n("p"),ci=i("The call function to the pipeline for generation."),pi=d(),g(Te.$$.fragment),mi=d(),Ae=n("div"),g(it.$$.fragment),ui=d(),rt=n("p"),fi=i("Disable sliced VAE decoding. If "),Io=n("code"),hi=i("enable_vae_slicing"),gi=i(` was previously enabled, this method will go back to
computing decoding in one step.`),_i=d(),ke=n("div"),g(dt.$$.fragment),vi=d(),C=n("p"),bi=i(`Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared
to `),Jo=n("code"),Mi=i("enable_sequential_cpu_offload"),wi=i(", this method moves one whole model at a time to the GPU when its "),qo=n("code"),yi=i("forward"),Di=i(`
method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with
`),Go=n("code"),Ti=i("enable_sequential_cpu_offload"),Ai=i(", but performance is much better due to the iterative execution of the "),So=n("code"),ki=i("unet"),xi=i("."),Ei=d(),xe=n("div"),g(lt.$$.fragment),Li=d(),Fo=n("p"),$i=i(`Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.`),ji=d(),O=n("div"),g(ct.$$.fragment),Ui=d(),Zo=n("p"),Ni=i("Encodes the prompt into text encoder hidden states."),Pi=d(),g(Ee.$$.fragment),Ci=d(),Le=n("div"),g(pt.$$.fragment),Bi=d(),Oo=n("p"),Ii=i("Generates a sequence of hidden-states from the language model, conditioned on the embedding inputs."),Dn=d(),de=n("h2"),$e=n("a"),Wo=n("span"),g(mt.$$.fragment),Ji=d(),zo=n("span"),qi=i("AudioLDM2ProjectionModel"),Tn=d(),q=n("div"),g(ut.$$.fragment),Gi=d(),ft=n("p"),Si=i(`A simple linear projection model to map two text embeddings to a shared latent space. It also inserts learned
embedding vectors at the start and end of each text embedding sequence respectively. Each variable appended with
`),Qo=n("code"),Fi=i("_1"),Zi=i(" refers to that corresponding to the second text encoder. Otherwise, it is from the first."),Oi=d(),$t=n("div"),g(ht.$$.fragment),An=d(),le=n("h2"),je=n("a"),Ro=n("span"),g(gt.$$.fragment),Wi=d(),Xo=n("span"),zi=i("AudioLDM2UNet2DConditionModel"),kn=d(),B=n("div"),g(_t.$$.fragment),Qi=d(),G=n("p"),Ri=i(`A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample
shaped output. Compared to the vanilla `),jt=n("a"),Xi=i("UNet2DConditionModel"),Vi=i(`, this variant optionally includes an additional
self-attention layer in each Transformer block, as well as multiple cross-attention layers. It also allows for up
to two cross-attention embeddings, `),Vo=n("code"),Hi=i("encoder_hidden_states"),Yi=i(" and "),Ho=n("code"),Ki=i("encoder_hidden_states_1"),er=i("."),tr=d(),vt=n("p"),or=i("This model inherits from "),Ut=n("a"),nr=i("ModelMixin"),ar=i(`. Check the superclass documentation for it\u2019s generic methods implemented
for all models (such as downloading or saving).`),sr=d(),Ue=n("div"),g(bt.$$.fragment),ir=d(),Mt=n("p"),rr=i("The "),Nt=n("a"),dr=i("AudioLDM2UNet2DConditionModel"),lr=i(" forward method."),xn=d(),ce=n("h2"),Ne=n("a"),Yo=n("span"),g(wt.$$.fragment),cr=d(),Ko=n("span"),pr=i("AudioPipelineOutput"),En=d(),pe=n("div"),g(yt.$$.fragment),mr=d(),en=n("p"),ur=i("Output class for audio pipelines."),this.h()},l(t){const p=cl('[data-svelte="svelte-1phssyn"]',document.head);f=a(p,"META",{name:!0,content:!0}),p.forEach(o),L=l(t),y=a(t,"H1",{class:!0});var Dt=s(y);h=a(Dt,"A",{id:!0,class:!0,href:!0});var tn=s(h);x=a(tn,"SPAN",{});var on=s(x);_(u.$$.fragment,on),on.forEach(o),tn.forEach(o),A=l(Dt),H=a(Dt,"SPAN",{});var fr=s(H);Y=r(fr,"AudioLDM 2"),fr.forEach(o),Dt.forEach(o),$=l(t),J=a(t,"P",{});var $n=s(J);Tt=r($n,"AudioLDM 2 was proposed in "),Be=a($n,"A",{href:!0,rel:!0});var hr=s(Be);na=r(hr,"AudioLDM 2: Learning Holistic Audio Generation with Self-supervised Pretraining"),hr.forEach(o),aa=r($n,`
by Haohe Liu et al. AudioLDM 2 takes a text prompt as input and predicts the corresponding audio. It can generate
text-conditional sound effects, human speech and music.`),$n.forEach(o),nn=l(t),D=a(t,"P",{});var k=s(D);sa=r(k,"Inspired by "),Ie=a(k,"A",{href:!0,rel:!0});var gr=s(Ie);ia=r(gr,"Stable Diffusion"),gr.forEach(o),ra=r(k,`, AudioLDM 2
is a text-to-audio `),Ft=a(k,"EM",{});var _r=s(Ft);da=r(_r,"latent diffusion model (LDM)"),_r.forEach(o),la=r(k,` that learns continuous audio representations from text embeddings. Two
text encoder models are used to compute the text embeddings from a prompt input: the text-branch of `),Je=a(k,"A",{href:!0,rel:!0});var vr=s(Je);ca=r(vr,"CLAP"),vr.forEach(o),pa=r(k,`
and the encoder of `),qe=a(k,"A",{href:!0,rel:!0});var br=s(qe);ma=r(br,"Flan-T5"),br.forEach(o),ua=r(k,`. These text embeddings
are then projected to a shared embedding space by an `),Ge=a(k,"A",{href:!0,rel:!0});var Mr=s(Ge);fa=r(Mr,"AudioLDM2ProjectionModel"),Mr.forEach(o),ha=r(k,`.
A `),Se=a(k,"A",{href:!0,rel:!0});var wr=s(Se);ga=r(wr,"GPT2"),wr.forEach(o),_a=l(k),Zt=a(k,"EM",{});var yr=s(Zt);va=r(yr,"language model (LM)"),yr.forEach(o),ba=r(k,` is used to auto-regressively
predict eight new embedding vectors, conditional on the projected CLAP and Flan-T5 embeddings. The generated embedding
vectors and Flan-T5 text embeddings are used as cross-attention conditioning in the LDM. The `),Fe=a(k,"A",{href:!0,rel:!0});var Dr=s(Fe);Ma=r(Dr,"UNet"),Dr.forEach(o),wa=r(k,`
of AudioLDM 2 is unique in the sense that it takes `),Ot=a(k,"STRONG",{});var Tr=s(Ot);ya=r(Tr,"two"),Tr.forEach(o),Da=r(k,` cross-attention embeddings, as opposed to one cross-attention
conditioning, as in most other LDMs.`),k.forEach(o),an=l(t),At=a(t,"P",{});var Ar=s(At);Ta=r(Ar,"The abstract of the paper is the following:"),Ar.forEach(o),sn=l(t),kt=a(t,"P",{});var kr=s(kt);Wt=a(kr,"EM",{});var xr=s(Wt);Aa=r(xr,"Although audio generation shares commonalities across different types of audio, such as speech, music, and sound effects, designing models for each type requires careful consideration of specific objectives and biases that can significantly differ from those of other types. To bring us closer to a unified perspective of audio generation, this paper proposes a framework that utilizes the same learning method for speech, music, and sound effect generation. Our framework introduces a general representation of audio, called language of audio (LOA). Any audio can be translated into LOA based on AudioMAE, a self-supervised pre-trained representation learning model. In the generation process, we translate any modalities into LOA by using a GPT-2 model, and we perform self-supervised audio generation learning with a latent diffusion model conditioned on LOA. The proposed framework naturally brings advantages such as in-context learning abilities and reusable self-supervised pretrained AudioMAE and latent diffusion models. Experiments on the major benchmarks of text-to-audio, text-to-music, and text-to-speech demonstrate new state-of-the-art or competitive performance to previous approaches."),xr.forEach(o),kr.forEach(o),rn=l(t),S=a(t,"P",{});var Pt=s(S);ka=r(Pt,"This pipeline was contributed by "),Ze=a(Pt,"A",{href:!0,rel:!0});var Er=s(Ze);xa=r(Er,"sanchit-gandhi"),Er.forEach(o),Ea=r(Pt,`. The original codebase can be
found at `),Oe=a(Pt,"A",{href:!0,rel:!0});var Lr=s(Oe);La=r(Lr,"haoheliu/audioldm2"),Lr.forEach(o),$a=r(Pt,"."),Pt.forEach(o),dn=l(t),K=a(t,"H2",{class:!0});var jn=s(K);ue=a(jn,"A",{id:!0,class:!0,href:!0});var $r=s(ue);zt=a($r,"SPAN",{});var jr=s(zt);_(We.$$.fragment,jr),jr.forEach(o),$r.forEach(o),ja=l(jn),Qt=a(jn,"SPAN",{});var Ur=s(Qt);Ua=r(Ur,"Tips"),Ur.forEach(o),jn.forEach(o),ln=l(t),ee=a(t,"H3",{class:!0});var Un=s(ee);fe=a(Un,"A",{id:!0,class:!0,href:!0});var Nr=s(fe);Rt=a(Nr,"SPAN",{});var Pr=s(Rt);_(ze.$$.fragment,Pr),Pr.forEach(o),Nr.forEach(o),Na=l(Un),Xt=a(Un,"SPAN",{});var Cr=s(Xt);Pa=r(Cr,"Choosing a checkpoint"),Cr.forEach(o),Un.forEach(o),cn=l(t),xt=a(t,"P",{});var Br=s(xt);Ca=r(Br,`AudioLDM2 comes in three variants. Two of these checkpoints are applicable to the general task of text-to-audio
generation. The third checkpoint is trained exclusively on text-to-music generation.`),Br.forEach(o),pn=l(t),Et=a(t,"P",{});var Ir=s(Et);Ba=r(Ir,`All checkpoints share the same model size for the text encoders and VAE. They differ in the size and depth of the UNet.
See table below for details on the three checkpoints:`),Ir.forEach(o),mn=l(t),he=a(t,"TABLE",{});var Nn=s(he);Vt=a(Nn,"THEAD",{});var Jr=s(Vt);j=a(Jr,"TR",{});var W=s(j);Ht=a(W,"TH",{});var qr=s(Ht);Ia=r(qr,"Checkpoint"),qr.forEach(o),Ja=l(W),Yt=a(W,"TH",{});var Gr=s(Yt);qa=r(Gr,"Task"),Gr.forEach(o),Ga=l(W),Kt=a(W,"TH",{});var Sr=s(Kt);Sa=r(Sr,"UNet Model Size"),Sr.forEach(o),Fa=l(W),eo=a(W,"TH",{});var Fr=s(eo);Za=r(Fr,"Total Model Size"),Fr.forEach(o),Oa=l(W),to=a(W,"TH",{});var Zr=s(to);Wa=r(Zr,"Training Data / h"),Zr.forEach(o),W.forEach(o),Jr.forEach(o),za=l(Nn),te=a(Nn,"TBODY",{});var Ct=s(te);U=a(Ct,"TR",{});var z=s(U);oo=a(z,"TD",{});var Or=s(oo);Qe=a(Or,"A",{href:!0,rel:!0});var Wr=s(Qe);Qa=r(Wr,"audioldm2"),Wr.forEach(o),Or.forEach(o),Ra=l(z),no=a(z,"TD",{});var zr=s(no);Xa=r(zr,"Text-to-audio"),zr.forEach(o),Va=l(z),ao=a(z,"TD",{});var Qr=s(ao);Ha=r(Qr,"350M"),Qr.forEach(o),Ya=l(z),so=a(z,"TD",{});var Rr=s(so);Ka=r(Rr,"1.1B"),Rr.forEach(o),es=l(z),io=a(z,"TD",{});var Xr=s(io);ts=r(Xr,"1150k"),Xr.forEach(o),z.forEach(o),os=l(Ct),N=a(Ct,"TR",{});var Q=s(N);ro=a(Q,"TD",{});var Vr=s(ro);Re=a(Vr,"A",{href:!0,rel:!0});var Hr=s(Re);ns=r(Hr,"audioldm2-large"),Hr.forEach(o),Vr.forEach(o),as=l(Q),lo=a(Q,"TD",{});var Yr=s(lo);ss=r(Yr,"Text-to-audio"),Yr.forEach(o),is=l(Q),co=a(Q,"TD",{});var Kr=s(co);rs=r(Kr,"750M"),Kr.forEach(o),ds=l(Q),po=a(Q,"TD",{});var ed=s(po);ls=r(ed,"1.5B"),ed.forEach(o),cs=l(Q),mo=a(Q,"TD",{});var td=s(mo);ps=r(td,"1150k"),td.forEach(o),Q.forEach(o),ms=l(Ct),P=a(Ct,"TR",{});var R=s(P);uo=a(R,"TD",{});var od=s(uo);Xe=a(od,"A",{href:!0,rel:!0});var nd=s(Xe);us=r(nd,"audioldm2-music"),nd.forEach(o),od.forEach(o),fs=l(R),fo=a(R,"TD",{});var ad=s(fo);hs=r(ad,"Text-to-music"),ad.forEach(o),gs=l(R),ho=a(R,"TD",{});var sd=s(ho);_s=r(sd,"350M"),sd.forEach(o),vs=l(R),go=a(R,"TD",{});var id=s(go);bs=r(id,"1.1B"),id.forEach(o),Ms=l(R),_o=a(R,"TD",{});var rd=s(_o);ws=r(rd,"665k"),rd.forEach(o),R.forEach(o),Ct.forEach(o),Nn.forEach(o),un=l(t),oe=a(t,"H3",{class:!0});var Pn=s(oe);ge=a(Pn,"A",{id:!0,class:!0,href:!0});var dd=s(ge);vo=a(dd,"SPAN",{});var ld=s(vo);_(Ve.$$.fragment,ld),ld.forEach(o),dd.forEach(o),ys=l(Pn),bo=a(Pn,"SPAN",{});var cd=s(bo);Ds=r(cd,"Constructing a prompt"),cd.forEach(o),Pn.forEach(o),fn=l(t),F=a(t,"UL",{});var Bt=s(F);Mo=a(Bt,"LI",{});var pd=s(Mo);Ts=r(pd,"Descriptive prompt inputs work best: use adjectives to describe the sound (e.g. \u201Chigh quality\u201D or \u201Cclear\u201D) and make the prompt context specific (e.g. \u201Cwater stream in a forest\u201D instead of \u201Cstream\u201D)."),pd.forEach(o),As=l(Bt),wo=a(Bt,"LI",{});var md=s(wo);ks=r(md,"It\u2019s best to use general terms like \u201Ccat\u201D or \u201Cdog\u201D instead of specific names or abstract objects the model may not be familiar with."),md.forEach(o),xs=l(Bt),He=a(Bt,"LI",{});var Cn=s(He);Es=r(Cn,"Using a "),yo=a(Cn,"STRONG",{});var ud=s(yo);Ls=r(ud,"negative prompt"),ud.forEach(o),$s=r(Cn," can significantly improve the quality of the generated waveform, by guiding the generation away from terms that correspond to poor quality audio. Try using a negative prompt of \u201CLow quality.\u201D"),Cn.forEach(o),Bt.forEach(o),hn=l(t),ne=a(t,"H3",{class:!0});var Bn=s(ne);_e=a(Bn,"A",{id:!0,class:!0,href:!0});var fd=s(_e);Do=a(fd,"SPAN",{});var hd=s(Do);_(Ye.$$.fragment,hd),hd.forEach(o),fd.forEach(o),js=l(Bn),To=a(Bn,"SPAN",{});var gd=s(To);Us=r(gd,"Controlling inference"),gd.forEach(o),Bn.forEach(o),gn=l(t),ve=a(t,"UL",{});var In=s(ve);ae=a(In,"LI",{});var It=s(ae);Ns=r(It,"The "),Ao=a(It,"EM",{});var _d=s(Ao);Ps=r(_d,"quality"),_d.forEach(o),Cs=r(It," of the predicted audio sample can be controlled by the "),ko=a(It,"CODE",{});var vd=s(ko);Bs=r(vd,"num_inference_steps"),vd.forEach(o),Is=r(It," argument; higher steps give higher quality audio at the expense of slower inference."),It.forEach(o),Js=l(In),se=a(In,"LI",{});var Jt=s(se);qs=r(Jt,"The "),xo=a(Jt,"EM",{});var bd=s(xo);Gs=r(bd,"length"),bd.forEach(o),Ss=r(Jt," of the predicted audio sample can be controlled by varying the "),Eo=a(Jt,"CODE",{});var Md=s(Eo);Fs=r(Md,"audio_length_in_s"),Md.forEach(o),Zs=r(Jt," argument."),Jt.forEach(o),In.forEach(o),_n=l(t),ie=a(t,"H3",{class:!0});var Jn=s(ie);be=a(Jn,"A",{id:!0,class:!0,href:!0});var wd=s(be);Lo=a(wd,"SPAN",{});var yd=s(Lo);_(Ke.$$.fragment,yd),yd.forEach(o),wd.forEach(o),Os=l(Jn),$o=a(Jn,"SPAN",{});var Dd=s($o);Ws=r(Dd,"Evaluating generated waveforms:"),Dd.forEach(o),Jn.forEach(o),vn=l(t),Me=a(t,"UL",{});var qn=s(Me);jo=a(qn,"LI",{});var Td=s(jo);zs=r(Td,"The quality of the generated waveforms can vary significantly based on the seed. Try generating with different seeds until you find a satisfactory generation"),Td.forEach(o),Qs=l(qn),et=a(qn,"LI",{});var Gn=s(et);Rs=r(Gn,"Multiple waveforms can be generated in one go: set "),Uo=a(Gn,"CODE",{});var Ad=s(Uo);Xs=r(Ad,"num_waveforms_per_prompt"),Ad.forEach(o),Vs=r(Gn," to a value greater than 1. Automatic scoring will be performed between the generated waveforms and prompt text, and the audios ranked from best to worst accordingly."),Gn.forEach(o),qn.forEach(o),bn=l(t),we=a(t,"P",{});var Sn=s(we);Hs=r(Sn,"The following example demonstrates how to construct good music generation using the aforementioned tips: "),tt=a(Sn,"A",{href:!0,rel:!0});var kd=s(tt);Ys=r(kd,"example"),kd.forEach(o),Ks=r(Sn,"."),Sn.forEach(o),Mn=l(t),_(ye.$$.fragment,t),wn=l(t),re=a(t,"H2",{class:!0});var Fn=s(re);De=a(Fn,"A",{id:!0,class:!0,href:!0});var xd=s(De);No=a(xd,"SPAN",{});var Ed=s(No);_(ot.$$.fragment,Ed),Ed.forEach(o),xd.forEach(o),ei=l(Fn),Po=a(Fn,"SPAN",{});var Ld=s(Po);ti=r(Ld,"AudioLDM2Pipeline"),Ld.forEach(o),Fn.forEach(o),yn=l(t),T=a(t,"DIV",{class:!0});var E=s(T);_(nt.$$.fragment,E),oi=l(E),Co=a(E,"P",{});var $d=s(Co);ni=r($d,"Pipeline for text-to-audio generation using AudioLDM2."),$d.forEach(o),ai=l(E),at=a(E,"P",{});var Zn=s(at);si=r(Zn,"This model inherits from "),Lt=a(Zn,"A",{href:!0});var jd=s(Lt);ii=r(jd,"DiffusionPipeline"),jd.forEach(o),ri=r(Zn,`. Check the superclass documentation for the generic methods
implemented for all pipelines (downloading, saving, running on a particular device, etc.).`),Zn.forEach(o),di=l(E),Z=a(E,"DIV",{class:!0});var qt=s(Z);_(st.$$.fragment,qt),li=l(qt),Bo=a(qt,"P",{});var Ud=s(Bo);ci=r(Ud,"The call function to the pipeline for generation."),Ud.forEach(o),pi=l(qt),_(Te.$$.fragment,qt),qt.forEach(o),mi=l(E),Ae=a(E,"DIV",{class:!0});var On=s(Ae);_(it.$$.fragment,On),ui=l(On),rt=a(On,"P",{});var Wn=s(rt);fi=r(Wn,"Disable sliced VAE decoding. If "),Io=a(Wn,"CODE",{});var Nd=s(Io);hi=r(Nd,"enable_vae_slicing"),Nd.forEach(o),gi=r(Wn,` was previously enabled, this method will go back to
computing decoding in one step.`),Wn.forEach(o),On.forEach(o),_i=l(E),ke=a(E,"DIV",{class:!0});var zn=s(ke);_(dt.$$.fragment,zn),vi=l(zn),C=a(zn,"P",{});var X=s(C);bi=r(X,`Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared
to `),Jo=a(X,"CODE",{});var Pd=s(Jo);Mi=r(Pd,"enable_sequential_cpu_offload"),Pd.forEach(o),wi=r(X,", this method moves one whole model at a time to the GPU when its "),qo=a(X,"CODE",{});var Cd=s(qo);yi=r(Cd,"forward"),Cd.forEach(o),Di=r(X,`
method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with
`),Go=a(X,"CODE",{});var Bd=s(Go);Ti=r(Bd,"enable_sequential_cpu_offload"),Bd.forEach(o),Ai=r(X,", but performance is much better due to the iterative execution of the "),So=a(X,"CODE",{});var Id=s(So);ki=r(Id,"unet"),Id.forEach(o),xi=r(X,"."),X.forEach(o),zn.forEach(o),Ei=l(E),xe=a(E,"DIV",{class:!0});var Qn=s(xe);_(lt.$$.fragment,Qn),Li=l(Qn),Fo=a(Qn,"P",{});var Jd=s(Fo);$i=r(Jd,`Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.`),Jd.forEach(o),Qn.forEach(o),ji=l(E),O=a(E,"DIV",{class:!0});var Gt=s(O);_(ct.$$.fragment,Gt),Ui=l(Gt),Zo=a(Gt,"P",{});var qd=s(Zo);Ni=r(qd,"Encodes the prompt into text encoder hidden states."),qd.forEach(o),Pi=l(Gt),_(Ee.$$.fragment,Gt),Gt.forEach(o),Ci=l(E),Le=a(E,"DIV",{class:!0});var Rn=s(Le);_(pt.$$.fragment,Rn),Bi=l(Rn),Oo=a(Rn,"P",{});var Gd=s(Oo);Ii=r(Gd,"Generates a sequence of hidden-states from the language model, conditioned on the embedding inputs."),Gd.forEach(o),Rn.forEach(o),E.forEach(o),Dn=l(t),de=a(t,"H2",{class:!0});var Xn=s(de);$e=a(Xn,"A",{id:!0,class:!0,href:!0});var Sd=s($e);Wo=a(Sd,"SPAN",{});var Fd=s(Wo);_(mt.$$.fragment,Fd),Fd.forEach(o),Sd.forEach(o),Ji=l(Xn),zo=a(Xn,"SPAN",{});var Zd=s(zo);qi=r(Zd,"AudioLDM2ProjectionModel"),Zd.forEach(o),Xn.forEach(o),Tn=l(t),q=a(t,"DIV",{class:!0});var St=s(q);_(ut.$$.fragment,St),Gi=l(St),ft=a(St,"P",{});var Vn=s(ft);Si=r(Vn,`A simple linear projection model to map two text embeddings to a shared latent space. It also inserts learned
embedding vectors at the start and end of each text embedding sequence respectively. Each variable appended with
`),Qo=a(Vn,"CODE",{});var Od=s(Qo);Fi=r(Od,"_1"),Od.forEach(o),Zi=r(Vn," refers to that corresponding to the second text encoder. Otherwise, it is from the first."),Vn.forEach(o),Oi=l(St),$t=a(St,"DIV",{class:!0});var Wd=s($t);_(ht.$$.fragment,Wd),Wd.forEach(o),St.forEach(o),An=l(t),le=a(t,"H2",{class:!0});var Hn=s(le);je=a(Hn,"A",{id:!0,class:!0,href:!0});var zd=s(je);Ro=a(zd,"SPAN",{});var Qd=s(Ro);_(gt.$$.fragment,Qd),Qd.forEach(o),zd.forEach(o),Wi=l(Hn),Xo=a(Hn,"SPAN",{});var Rd=s(Xo);zi=r(Rd,"AudioLDM2UNet2DConditionModel"),Rd.forEach(o),Hn.forEach(o),kn=l(t),B=a(t,"DIV",{class:!0});var Pe=s(B);_(_t.$$.fragment,Pe),Qi=l(Pe),G=a(Pe,"P",{});var Ce=s(G);Ri=r(Ce,`A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample
shaped output. Compared to the vanilla `),jt=a(Ce,"A",{href:!0});var Xd=s(jt);Xi=r(Xd,"UNet2DConditionModel"),Xd.forEach(o),Vi=r(Ce,`, this variant optionally includes an additional
self-attention layer in each Transformer block, as well as multiple cross-attention layers. It also allows for up
to two cross-attention embeddings, `),Vo=a(Ce,"CODE",{});var Vd=s(Vo);Hi=r(Vd,"encoder_hidden_states"),Vd.forEach(o),Yi=r(Ce," and "),Ho=a(Ce,"CODE",{});var Hd=s(Ho);Ki=r(Hd,"encoder_hidden_states_1"),Hd.forEach(o),er=r(Ce,"."),Ce.forEach(o),tr=l(Pe),vt=a(Pe,"P",{});var Yn=s(vt);or=r(Yn,"This model inherits from "),Ut=a(Yn,"A",{href:!0});var Yd=s(Ut);nr=r(Yd,"ModelMixin"),Yd.forEach(o),ar=r(Yn,`. Check the superclass documentation for it\u2019s generic methods implemented
for all models (such as downloading or saving).`),Yn.forEach(o),sr=l(Pe),Ue=a(Pe,"DIV",{class:!0});var Kn=s(Ue);_(bt.$$.fragment,Kn),ir=l(Kn),Mt=a(Kn,"P",{});var ea=s(Mt);rr=r(ea,"The "),Nt=a(ea,"A",{href:!0});var Kd=s(Nt);dr=r(Kd,"AudioLDM2UNet2DConditionModel"),Kd.forEach(o),lr=r(ea," forward method."),ea.forEach(o),Kn.forEach(o),Pe.forEach(o),xn=l(t),ce=a(t,"H2",{class:!0});var ta=s(ce);Ne=a(ta,"A",{id:!0,class:!0,href:!0});var el=s(Ne);Yo=a(el,"SPAN",{});var tl=s(Yo);_(wt.$$.fragment,tl),tl.forEach(o),el.forEach(o),cr=l(ta),Ko=a(ta,"SPAN",{});var ol=s(Ko);pr=r(ol,"AudioPipelineOutput"),ol.forEach(o),ta.forEach(o),En=l(t),pe=a(t,"DIV",{class:!0});var oa=s(pe);_(yt.$$.fragment,oa),mr=l(oa),en=a(oa,"P",{});var nl=s(en);ur=r(nl,"Output class for audio pipelines."),nl.forEach(o),oa.forEach(o),this.h()},h(){c(f,"name","hf:doc:metadata"),c(f,"content",JSON.stringify(_l)),c(h,"id","audioldm-2"),c(h,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(h,"href","#audioldm-2"),c(y,"class","relative group"),c(Be,"href","https://arxiv.org/abs/2308.05734"),c(Be,"rel","nofollow"),c(Ie,"href","https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/overview"),c(Ie,"rel","nofollow"),c(Je,"href","https://huggingface.co/docs/transformers/main/en/model_doc/clap"),c(Je,"rel","nofollow"),c(qe,"href","https://huggingface.co/docs/transformers/main/en/model_doc/flan-t5"),c(qe,"rel","nofollow"),c(Ge,"href","https://huggingface.co/docs/diffusers/main/api/pipelines/audioldm2#diffusers.AudioLDM2ProjectionModel"),c(Ge,"rel","nofollow"),c(Se,"href","https://huggingface.co/docs/transformers/main/en/model_doc/gpt2"),c(Se,"rel","nofollow"),c(Fe,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/audioldm2#diffusers.AudioLDM2UNet2DConditionModel"),c(Fe,"rel","nofollow"),c(Ze,"href","https://huggingface.co/sanchit-gandhi"),c(Ze,"rel","nofollow"),c(Oe,"href","https://github.com/haoheliu/audioldm2"),c(Oe,"rel","nofollow"),c(ue,"id","tips"),c(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ue,"href","#tips"),c(K,"class","relative group"),c(fe,"id","choosing-a-checkpoint"),c(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(fe,"href","#choosing-a-checkpoint"),c(ee,"class","relative group"),c(Qe,"href","https://huggingface.co/cvssp/audioldm2"),c(Qe,"rel","nofollow"),c(Re,"href","https://huggingface.co/cvssp/audioldm2-large"),c(Re,"rel","nofollow"),c(Xe,"href","https://huggingface.co/cvssp/audioldm2-music"),c(Xe,"rel","nofollow"),c(ge,"id","constructing-a-prompt"),c(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ge,"href","#constructing-a-prompt"),c(oe,"class","relative group"),c(_e,"id","controlling-inference"),c(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(_e,"href","#controlling-inference"),c(ne,"class","relative group"),c(be,"id","evaluating-generated-waveforms"),c(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(be,"href","#evaluating-generated-waveforms"),c(ie,"class","relative group"),c(tt,"href","https://huggingface.co/docs/diffusers/main/en/api/pipelines/audioldm2#diffusers.AudioLDM2Pipeline.__call__.example"),c(tt,"rel","nofollow"),c(De,"id","diffusers.AudioLDM2Pipeline"),c(De,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(De,"href","#diffusers.AudioLDM2Pipeline"),c(re,"class","relative group"),c(Lt,"href","/docs/diffusers/main/en/api/pipelines/overview#diffusers.DiffusionPipeline"),c(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Ae,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(ke,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(xe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(O,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(T,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c($e,"id","diffusers.AudioLDM2ProjectionModel"),c($e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c($e,"href","#diffusers.AudioLDM2ProjectionModel"),c(de,"class","relative group"),c($t,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(je,"id","diffusers.AudioLDM2UNet2DConditionModel"),c(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(je,"href","#diffusers.AudioLDM2UNet2DConditionModel"),c(le,"class","relative group"),c(jt,"href","/docs/diffusers/main/en/api/models/unet2d-cond#diffusers.UNet2DConditionModel"),c(Ut,"href","/docs/diffusers/main/en/api/models/overview#diffusers.ModelMixin"),c(Nt,"href","/docs/diffusers/main/en/api/pipelines/audioldm2#diffusers.AudioLDM2UNet2DConditionModel"),c(Ue,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Ne,"id","diffusers.AudioPipelineOutput"),c(Ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(Ne,"href","#diffusers.AudioPipelineOutput"),c(ce,"class","relative group"),c(pe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,p){e(document.head,f),m(t,L,p),m(t,y,p),e(y,h),e(h,x),v(u,x,null),e(y,A),e(y,H),e(H,Y),m(t,$,p),m(t,J,p),e(J,Tt),e(J,Be),e(Be,na),e(J,aa),m(t,nn,p),m(t,D,p),e(D,sa),e(D,Ie),e(Ie,ia),e(D,ra),e(D,Ft),e(Ft,da),e(D,la),e(D,Je),e(Je,ca),e(D,pa),e(D,qe),e(qe,ma),e(D,ua),e(D,Ge),e(Ge,fa),e(D,ha),e(D,Se),e(Se,ga),e(D,_a),e(D,Zt),e(Zt,va),e(D,ba),e(D,Fe),e(Fe,Ma),e(D,wa),e(D,Ot),e(Ot,ya),e(D,Da),m(t,an,p),m(t,At,p),e(At,Ta),m(t,sn,p),m(t,kt,p),e(kt,Wt),e(Wt,Aa),m(t,rn,p),m(t,S,p),e(S,ka),e(S,Ze),e(Ze,xa),e(S,Ea),e(S,Oe),e(Oe,La),e(S,$a),m(t,dn,p),m(t,K,p),e(K,ue),e(ue,zt),v(We,zt,null),e(K,ja),e(K,Qt),e(Qt,Ua),m(t,ln,p),m(t,ee,p),e(ee,fe),e(fe,Rt),v(ze,Rt,null),e(ee,Na),e(ee,Xt),e(Xt,Pa),m(t,cn,p),m(t,xt,p),e(xt,Ca),m(t,pn,p),m(t,Et,p),e(Et,Ba),m(t,mn,p),m(t,he,p),e(he,Vt),e(Vt,j),e(j,Ht),e(Ht,Ia),e(j,Ja),e(j,Yt),e(Yt,qa),e(j,Ga),e(j,Kt),e(Kt,Sa),e(j,Fa),e(j,eo),e(eo,Za),e(j,Oa),e(j,to),e(to,Wa),e(he,za),e(he,te),e(te,U),e(U,oo),e(oo,Qe),e(Qe,Qa),e(U,Ra),e(U,no),e(no,Xa),e(U,Va),e(U,ao),e(ao,Ha),e(U,Ya),e(U,so),e(so,Ka),e(U,es),e(U,io),e(io,ts),e(te,os),e(te,N),e(N,ro),e(ro,Re),e(Re,ns),e(N,as),e(N,lo),e(lo,ss),e(N,is),e(N,co),e(co,rs),e(N,ds),e(N,po),e(po,ls),e(N,cs),e(N,mo),e(mo,ps),e(te,ms),e(te,P),e(P,uo),e(uo,Xe),e(Xe,us),e(P,fs),e(P,fo),e(fo,hs),e(P,gs),e(P,ho),e(ho,_s),e(P,vs),e(P,go),e(go,bs),e(P,Ms),e(P,_o),e(_o,ws),m(t,un,p),m(t,oe,p),e(oe,ge),e(ge,vo),v(Ve,vo,null),e(oe,ys),e(oe,bo),e(bo,Ds),m(t,fn,p),m(t,F,p),e(F,Mo),e(Mo,Ts),e(F,As),e(F,wo),e(wo,ks),e(F,xs),e(F,He),e(He,Es),e(He,yo),e(yo,Ls),e(He,$s),m(t,hn,p),m(t,ne,p),e(ne,_e),e(_e,Do),v(Ye,Do,null),e(ne,js),e(ne,To),e(To,Us),m(t,gn,p),m(t,ve,p),e(ve,ae),e(ae,Ns),e(ae,Ao),e(Ao,Ps),e(ae,Cs),e(ae,ko),e(ko,Bs),e(ae,Is),e(ve,Js),e(ve,se),e(se,qs),e(se,xo),e(xo,Gs),e(se,Ss),e(se,Eo),e(Eo,Fs),e(se,Zs),m(t,_n,p),m(t,ie,p),e(ie,be),e(be,Lo),v(Ke,Lo,null),e(ie,Os),e(ie,$o),e($o,Ws),m(t,vn,p),m(t,Me,p),e(Me,jo),e(jo,zs),e(Me,Qs),e(Me,et),e(et,Rs),e(et,Uo),e(Uo,Xs),e(et,Vs),m(t,bn,p),m(t,we,p),e(we,Hs),e(we,tt),e(tt,Ys),e(we,Ks),m(t,Mn,p),v(ye,t,p),m(t,wn,p),m(t,re,p),e(re,De),e(De,No),v(ot,No,null),e(re,ei),e(re,Po),e(Po,ti),m(t,yn,p),m(t,T,p),v(nt,T,null),e(T,oi),e(T,Co),e(Co,ni),e(T,ai),e(T,at),e(at,si),e(at,Lt),e(Lt,ii),e(at,ri),e(T,di),e(T,Z),v(st,Z,null),e(Z,li),e(Z,Bo),e(Bo,ci),e(Z,pi),v(Te,Z,null),e(T,mi),e(T,Ae),v(it,Ae,null),e(Ae,ui),e(Ae,rt),e(rt,fi),e(rt,Io),e(Io,hi),e(rt,gi),e(T,_i),e(T,ke),v(dt,ke,null),e(ke,vi),e(ke,C),e(C,bi),e(C,Jo),e(Jo,Mi),e(C,wi),e(C,qo),e(qo,yi),e(C,Di),e(C,Go),e(Go,Ti),e(C,Ai),e(C,So),e(So,ki),e(C,xi),e(T,Ei),e(T,xe),v(lt,xe,null),e(xe,Li),e(xe,Fo),e(Fo,$i),e(T,ji),e(T,O),v(ct,O,null),e(O,Ui),e(O,Zo),e(Zo,Ni),e(O,Pi),v(Ee,O,null),e(T,Ci),e(T,Le),v(pt,Le,null),e(Le,Bi),e(Le,Oo),e(Oo,Ii),m(t,Dn,p),m(t,de,p),e(de,$e),e($e,Wo),v(mt,Wo,null),e(de,Ji),e(de,zo),e(zo,qi),m(t,Tn,p),m(t,q,p),v(ut,q,null),e(q,Gi),e(q,ft),e(ft,Si),e(ft,Qo),e(Qo,Fi),e(ft,Zi),e(q,Oi),e(q,$t),v(ht,$t,null),m(t,An,p),m(t,le,p),e(le,je),e(je,Ro),v(gt,Ro,null),e(le,Wi),e(le,Xo),e(Xo,zi),m(t,kn,p),m(t,B,p),v(_t,B,null),e(B,Qi),e(B,G),e(G,Ri),e(G,jt),e(jt,Xi),e(G,Vi),e(G,Vo),e(Vo,Hi),e(G,Yi),e(G,Ho),e(Ho,Ki),e(G,er),e(B,tr),e(B,vt),e(vt,or),e(vt,Ut),e(Ut,nr),e(vt,ar),e(B,sr),e(B,Ue),v(bt,Ue,null),e(Ue,ir),e(Ue,Mt),e(Mt,rr),e(Mt,Nt),e(Nt,dr),e(Mt,lr),m(t,xn,p),m(t,ce,p),e(ce,Ne),e(Ne,Yo),v(wt,Yo,null),e(ce,cr),e(ce,Ko),e(Ko,pr),m(t,En,p),m(t,pe,p),v(yt,pe,null),e(pe,mr),e(pe,en),e(en,ur),Ln=!0},p(t,[p]){const Dt={};p&2&&(Dt.$$scope={dirty:p,ctx:t}),ye.$set(Dt);const tn={};p&2&&(tn.$$scope={dirty:p,ctx:t}),Te.$set(tn);const on={};p&2&&(on.$$scope={dirty:p,ctx:t}),Ee.$set(on)},i(t){Ln||(b(u.$$.fragment,t),b(We.$$.fragment,t),b(ze.$$.fragment,t),b(Ve.$$.fragment,t),b(Ye.$$.fragment,t),b(Ke.$$.fragment,t),b(ye.$$.fragment,t),b(ot.$$.fragment,t),b(nt.$$.fragment,t),b(st.$$.fragment,t),b(Te.$$.fragment,t),b(it.$$.fragment,t),b(dt.$$.fragment,t),b(lt.$$.fragment,t),b(ct.$$.fragment,t),b(Ee.$$.fragment,t),b(pt.$$.fragment,t),b(mt.$$.fragment,t),b(ut.$$.fragment,t),b(ht.$$.fragment,t),b(gt.$$.fragment,t),b(_t.$$.fragment,t),b(bt.$$.fragment,t),b(wt.$$.fragment,t),b(yt.$$.fragment,t),Ln=!0)},o(t){M(u.$$.fragment,t),M(We.$$.fragment,t),M(ze.$$.fragment,t),M(Ve.$$.fragment,t),M(Ye.$$.fragment,t),M(Ke.$$.fragment,t),M(ye.$$.fragment,t),M(ot.$$.fragment,t),M(nt.$$.fragment,t),M(st.$$.fragment,t),M(Te.$$.fragment,t),M(it.$$.fragment,t),M(dt.$$.fragment,t),M(lt.$$.fragment,t),M(ct.$$.fragment,t),M(Ee.$$.fragment,t),M(pt.$$.fragment,t),M(mt.$$.fragment,t),M(ut.$$.fragment,t),M(ht.$$.fragment,t),M(gt.$$.fragment,t),M(_t.$$.fragment,t),M(bt.$$.fragment,t),M(wt.$$.fragment,t),M(yt.$$.fragment,t),Ln=!1},d(t){o(f),t&&o(L),t&&o(y),w(u),t&&o($),t&&o(J),t&&o(nn),t&&o(D),t&&o(an),t&&o(At),t&&o(sn),t&&o(kt),t&&o(rn),t&&o(S),t&&o(dn),t&&o(K),w(We),t&&o(ln),t&&o(ee),w(ze),t&&o(cn),t&&o(xt),t&&o(pn),t&&o(Et),t&&o(mn),t&&o(he),t&&o(un),t&&o(oe),w(Ve),t&&o(fn),t&&o(F),t&&o(hn),t&&o(ne),w(Ye),t&&o(gn),t&&o(ve),t&&o(_n),t&&o(ie),w(Ke),t&&o(vn),t&&o(Me),t&&o(bn),t&&o(we),t&&o(Mn),w(ye,t),t&&o(wn),t&&o(re),w(ot),t&&o(yn),t&&o(T),w(nt),w(st),w(Te),w(it),w(dt),w(lt),w(ct),w(Ee),w(pt),t&&o(Dn),t&&o(de),w(mt),t&&o(Tn),t&&o(q),w(ut),w(ht),t&&o(An),t&&o(le),w(gt),t&&o(kn),t&&o(B),w(_t),w(bt),t&&o(xn),t&&o(ce),w(wt),t&&o(En),t&&o(pe),w(yt)}}}const _l={local:"audioldm-2",sections:[{local:"tips",sections:[{local:"choosing-a-checkpoint",title:"Choosing a checkpoint"},{local:"constructing-a-prompt",title:"Constructing a prompt"},{local:"controlling-inference",title:"Controlling inference"},{local:"evaluating-generated-waveforms",title:"Evaluating generated waveforms:"}],title:"Tips"},{local:"diffusers.AudioLDM2Pipeline",title:"AudioLDM2Pipeline"},{local:"diffusers.AudioLDM2ProjectionModel",title:"AudioLDM2ProjectionModel"},{local:"diffusers.AudioLDM2UNet2DConditionModel",title:"AudioLDM2UNet2DConditionModel"},{local:"diffusers.AudioPipelineOutput",title:"AudioPipelineOutput"}],title:"AudioLDM 2"};function vl(me){return pl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Al extends rl{constructor(f){super();dl(this,f,vl,gl,ll,{})}}export{Al as default,_l as metadata};
