import{S as tc,i as nc,s as ac,e as r,k as i,w as v,t as n,M as rc,c as s,d as t,m as p,a as d,x as M,h as a,b as c,G as e,g as u,y as w,q as j,o as T,B as k,v as sc,L as en}from"../../chunks/vendor-hf-doc-builder.js";import{T as oc}from"../../chunks/Tip-hf-doc-builder.js";import{D as Ve}from"../../chunks/Docstring-hf-doc-builder.js";import{C as le}from"../../chunks/CodeBlock-hf-doc-builder.js";import{I as $e}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{E as Kt}from"../../chunks/ExampleCodeBlock-hf-doc-builder.js";function dc(V){let m,b,_,f,y;return f=new le({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJlcnRDb25maWclMkMlMjBXYXYyVmVjMkNvbmZpZyUyQyUyMFNwZWVjaEVuY29kZXJEZWNvZGVyQ29uZmlnJTJDJTIwU3BlZWNoRW5jb2RlckRlY29kZXJNb2RlbCUwQSUwQSUyMyUyMEluaXRpYWxpemluZyUyMGElMjBXYXYyVmVjMiUyMCUyNiUyMEJFUlQlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWdfZW5jb2RlciUyMCUzRCUyMFdhdjJWZWMyQ29uZmlnKCklMEFjb25maWdfZGVjb2RlciUyMCUzRCUyMEJlcnRDb25maWcoKSUwQSUwQWNvbmZpZyUyMCUzRCUyMFNwZWVjaEVuY29kZXJEZWNvZGVyQ29uZmlnLmZyb21fZW5jb2Rlcl9kZWNvZGVyX2NvbmZpZ3MoY29uZmlnX2VuY29kZXIlMkMlMjBjb25maWdfZGVjb2RlciklMEElMEElMjMlMjBJbml0aWFsaXppbmclMjBhJTIwV2F2MlZlYzJCZXJ0JTIwbW9kZWwlMjBmcm9tJTIwYSUyMFdhdjJWZWMyJTIwJTI2JTIwYmVydC1iYXNlLXVuY2FzZWQlMjBzdHlsZSUyMGNvbmZpZ3VyYXRpb25zJTBBbW9kZWwlMjAlM0QlMjBTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsKGNvbmZpZyUzRGNvbmZpZyklMEElMEElMjMlMjBBY2Nlc3NpbmclMjB0aGUlMjBtb2RlbCUyMGNvbmZpZ3VyYXRpb24lMEFjb25maWdfZW5jb2RlciUyMCUzRCUyMG1vZGVsLmNvbmZpZy5lbmNvZGVyJTBBY29uZmlnX2RlY29kZXIlMjAlM0QlMjBtb2RlbC5jb25maWcuZGVjb2RlciUwQSUyMyUyMHNldCUyMGRlY29kZXIlMjBjb25maWclMjB0byUyMGNhdXNhbCUyMGxtJTBBY29uZmlnX2RlY29kZXIuaXNfZGVjb2RlciUyMCUzRCUyMFRydWUlMEFjb25maWdfZGVjb2Rlci5hZGRfY3Jvc3NfYXR0ZW50aW9uJTIwJTNEJTIwVHJ1ZSUwQSUwQSUyMyUyMFNhdmluZyUyMHRoZSUyMG1vZGVsJTJDJTIwaW5jbHVkaW5nJTIwaXRzJTIwY29uZmlndXJhdGlvbiUwQW1vZGVsLnNhdmVfcHJldHJhaW5lZCglMjJteS1tb2RlbCUyMiklMEElMEElMjMlMjBsb2FkaW5nJTIwbW9kZWwlMjBhbmQlMjBjb25maWclMjBmcm9tJTIwcHJldHJhaW5lZCUyMGZvbGRlciUwQWVuY29kZXJfZGVjb2Rlcl9jb25maWclMjAlM0QlMjBTcGVlY2hFbmNvZGVyRGVjb2RlckNvbmZpZy5mcm9tX3ByZXRyYWluZWQoJTIybXktbW9kZWwlMjIpJTBBbW9kZWwlMjAlM0QlMjBTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJteS1tb2RlbCUyMiUyQyUyMGNvbmZpZyUzRGVuY29kZXJfZGVjb2Rlcl9jb25maWcp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertConfig, Wav2Vec2Config, SpeechEncoderDecoderConfig, SpeechEncoderDecoderModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a Wav2Vec2 &amp; BERT style configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>config_encoder = Wav2Vec2Config()
<span class="hljs-meta">&gt;&gt;&gt; </span>config_decoder = BertConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span>config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Initializing a Wav2Vec2Bert model from a Wav2Vec2 &amp; bert-base-uncased style configurations</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = SpeechEncoderDecoderModel(config=config)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Accessing the model configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>config_encoder = model.config.encoder
<span class="hljs-meta">&gt;&gt;&gt; </span>config_decoder = model.config.decoder
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># set decoder config to causal lm</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>config_decoder.is_decoder = <span class="hljs-literal">True</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>config_decoder.add_cross_attention = <span class="hljs-literal">True</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Saving the model, including its configuration</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.save_pretrained(<span class="hljs-string">&quot;my-model&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># loading model and config from pretrained folder</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>encoder_decoder_config = SpeechEncoderDecoderConfig.from_pretrained(<span class="hljs-string">&quot;my-model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = SpeechEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;my-model&quot;</span>, config=encoder_decoder_config)`}}),{c(){m=r("p"),b=n("Examples:"),_=i(),v(f.$$.fragment)},l(l){m=s(l,"P",{});var g=d(m);b=a(g,"Examples:"),g.forEach(t),_=p(l),M(f.$$.fragment,l)},m(l,g){u(l,m,g),e(m,b),u(l,_,g),w(f,l,g),y=!0},p:en,i(l){y||(j(f.$$.fragment,l),y=!0)},o(l){T(f.$$.fragment,l),y=!1},d(l){l&&t(m),l&&t(_),k(f,l)}}}function lc(V){let m,b,_,f,y;return{c(){m=r("p"),b=n("Although the recipe for forward pass needs to be defined within this function, one should call the "),_=r("code"),f=n("Module"),y=n(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(l){m=s(l,"P",{});var g=d(m);b=a(g,"Although the recipe for forward pass needs to be defined within this function, one should call the "),_=s(g,"CODE",{});var I=d(_);f=a(I,"Module"),I.forEach(t),y=a(g,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),g.forEach(t)},m(l,g){u(l,m,g),e(m,b),e(m,_),e(_,f),e(m,y)},d(l){l&&t(m)}}}function cc(V){let m,b,_,f,y;return f=new le({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNwZWVjaEVuY29kZXJEZWNvZGVyTW9kZWwlMkMlMjBBdXRvUHJvY2Vzc29yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBaW1wb3J0JTIwdG9yY2glMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRndhdjJ2ZWMyLXhscy1yLTMwMG0tZW4tdG8tMTUlMjIpJTBBbW9kZWwlMjAlM0QlMjBTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRndhdjJ2ZWMyLXhscy1yLTMwMG0tZW4tdG8tMTUlMjIpJTBBJTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyaGYtaW50ZXJuYWwtdGVzdGluZyUyRmxpYnJpc3BlZWNoX2Fzcl9kdW1teSUyMiUyQyUyMCUyMmNsZWFuJTIyJTJDJTIwc3BsaXQlM0QlMjJ2YWxpZGF0aW9uJTIyKSUwQSUwQWlucHV0X3ZhbHVlcyUyMCUzRCUyMHByb2Nlc3NvcihkcyU1QjAlNUQlNUIlMjJhdWRpbyUyMiU1RCU1QiUyMmFycmF5JTIyJTVEJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikuaW5wdXRfdmFsdWVzJTBBJTIzJTIwSW5mZXJlbmNlJTNBJTIwVHJhbnNsYXRlJTIwRW5nbGlzaCUyMHNwZWVjaCUyMHRvJTIwR2VybWFuJTBBZ2VuZXJhdGVkJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoaW5wdXRfdmFsdWVzKSUwQWRlY29kZWQlMjAlM0QlMjBwcm9jZXNzb3IuYmF0Y2hfZGVjb2RlKGdlbmVyYXRlZCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQlMEFkZWNvZGVkJTBBJTBBJTIzJTIwVHJhaW5pbmclM0ElMjBUcmFpbiUyMG1vZGVsJTIwb24lMjBFbmdsaXNoJTIwdHJhbnNjcmlwdGlvbiUwQWxhYmVscyUyMCUzRCUyMHByb2Nlc3Nvcih0ZXh0JTNEZHMlNUIwJTVEJTVCJTIydGV4dCUyMiU1RCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLmlucHV0X2lkcyUwQSUwQWxvc3MlMjAlM0QlMjBtb2RlbChpbnB1dF92YWx1ZXMlMkMlMjBsYWJlbHMlM0RsYWJlbHMpLmxvc3MlMEFsb3NzLmJhY2t3YXJkKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> SpeechEncoderDecoderModel, AutoProcessor
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-xls-r-300m-en-to-15&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = SpeechEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-xls-r-300m-en-to-15&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&quot;hf-internal-testing/librispeech_asr_dummy&quot;</span>, <span class="hljs-string">&quot;clean&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_values = processor(ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_values
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Inference: Translate English speech to German</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generated = model.generate(input_values)
<span class="hljs-meta">&gt;&gt;&gt; </span>decoded = processor.batch_decode(generated, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>decoded
<span class="hljs-string">&#x27;Mr. Quilter ist der Apostel der Mittelschicht und wir freuen uns, sein Evangelium willkommen hei\xDFen zu k\xF6nnen.&#x27;</span>

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Training: Train model on English transcription</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>labels = processor(text=ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids

<span class="hljs-meta">&gt;&gt;&gt; </span>loss = model(input_values, labels=labels).loss
<span class="hljs-meta">&gt;&gt;&gt; </span>loss.backward()`}}),{c(){m=r("p"),b=n("Examples:"),_=i(),v(f.$$.fragment)},l(l){m=s(l,"P",{});var g=d(m);b=a(g,"Examples:"),g.forEach(t),_=p(l),M(f.$$.fragment,l)},m(l,g){u(l,m,g),e(m,b),u(l,_,g),w(f,l,g),y=!0},p:en,i(l){y||(j(f.$$.fragment,l),y=!0)},o(l){T(f.$$.fragment,l),y=!1},d(l){l&&t(m),l&&t(_),k(f,l)}}}function ic(V){let m,b,_,f,y;return f=new le({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNwZWVjaEVuY29kZXJEZWNvZGVyTW9kZWwlMEElMEElMjMlMjBpbml0aWFsaXplJTIwYSUyMHdhdjJ2ZWMyYmVydCUyMGZyb20lMjBhJTIwcHJldHJhaW5lZCUyMFdhdjJWZWMyJTIwYW5kJTIwYSUyMHByZXRyYWluZWQlMjBCRVJUJTIwbW9kZWwuJTIwTm90ZSUyMHRoYXQlMjB0aGUlMjBjcm9zcy1hdHRlbnRpb24lMjBsYXllcnMlMjB3aWxsJTIwYmUlMjByYW5kb21seSUyMGluaXRpYWxpemVkJTBBbW9kZWwlMjAlM0QlMjBTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsLmZyb21fZW5jb2Rlcl9kZWNvZGVyX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyZmFjZWJvb2slMkZ3YXYydmVjMi1iYXNlLTk2MGglMjIlMkMlMjAlMjJiZXJ0LWJhc2UtdW5jYXNlZCUyMiUwQSklMEElMjMlMjBzYXZpbmclMjBtb2RlbCUyMGFmdGVyJTIwZmluZS10dW5pbmclMEFtb2RlbC5zYXZlX3ByZXRyYWluZWQoJTIyLiUyRndhdjJ2ZWMyYmVydCUyMiklMEElMjMlMjBsb2FkJTIwZmluZS10dW5lZCUyMG1vZGVsJTBBbW9kZWwlMjAlM0QlMjBTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMjIuJTJGd2F2MnZlYzJiZXJ0JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> SpeechEncoderDecoderModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># initialize a wav2vec2bert from a pretrained Wav2Vec2 and a pretrained BERT model. Note that the cross-attention layers will be randomly initialized</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = SpeechEncoderDecoderModel.from_encoder_decoder_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>, <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># saving model after fine-tuning</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.save_pretrained(<span class="hljs-string">&quot;./wav2vec2bert&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load fine-tuned model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = SpeechEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;./wav2vec2bert&quot;</span>)`}}),{c(){m=r("p"),b=n("Example:"),_=i(),v(f.$$.fragment)},l(l){m=s(l,"P",{});var g=d(m);b=a(g,"Example:"),g.forEach(t),_=p(l),M(f.$$.fragment,l)},m(l,g){u(l,m,g),e(m,b),u(l,_,g),w(f,l,g),y=!0},p:en,i(l){y||(j(f.$$.fragment,l),y=!0)},o(l){T(f.$$.fragment,l),y=!1},d(l){l&&t(m),l&&t(_),k(f,l)}}}function pc(V){let m,b,_,f,y;return{c(){m=r("p"),b=n("Although the recipe for forward pass needs to be defined within this function, one should call the "),_=r("code"),f=n("Module"),y=n(`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`)},l(l){m=s(l,"P",{});var g=d(m);b=a(g,"Although the recipe for forward pass needs to be defined within this function, one should call the "),_=s(g,"CODE",{});var I=d(_);f=a(I,"Module"),I.forEach(t),y=a(g,`
instance afterwards instead of this since the former takes care of running the pre and post processing steps while
the latter silently ignores them.`),g.forEach(t)},m(l,g){u(l,m,g),e(m,b),e(m,_),e(_,f),e(m,y)},d(l){l&&t(m)}}}function hc(V){let m,b,_,f,y;return f=new le({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEZsYXhTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQSUyMyUyMGxvYWQlMjBhJTIwZmluZS10dW5lZCUyMHdhdjJ2ZWMyLTItYmFydCUyMG1vZGVsJTBBbW9kZWwlMjAlM0QlMjBGbGF4U3BlZWNoRW5jb2RlckRlY29kZXJNb2RlbC5mcm9tX3ByZXRyYWluZWQoJTIycGF0cmlja3ZvbnBsYXRlbiUyRndhdjJ2ZWMyLTItYmFydC1sYXJnZSUyMiklMEElMjMlMjBsb2FkJTIwb3V0cHV0JTIwdG9rZW5pemVyJTBBdG9rZW5pemVyX291dHB1dCUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGYmFydC1sYXJnZSUyMiklMEElMEFpbnB1dHMlMjAlM0QlMjBqbnAub25lcygoMiUyQyUyMDUwMDApJTJDJTIwZHR5cGUlM0RqbnAuZmxvYXQzMiklMEElMEElMjMlMjB1c2UlMjBiYXJ0J3MlMjBzcGVjaWFsJTIwYm9zJTJDJTIwcGFkJTIwYW5kJTIwZW9zJTIwdG9rZW5zJTBBbW9kZWwuY29uZmlnLmRlY29kZXJfc3RhcnRfdG9rZW5faWQlMjAlM0QlMjBtb2RlbC5kZWNvZGVyLmNvbmZpZy5ib3NfdG9rZW5faWQlMEFtb2RlbC5jb25maWcucGFkX3Rva2VuX2lkJTIwJTNEJTIwbW9kZWwuZGVjb2Rlci5jb25maWcucGFkX3Rva2VuX2lkJTBBbW9kZWwuY29uZmlnLmVvc190b2tlbl9pZCUyMCUzRCUyMG1vZGVsLmRlY29kZXIuY29uZmlnLmVvc190b2tlbl9pZCUwQSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dHMp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> FlaxSpeechEncoderDecoderModel, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load a fine-tuned wav2vec2-2-bart model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = FlaxSpeechEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;patrickvonplaten/wav2vec2-2-bart-large&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load output tokenizer</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer_output = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/bart-large&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = jnp.ones((<span class="hljs-number">2</span>, <span class="hljs-number">5000</span>), dtype=jnp.float32)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># use bart&#x27;s special bos, pad and eos tokens</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.decoder_start_token_id = model.decoder.config.bos_token_id
<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.pad_token_id = model.decoder.config.pad_token_id
<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.eos_token_id = model.decoder.config.eos_token_id

<span class="hljs-meta">&gt;&gt;&gt; </span>outputs = model.generate(inputs)
<span class="hljs-comment"># Assert something? More interesting input? dtype correct?</span>`}}),{c(){m=r("p"),b=n("Examples:"),_=i(),v(f.$$.fragment)},l(l){m=s(l,"P",{});var g=d(m);b=a(g,"Examples:"),g.forEach(t),_=p(l),M(f.$$.fragment,l)},m(l,g){u(l,m,g),e(m,b),u(l,_,g),w(f,l,g),y=!0},p:en,i(l){y||(j(f.$$.fragment,l),y=!0)},o(l){T(f.$$.fragment,l),y=!1},d(l){l&&t(m),l&&t(_),k(f,l)}}}function mc(V){let m,b,_,f,y;return f=new le({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEZsYXhTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsJTBBJTBBJTIzJTIwaW5pdGlhbGl6ZSUyMGElMjB3YXYydmVjMi0yLWJhcnQlMjBmcm9tJTIwcHJldHJhaW5lZCUyMHdhdjJ2ZWMyJTIwYW5kJTIwYmFydCUyMG1vZGVscy4lMjBOb3RlJTIwdGhhdCUyMHRoZSUyMGNyb3NzLWF0dGVudGlvbiUyMGxheWVycyUyMHdpbGwlMjBiZSUyMHJhbmRvbWx5JTIwaW5pdGlhbGl6ZWQlMEFtb2RlbCUyMCUzRCUyMEZsYXhTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsLmZyb21fZW5jb2Rlcl9kZWNvZGVyX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyZmFjZWJvb2slMkZ3YXYydmVjMi1sYXJnZS1sdjYwJTIyJTJDJTIwJTIyZmFjZWJvb2slMkZiYXJ0LWxhcmdlJTIyJTBBKSUwQSUyMyUyMHNhdmluZyUyMG1vZGVsJTIwYWZ0ZXIlMjBmaW5lLXR1bmluZyUwQW1vZGVsLnNhdmVfcHJldHJhaW5lZCglMjIuJTJGd2F2MnZlYzItMi1iYXJ0LWxhcmdlJTIyKSUwQSUyMyUyMGxvYWQlMjBmaW5lLXR1bmVkJTIwbW9kZWwlMEFtb2RlbCUyMCUzRCUyMEZsYXhTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsLmZyb21fcHJldHJhaW5lZCglMjIuJTJGd2F2MnZlYzItMi1iYXJ0LWxhcmdlJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> FlaxSpeechEncoderDecoderModel

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># initialize a wav2vec2-2-bart from pretrained wav2vec2 and bart models. Note that the cross-attention layers will be randomly initialized</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = FlaxSpeechEncoderDecoderModel.from_encoder_decoder_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;facebook/wav2vec2-large-lv60&quot;</span>, <span class="hljs-string">&quot;facebook/bart-large&quot;</span>
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># saving model after fine-tuning</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model.save_pretrained(<span class="hljs-string">&quot;./wav2vec2-2-bart-large&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load fine-tuned model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = FlaxSpeechEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;./wav2vec2-2-bart-large&quot;</span>)`}}),{c(){m=r("p"),b=n("Example:"),_=i(),v(f.$$.fragment)},l(l){m=s(l,"P",{});var g=d(m);b=a(g,"Example:"),g.forEach(t),_=p(l),M(f.$$.fragment,l)},m(l,g){u(l,m,g),e(m,b),u(l,_,g),w(f,l,g),y=!0},p:en,i(l){y||(j(f.$$.fragment,l),y=!0)},o(l){T(f.$$.fragment,l),y=!1},d(l){l&&t(m),l&&t(_),k(f,l)}}}function fc(V){let m,b,_,f,y,l,g,I,qn,on,x,An,jo,Pn,Qn,st,Hn,Ln,To,On,Kn,ko,ea,oa,tn,Y,ta,dt,na,aa,xe,ra,sa,nn,N,da,Eo,la,ca,Zo,ia,pa,an,Q,ce,lt,Re,ha,Ge,ma,ct,fa,ua,rn,D,Jo,ga,_a,Wo,ya,ba,it,va,Ma,sn,Ce,dn,H,ie,pt,Fe,wa,Be,ja,ht,Ta,ka,ln,E,So,Ea,Za,mt,Ja,Wa,Uo,Sa,Ua,Vo,Va,$a,ft,xa,Ra,ut,Ga,Ca,gt,Fa,Ba,$o,za,Ia,pe,Da,_t,Ya,Na,yt,Xa,qa,xo,Aa,Pa,cn,ze,pn,L,he,bt,Ie,Qa,De,Ha,vt,La,Oa,hn,G,Ka,Mt,er,or,Ro,tr,nr,wt,ar,rr,mn,me,sr,jt,dr,lr,fn,Ye,un,O,fe,Tt,Ne,cr,kt,ir,gn,C,pr,Et,hr,mr,Zt,fr,ur,Jt,gr,_r,_n,Xe,yn,K,ue,Wt,qe,yr,St,br,bn,$,Ae,vr,ge,Go,Mr,wr,Co,jr,Tr,kr,ee,Er,Fo,Zr,Jr,Bo,Wr,Sr,Ur,_e,Vr,ye,Pe,$r,Qe,xr,zo,Rr,Gr,vn,oe,be,Ut,He,Cr,Vt,Fr,Mn,Z,Le,Br,te,zr,Io,Ir,Dr,Do,Yr,Nr,Xr,Oe,qr,Ke,Ar,Pr,Qr,eo,Hr,oo,Lr,Or,Kr,$t,es,os,to,ts,Yo,ns,as,rs,no,ss,ao,ds,ls,cs,X,No,is,ps,xt,hs,ms,Rt,fs,us,gs,F,ro,_s,ne,ys,Xo,bs,vs,Gt,Ms,ws,js,ve,Ts,Me,ks,B,so,Es,Ct,Zs,Js,ae,Ws,Ft,Ss,Us,Bt,Vs,$s,xs,we,wn,re,je,zt,lo,Rs,It,Gs,jn,J,co,Cs,se,Fs,qo,Bs,zs,Ao,Is,Ds,Ys,io,Ns,po,Xs,qs,As,ho,Ps,mo,Qs,Hs,Ls,Dt,Os,Ks,fo,ed,Po,od,td,nd,uo,ad,go,rd,sd,dd,q,Qo,ld,cd,Yt,id,pd,Nt,hd,md,fd,z,_o,ud,de,gd,Ho,_d,yd,Xt,bd,vd,Md,Te,wd,ke,jd,A,yo,Td,qt,kd,Ed,Ee,Tn;return l=new $e({}),Re=new $e({}),Ce=new le({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEJlcnRDb25maWclMkMlMjBXYXYyVmVjMkNvbmZpZyUyQyUyMFNwZWVjaEVuY29kZXJEZWNvZGVyQ29uZmlnJTJDJTIwU3BlZWNoRW5jb2RlckRlY29kZXJNb2RlbCUwQSUwQWNvbmZpZ19lbmNvZGVyJTIwJTNEJTIwV2F2MlZlYzJDb25maWcoKSUwQWNvbmZpZ19kZWNvZGVyJTIwJTNEJTIwQmVydENvbmZpZygpJTBBJTBBY29uZmlnJTIwJTNEJTIwU3BlZWNoRW5jb2RlckRlY29kZXJDb25maWcuZnJvbV9lbmNvZGVyX2RlY29kZXJfY29uZmlncyhjb25maWdfZW5jb2RlciUyQyUyMGNvbmZpZ19kZWNvZGVyKSUwQW1vZGVsJTIwJTNEJTIwU3BlZWNoRW5jb2RlckRlY29kZXJNb2RlbChjb25maWclM0Rjb25maWcp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> BertConfig, Wav2Vec2Config, SpeechEncoderDecoderConfig, SpeechEncoderDecoderModel

<span class="hljs-meta">&gt;&gt;&gt; </span>config_encoder = Wav2Vec2Config()
<span class="hljs-meta">&gt;&gt;&gt; </span>config_decoder = BertConfig()

<span class="hljs-meta">&gt;&gt;&gt; </span>config = SpeechEncoderDecoderConfig.from_encoder_decoder_configs(config_encoder, config_decoder)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = SpeechEncoderDecoderModel(config=config)`}}),Fe=new $e({}),ze=new le({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNwZWVjaEVuY29kZXJEZWNvZGVyTW9kZWwlMEElMEFtb2RlbCUyMCUzRCUyMFNwZWVjaEVuY29kZXJEZWNvZGVyTW9kZWwuZnJvbV9lbmNvZGVyX2RlY29kZXJfcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJmYWNlYm9vayUyRmh1YmVydC1sYXJnZS1sbDYwayUyMiUyQyUyMCUyMmJlcnQtYmFzZS11bmNhc2VkJTIyJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> SpeechEncoderDecoderModel

<span class="hljs-meta">&gt;&gt;&gt; </span>model = SpeechEncoderDecoderModel.from_encoder_decoder_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;facebook/hubert-large-ll60k&quot;</span>, <span class="hljs-string">&quot;bert-base-uncased&quot;</span>
<span class="hljs-meta">... </span>)`}}),Ie=new $e({}),Ye=new le({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFdhdjJWZWMyUHJvY2Vzc29yJTJDJTIwU3BlZWNoRW5jb2RlckRlY29kZXJNb2RlbCUwQWZyb20lMjBkYXRhc2V0cyUyMGltcG9ydCUyMGxvYWRfZGF0YXNldCUwQWltcG9ydCUyMHRvcmNoJTBBJTBBJTIzJTIwbG9hZCUyMGElMjBmaW5lLXR1bmVkJTIwc3BlZWNoJTIwdHJhbnNsYXRpb24lMjBtb2RlbCUyMGFuZCUyMGNvcnJlc3BvbmRpbmclMjBwcm9jZXNzb3IlMEFtb2RlbCUyMCUzRCUyMFNwZWVjaEVuY29kZXJEZWNvZGVyTW9kZWwuZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGd2F2MnZlYzIteGxzLXItMzAwbS1lbi10by0xNSUyMiklMEFwcm9jZXNzb3IlMjAlM0QlMjBXYXYyVmVjMlByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZ3YXYydmVjMi14bHMtci0zMDBtLWVuLXRvLTE1JTIyKSUwQSUwQSUyMyUyMGxldCdzJTIwcGVyZm9ybSUyMGluZmVyZW5jZSUyMG9uJTIwYSUyMHBpZWNlJTIwb2YlMjBFbmdsaXNoJTIwc3BlZWNoJTIwKHdoaWNoJTIwd2UnbGwlMjB0cmFuc2xhdGUlMjB0byUyMEdlcm1hbiklMEFkcyUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJoZi1pbnRlcm5hbC10ZXN0aW5nJTJGbGlicmlzcGVlY2hfYXNyX2R1bW15JTIyJTJDJTIwJTIyY2xlYW4lMjIlMkMlMjBzcGxpdCUzRCUyMnZhbGlkYXRpb24lMjIpJTBBaW5wdXRfdmFsdWVzJTIwJTNEJTIwcHJvY2Vzc29yKGRzJTVCMCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTVCJTIyYXJyYXklMjIlNUQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS5pbnB1dF92YWx1ZXMlMEElMEElMjMlMjBhdXRvcmVncmVzc2l2ZWx5JTIwZ2VuZXJhdGUlMjB0cmFuc2NyaXB0aW9uJTIwKHVzZXMlMjBncmVlZHklMjBkZWNvZGluZyUyMGJ5JTIwZGVmYXVsdCklMEFnZW5lcmF0ZWRfaWRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoaW5wdXRfdmFsdWVzKSUwQWdlbmVyYXRlZF90ZXh0JTIwJTNEJTIwcHJvY2Vzc29yLmJhdGNoX2RlY29kZShnZW5lcmF0ZWRfaWRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTVCMCU1RCUwQXByaW50KGdlbmVyYXRlZF90ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> Wav2Vec2Processor, SpeechEncoderDecoderModel
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load a fine-tuned speech translation model and corresponding processor</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = SpeechEncoderDecoderModel.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-xls-r-300m-en-to-15&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>processor = Wav2Vec2Processor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-xls-r-300m-en-to-15&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># let&#x27;s perform inference on a piece of English speech (which we&#x27;ll translate to German)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&quot;hf-internal-testing/librispeech_asr_dummy&quot;</span>, <span class="hljs-string">&quot;clean&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_values = processor(ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_values

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># autoregressively generate transcription (uses greedy decoding by default)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(input_values)
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_text = processor.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(generated_text)
Mr. Quilter ist der Apostel der Mittelschicht und wir freuen uns, sein Evangelium willkommen hei\xDFen zu k\xF6nnen.`}}),Ne=new $e({}),Xe=new le({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvRmVhdHVyZUV4dHJhY3RvciUyQyUyMFNwZWVjaEVuY29kZXJEZWNvZGVyTW9kZWwlMEFmcm9tJTIwZGF0YXNldHMlMjBpbXBvcnQlMjBsb2FkX2RhdGFzZXQlMEElMEFlbmNvZGVyX2lkJTIwJTNEJTIwJTIyZmFjZWJvb2slMkZ3YXYydmVjMi1iYXNlLTk2MGglMjIlMjAlMjAlMjMlMjBhY291c3RpYyUyMG1vZGVsJTIwZW5jb2RlciUwQWRlY29kZXJfaWQlMjAlM0QlMjAlMjJiZXJ0LWJhc2UtdW5jYXNlZCUyMiUyMCUyMCUyMyUyMHRleHQlMjBkZWNvZGVyJTBBJTBBZmVhdHVyZV9leHRyYWN0b3IlMjAlM0QlMjBBdXRvRmVhdHVyZUV4dHJhY3Rvci5mcm9tX3ByZXRyYWluZWQoZW5jb2Rlcl9pZCklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChkZWNvZGVyX2lkKSUwQSUyMyUyMENvbWJpbmUlMjBwcmUtdHJhaW5lZCUyMGVuY29kZXIlMjBhbmQlMjBwcmUtdHJhaW5lZCUyMGRlY29kZXIlMjB0byUyMGZvcm0lMjBhJTIwU2VxMlNlcSUyMG1vZGVsJTBBbW9kZWwlMjAlM0QlMjBTcGVlY2hFbmNvZGVyRGVjb2Rlck1vZGVsLmZyb21fZW5jb2Rlcl9kZWNvZGVyX3ByZXRyYWluZWQoZW5jb2Rlcl9pZCUyQyUyMGRlY29kZXJfaWQpJTBBJTBBbW9kZWwuY29uZmlnLmRlY29kZXJfc3RhcnRfdG9rZW5faWQlMjAlM0QlMjB0b2tlbml6ZXIuY2xzX3Rva2VuX2lkJTBBbW9kZWwuY29uZmlnLnBhZF90b2tlbl9pZCUyMCUzRCUyMHRva2VuaXplci5wYWRfdG9rZW5faWQlMEElMEElMjMlMjBsb2FkJTIwYW4lMjBhdWRpbyUyMGlucHV0JTIwYW5kJTIwcHJlLXByb2Nlc3MlMjAobm9ybWFsaXNlJTIwbWVhbiUyRnN0ZCUyMHRvJTIwMCUyRjEpJTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyaGYtaW50ZXJuYWwtdGVzdGluZyUyRmxpYnJpc3BlZWNoX2Fzcl9kdW1teSUyMiUyQyUyMCUyMmNsZWFuJTIyJTJDJTIwc3BsaXQlM0QlMjJ2YWxpZGF0aW9uJTIyKSUwQWlucHV0X3ZhbHVlcyUyMCUzRCUyMGZlYXR1cmVfZXh0cmFjdG9yKGRzJTVCMCU1RCU1QiUyMmF1ZGlvJTIyJTVEJTVCJTIyYXJyYXklMjIlNUQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS5pbnB1dF92YWx1ZXMlMEElMEElMjMlMjBsb2FkJTIwaXRzJTIwY29ycmVzcG9uZGluZyUyMHRyYW5zY3JpcHRpb24lMjBhbmQlMjB0b2tlbml6ZSUyMHRvJTIwZ2VuZXJhdGUlMjBsYWJlbHMlMEFsYWJlbHMlMjAlM0QlMjB0b2tlbml6ZXIoZHMlNUIwJTVEJTVCJTIydGV4dCUyMiU1RCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLmlucHV0X2lkcyUwQSUwQSUyMyUyMHRoZSUyMGZvcndhcmQlMjBmdW5jdGlvbiUyMGF1dG9tYXRpY2FsbHklMjBjcmVhdGVzJTIwdGhlJTIwY29ycmVjdCUyMGRlY29kZXJfaW5wdXRfaWRzJTBBbG9zcyUyMCUzRCUyMG1vZGVsKCoqaW5wdXRfZmVhdHVyZXMpLmxvc3MlMEFsb3NzLmJhY2t3YXJkKCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoFeatureExtractor, SpeechEncoderDecoderModel
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>encoder_id = <span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>  <span class="hljs-comment"># acoustic model encoder</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>decoder_id = <span class="hljs-string">&quot;bert-base-uncased&quot;</span>  <span class="hljs-comment"># text decoder</span>

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(encoder_id)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(decoder_id)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># Combine pre-trained encoder and pre-trained decoder to form a Seq2Seq model</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>model = SpeechEncoderDecoderModel.from_encoder_decoder_pretrained(encoder_id, decoder_id)

<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.decoder_start_token_id = tokenizer.cls_token_id
<span class="hljs-meta">&gt;&gt;&gt; </span>model.config.pad_token_id = tokenizer.pad_token_id

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load an audio input and pre-process (normalise mean/std to 0/1)</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = load_dataset(<span class="hljs-string">&quot;hf-internal-testing/librispeech_asr_dummy&quot;</span>, <span class="hljs-string">&quot;clean&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>input_values = feature_extractor(ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_values

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># load its corresponding transcription and tokenize to generate labels</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>labels = tokenizer(ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>], return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).input_ids

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-comment"># the forward function automatically creates the correct decoder_input_ids</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>loss = model(**input_features).loss
<span class="hljs-meta">&gt;&gt;&gt; </span>loss.backward()`}}),qe=new $e({}),Ae=new Ve({props:{name:"class transformers.SpeechEncoderDecoderConfig",anchor:"transformers.SpeechEncoderDecoderConfig",parameters:[{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.SpeechEncoderDecoderConfig.kwargs",description:`<strong>kwargs</strong> (<em>optional</em>) &#x2014;
Dictionary of keyword arguments. Notably:</p>
<ul>
<li><strong>encoder</strong> (<a href="/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a>, <em>optional</em>) &#x2014; An instance of a configuration object that defines
the encoder config.</li>
<li><strong>decoder</strong> (<a href="/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig">PretrainedConfig</a>, <em>optional</em>) &#x2014; An instance of a configuration object that defines
the decoder config.</li>
</ul>`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/speech_encoder_decoder/configuration_speech_encoder_decoder.py#L26"}}),_e=new Kt({props:{anchor:"transformers.SpeechEncoderDecoderConfig.example",$$slots:{default:[dc]},$$scope:{ctx:V}}}),Pe=new Ve({props:{name:"from_encoder_decoder_configs",anchor:"transformers.SpeechEncoderDecoderConfig.from_encoder_decoder_configs",parameters:[{name:"encoder_config",val:": PretrainedConfig"},{name:"decoder_config",val:": PretrainedConfig"},{name:"**kwargs",val:""}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/speech_encoder_decoder/configuration_speech_encoder_decoder.py#L92",returnDescription:`
<p>An instance of a configuration object</p>
`,returnType:`
<p><a
  href="/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderConfig"
>SpeechEncoderDecoderConfig</a></p>
`}}),He=new $e({}),Le=new Ve({props:{name:"class transformers.SpeechEncoderDecoderModel",anchor:"transformers.SpeechEncoderDecoderModel",parameters:[{name:"config",val:": typing.Optional[transformers.configuration_utils.PretrainedConfig] = None"},{name:"encoder",val:": typing.Optional[transformers.modeling_utils.PreTrainedModel] = None"},{name:"decoder",val:": typing.Optional[transformers.modeling_utils.PreTrainedModel] = None"}],parametersDescription:[{anchor:"transformers.SpeechEncoderDecoderModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderConfig">SpeechEncoderDecoderConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py#L173"}}),ro=new Ve({props:{name:"forward",anchor:"transformers.SpeechEncoderDecoderModel.forward",parameters:[{name:"inputs",val:": typing.Optional[torch.FloatTensor] = None"},{name:"attention_mask",val:": typing.Optional[torch.FloatTensor] = None"},{name:"decoder_input_ids",val:": typing.Optional[torch.LongTensor] = None"},{name:"decoder_attention_mask",val:": typing.Optional[torch.BoolTensor] = None"},{name:"encoder_outputs",val:": typing.Optional[typing.Tuple[torch.FloatTensor]] = None"},{name:"past_key_values",val:": typing.Optional[typing.Tuple[typing.Tuple[torch.FloatTensor]]] = None"},{name:"decoder_inputs_embeds",val:": typing.Optional[torch.FloatTensor] = None"},{name:"labels",val:": typing.Optional[torch.LongTensor] = None"},{name:"use_cache",val:": typing.Optional[bool] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"input_values",val:": typing.Optional[torch.FloatTensor] = None"},{name:"input_features",val:": typing.Optional[torch.FloatTensor] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.SpeechEncoderDecoderModel.forward.inputs",description:`<strong>inputs</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code> or <code>(batch_size, sequence_length, feature_dim)</code>, <em>optional</em>) &#x2014;
Float values of input raw speech waveform or speech features. Values can be obtained by loading a <code>.flac</code>
or <code>.wav</code> audio file into an array of type <code>List[float]</code> or a <code>numpy.ndarray</code>, <em>e.g.</em> via the soundfile
library (<code>pip install soundfile</code>). To prepare the array into <code>inputs</code>, either the <a href="/docs/transformers/v4.32.0/en/model_doc/wav2vec2#transformers.Wav2Vec2Processor">Wav2Vec2Processor</a> or
<a href="/docs/transformers/v4.32.0/en/model_doc/speech_to_text#transformers.Speech2TextProcessor">Speech2TextProcessor</a> should be used for padding and conversion into a tensor of type
<code>torch.FloatTensor</code>.`,name:"inputs"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.attention_mask",description:`<strong>attention_mask</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.decoder_input_ids",description:`<strong>decoder_input_ids</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of decoder input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.32.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a>. See <a href="/docs/transformers/v4.32.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.32.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a></p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>For training, <code>decoder_input_ids</code> are automatically created by the model by shifting the <code>labels</code> to the
right, replacing -100 by the <code>pad_token_id</code> and prepending them with the <code>decoder_start_token_id</code>.`,name:"decoder_input_ids"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.decoder_attention_mask",description:`<strong>decoder_attention_mask</strong> (<code>torch.BoolTensor</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Default behavior: generate a tensor that ignores pad tokens in <code>decoder_input_ids</code>. Causal mask will also
be used by default.`,name:"decoder_attention_mask"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.encoder_outputs",description:`<strong>encoder_outputs</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>) &#x2014;
This tuple must consist of (<code>last_hidden_state</code>, <em>optional</em>: <code>hidden_states</code>, <em>optional</em>: <code>attentions</code>)
<code>last_hidden_state</code> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>) is a tensor
of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the
decoder.`,name:"encoder_outputs"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.past_key_values",description:`<strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code> of length <code>config.n_layers</code> with each tuple having 4 tensors of shape <code>(batch_size, num_heads, sequence_length - 1, embed_size_per_head)</code>) &#x2014;
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.</p>
<p>If <code>past_key_values</code> are used, the user can optionally input only the last <code>decoder_input_ids</code> (those that
don&#x2019;t have their past key value states given to this model) of shape <code>(batch_size, 1)</code> instead of all
<code>decoder_input_ids</code> of shape <code>(batch_size, sequence_length)</code>.`,name:"past_key_values"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.inputs_embeds",description:`<strong>inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>input_ids</code> you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert <code>input_ids</code> indices into associated vectors than the
model&#x2019;s internal embedding lookup matrix.`,name:"inputs_embeds"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.decoder_inputs_embeds",description:`<strong>decoder_inputs_embeds</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, target_sequence_length, hidden_size)</code>, <em>optional</em>) &#x2014;
Optionally, instead of passing <code>decoder_input_ids</code> you can choose to directly pass an embedded
representation. This is useful if you want more control over how to convert <code>decoder_input_ids</code> indices
into associated vectors than the model&#x2019;s internal embedding lookup matrix.`,name:"decoder_inputs_embeds"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.labels",description:`<strong>labels</strong> (<code>torch.LongTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Labels for computing the masked language modeling loss for the decoder. Indices should be in <code>[-100, 0, ..., config.vocab_size]</code> (see <code>input_ids</code> docstring) Tokens with indices set to <code>-100</code> are ignored
(masked), the loss is only computed for the tokens with labels in <code>[0, ..., config.vocab_size]</code>`,name:"labels"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.use_cache",description:`<strong>use_cache</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, <code>past_key_values</code> key value states are returned and can be used to speed up decoding (see
<code>past_key_values</code>).`,name:"use_cache"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.output_attentions",description:`<strong>output_attentions</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the attentions tensors of all attention layers. See <code>attentions</code> under returned
tensors for more detail.`,name:"output_attentions"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.input_values",description:`<strong>input_values</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Float values of input raw speech waveform. Values can be obtained by loading a <em>.flac</em> or <em>.wav</em> audio file
into an array of type <em>List[float]</em> or a <em>numpy.ndarray</em>, <em>e.g.</em> via the soundfile library (<em>pip install
soundfile</em>). To prepare the array into <em>input_values</em>, the <a href="/docs/transformers/v4.32.0/en/model_doc/wav2vec2#transformers.Wav2Vec2Processor">Wav2Vec2Processor</a> should be used for padding
and conversion into a tensor of type <em>torch.FloatTensor</em>. See <a href="/docs/transformers/v4.32.0/en/model_doc/wav2vec2#transformers.Wav2Vec2Processor.__call__">Wav2Vec2Processor.<strong>call</strong>()</a> for details.`,name:"input_values"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.input_features",description:`<strong>input_features</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, feature_size)</code>, <em>optional</em>) &#x2014;
Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained
by loading a <code>.flac</code> or <code>.wav</code> audio file into an array of type <code>List[float]</code> or a <code>numpy.ndarray</code>, <em>e.g.</em>
via the soundfile library (<code>pip install soundfile</code>). To prepare the array into <code>input_features</code>, the
<a href="/docs/transformers/v4.32.0/en/model_doc/speech_to_text#transformers.Speech2TextFeatureExtractor">Speech2TextFeatureExtractor</a> should be used for extracting the fbank features, padding and conversion
into a tensor of type <code>torch.FloatTensor</code>. See <a href="/docs/transformers/v4.32.0/en/model_doc/speech_to_text#transformers.Speech2TextFeatureExtractor.__call__"><strong>call</strong>()</a>`,name:"input_features"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, the model will return a <code>~utils.Seq2SeqLMOutput</code> instead of a plain tuple.`,name:"return_dict"},{anchor:"transformers.SpeechEncoderDecoderModel.forward.kwargs",description:`<strong>kwargs</strong> (<em>optional</em>) &#x2014; Remaining dictionary of keyword arguments. Keyword arguments come in two flavors:</p>
<ul>
<li>Without a prefix which will be input as <code>**encoder_kwargs</code> for the encoder forward function.</li>
<li>With a <em>decoder_</em> prefix which will be input as <code>**decoder_kwargs</code> for the decoder forward function.</li>
</ul>`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py#L442",returnDescription:`
<p>A <a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_outputs.Seq2SeqLMOutput"
>transformers.modeling_outputs.Seq2SeqLMOutput</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderConfig"
>SpeechEncoderDecoderConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>loss</strong> (<code>torch.FloatTensor</code> of shape <code>(1,)</code>, <em>optional</em>, returned when <code>labels</code> is provided) \u2014 Language modeling loss.</p>
</li>
<li>
<p><strong>logits</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) \u2014 Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(torch.FloatTensor))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) \u2014 Tuple of <code>tuple(torch.FloatTensor)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>decoder_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>decoder_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
<li>
<p><strong>cross_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder\u2019s cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.</p>
</li>
<li>
<p><strong>encoder_last_hidden_state</strong> (<code>torch.FloatTensor</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) \u2014 Sequence of hidden-states at the output of the last layer of the encoder of the model.</p>
</li>
<li>
<p><strong>encoder_hidden_states</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for the output of the embeddings, if the model has an embedding layer, +
one for the output of each layer) of shape <code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>encoder_attentions</strong> (<code>tuple(torch.FloatTensor)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>torch.FloatTensor</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_outputs.Seq2SeqLMOutput"
>transformers.modeling_outputs.Seq2SeqLMOutput</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),ve=new oc({props:{$$slots:{default:[lc]},$$scope:{ctx:V}}}),Me=new Kt({props:{anchor:"transformers.SpeechEncoderDecoderModel.forward.example",$$slots:{default:[cc]},$$scope:{ctx:V}}}),so=new Ve({props:{name:"from_encoder_decoder_pretrained",anchor:"transformers.SpeechEncoderDecoderModel.from_encoder_decoder_pretrained",parameters:[{name:"encoder_pretrained_model_name_or_path",val:": str = None"},{name:"decoder_pretrained_model_name_or_path",val:": str = None"},{name:"*model_args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.SpeechEncoderDecoderModel.from_encoder_decoder_pretrained.encoder_pretrained_model_name_or_path",description:`<strong>encoder_pretrained_model_name_or_path</strong> (<code>str</code>, <em>optional</em>) &#x2014;
Information necessary to initiate the encoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co.
Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or namespaced under a
user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
<li>A path or url to a <em>tensorflow index checkpoint file</em> (e.g, <code>./tf_model/model.ckpt.index</code>). In
this case, <code>from_tf</code> should be set to <code>True</code> and a configuration object should be provided as
<code>config</code> argument. This loading path is slower than converting the TensorFlow checkpoint in a
PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.</li>
</ul>`,name:"encoder_pretrained_model_name_or_path"},{anchor:"transformers.SpeechEncoderDecoderModel.from_encoder_decoder_pretrained.decoder_pretrained_model_name_or_path",description:`<strong>decoder_pretrained_model_name_or_path</strong> (<code>str</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Information necessary to initiate the decoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co.
Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or namespaced under a
user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
<li>A path or url to a <em>tensorflow index checkpoint file</em> (e.g, <code>./tf_model/model.ckpt.index</code>). In
this case, <code>from_tf</code> should be set to <code>True</code> and a configuration object should be provided as
<code>config</code> argument. This loading path is slower than converting the TensorFlow checkpoint in a
PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.</li>
</ul>`,name:"decoder_pretrained_model_name_or_path"},{anchor:"transformers.SpeechEncoderDecoderModel.from_encoder_decoder_pretrained.model_args",description:`<strong>model_args</strong> (remaining positional arguments, <em>optional</em>) &#x2014;
All remaning positional arguments will be passed to the underlying model&#x2019;s <code>__init__</code> method.`,name:"model_args"},{anchor:"transformers.SpeechEncoderDecoderModel.from_encoder_decoder_pretrained.kwargs",description:`<strong>kwargs</strong> (remaining dictionary of keyword arguments, <em>optional</em>) &#x2014;
Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
<code>output_attentions=True</code>).</p>
<ul>
<li>To update the encoder configuration, use the prefix <em>encoder_</em> for each configuration parameter.</li>
<li>To update the decoder configuration, use the prefix <em>decoder_</em> for each configuration parameter.</li>
<li>To update the parent model configuration, do not use a prefix for each configuration parameter.</li>
</ul>
<p>Behaves differently depending on whether a <code>config</code> is provided or automatically loaded.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py#L287"}}),we=new Kt({props:{anchor:"transformers.SpeechEncoderDecoderModel.from_encoder_decoder_pretrained.example",$$slots:{default:[ic]},$$scope:{ctx:V}}}),lo=new $e({}),co=new Ve({props:{name:"class transformers.FlaxSpeechEncoderDecoderModel",anchor:"transformers.FlaxSpeechEncoderDecoderModel",parameters:[{name:"config",val:": SpeechEncoderDecoderConfig"},{name:"input_shape",val:": typing.Optional[typing.Tuple] = None"},{name:"seed",val:": int = 0"},{name:"dtype",val:": dtype = <class 'jax.numpy.float32'>"},{name:"_do_init",val:": bool = True"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.FlaxSpeechEncoderDecoderModel.config",description:`<strong>config</strong> (<a href="/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderConfig">SpeechEncoderDecoderConfig</a>) &#x2014; Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
configuration. Check out the <a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.FlaxPreTrainedModel.from_pretrained">from_pretrained()</a> method to load the model weights.`,name:"config"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.dtype",description:`<strong>dtype</strong> (<code>jax.numpy.dtype</code>, <em>optional</em>, defaults to <code>jax.numpy.float32</code>) &#x2014;
The data type of the computation. Can be one of <code>jax.numpy.float32</code>, <code>jax.numpy.float16</code> (on GPUs) and
<code>jax.numpy.bfloat16</code> (on TPUs).</p>
<p>This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given <code>dtype</code>.</p>
<p><strong>Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.</strong></p>
<p>If you wish to change the dtype of the model parameters, see <a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.FlaxPreTrainedModel.to_fp16">to_fp16()</a> and
<a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.FlaxPreTrainedModel.to_bf16">to_bf16()</a>.`,name:"dtype"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/speech_encoder_decoder/modeling_flax_speech_encoder_decoder.py#L329"}}),_o=new Ve({props:{name:"__call__",anchor:"transformers.FlaxSpeechEncoderDecoderModel.__call__",parameters:[{name:"inputs",val:": Array"},{name:"attention_mask",val:": typing.Optional[jax.Array] = None"},{name:"decoder_input_ids",val:": typing.Optional[jax.Array] = None"},{name:"decoder_attention_mask",val:": typing.Optional[jax.Array] = None"},{name:"decoder_position_ids",val:": typing.Optional[jax.Array] = None"},{name:"output_attentions",val:": typing.Optional[bool] = None"},{name:"output_hidden_states",val:": typing.Optional[bool] = None"},{name:"return_dict",val:": typing.Optional[bool] = None"},{name:"train",val:": bool = False"},{name:"freeze_feature_encoder",val:": bool = False"},{name:"params",val:": dict = None"},{name:"dropout_rng",val:": PRNGKey = None"}],parametersDescription:[{anchor:"transformers.FlaxSpeechEncoderDecoderModel.__call__.inputs",description:`<strong>inputs</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, sequence_length)</code> or <code>(batch_size, sequence_length, feature_dim)</code>, <em>optional</em>) &#x2014;
Float values of input raw speech waveform or speech features. Values can be obtained by loading a <code>.flac</code>
or <code>.wav</code> audio file into an array of type <code>List[float]</code> or a <code>numpy.ndarray</code>, <em>e.g.</em> via the soundfile
library (<code>pip install soundfile</code>). To prepare the array into <code>inputs</code>, either the <a href="/docs/transformers/v4.32.0/en/model_doc/wav2vec2#transformers.Wav2Vec2Processor">Wav2Vec2Processor</a> or
<a href="/docs/transformers/v4.32.0/en/model_doc/speech_to_text#transformers.Speech2TextProcessor">Speech2TextProcessor</a> should be used for padding and conversion into a tensor of type
<code>torch.FloatTensor</code>.`,name:"inputs"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.__call__.attention_mask",description:`<strong>attention_mask</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Mask to avoid performing attention on padding token indices. Mask values selected in <code>[0, 1]</code>:</p>
<ul>
<li>1 for tokens that are <strong>not masked</strong>,</li>
<li>0 for tokens that are <strong>masked</strong>.</li>
</ul>
<p><a href="../glossary#attention-mask">What are attention masks?</a>`,name:"attention_mask"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.__call__.decoder_input_ids",description:`<strong>decoder_input_ids</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of decoder input sequence tokens in the vocabulary.</p>
<p>Indices can be obtained using <a href="/docs/transformers/v4.32.0/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a>. See <a href="/docs/transformers/v4.32.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.encode">PreTrainedTokenizer.encode()</a> and
<a href="/docs/transformers/v4.32.0/en/internal/tokenization_utils#transformers.PreTrainedTokenizerBase.__call__">PreTrainedTokenizer.<strong>call</strong>()</a> for details.</p>
<p><a href="../glossary#input-ids">What are input IDs?</a></p>
<p>If <code>past_key_values</code> is used, optionally only the last <code>decoder_input_ids</code> have to be input (see
<code>past_key_values</code>).</p>
<p>For sequence to sequence training, <code>decoder_input_ids</code> should be provided. <code>decoder_input_ids</code> should be
created outside of the model by shifting the <code>labels</code> to the right, replacing -100 by the <code>pad_token_id</code>
and prepending them with the <code>decoder_start_token_id</code>.`,name:"decoder_input_ids"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.__call__.decoder_attention_mask",description:`<strong>decoder_attention_mask</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, target_sequence_length)</code>, <em>optional</em>) &#x2014;
Default behavior: generate a tensor that ignores pad tokens in <code>decoder_input_ids</code>. Causal mask will also
be used by default.`,name:"decoder_attention_mask"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.__call__.decoder_position_ids",description:`<strong>decoder_position_ids</strong> (<code>numpy.ndarray</code> of shape <code>(batch_size, sequence_length)</code>, <em>optional</em>) &#x2014;
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range <code>[0, config.decoder.max_position_embeddings - 1]</code>.`,name:"decoder_position_ids"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.__call__.output_hidden_states",description:`<strong>output_hidden_states</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
Whether or not to return the hidden states of all layers. See <code>hidden_states</code> under returned tensors for
more detail.`,name:"output_hidden_states"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.__call__.return_dict",description:`<strong>return_dict</strong> (<code>bool</code>, <em>optional</em>) &#x2014;
If set to <code>True</code>, the model will return a <code>~utils.FlaxSeq2SeqLMOutput</code> instead of a plain tuple.`,name:"return_dict"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/speech_encoder_decoder/modeling_flax_speech_encoder_decoder.py#L660",returnDescription:`
<p>A <a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_flax_outputs.FlaxSeq2SeqLMOutput"
>transformers.modeling_flax_outputs.FlaxSeq2SeqLMOutput</a> or a tuple of
<code>torch.FloatTensor</code> (if <code>return_dict=False</code> is passed or when <code>config.return_dict=False</code>) comprising various
elements depending on the configuration (<a
  href="/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderConfig"
>SpeechEncoderDecoderConfig</a>) and inputs.</p>
<ul>
<li>
<p><strong>logits</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, sequence_length, config.vocab_size)</code>) \u2014 Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).</p>
</li>
<li>
<p><strong>past_key_values</strong> (<code>tuple(tuple(jnp.ndarray))</code>, <em>optional</em>, returned when <code>use_cache=True</code> is passed or when <code>config.use_cache=True</code>) \u2014 Tuple of <code>tuple(jnp.ndarray)</code> of length <code>config.n_layers</code>, with each tuple having 2 tensors of shape
<code>(batch_size, num_heads, sequence_length, embed_size_per_head)</code>) and 2 additional tensors of shape
<code>(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)</code>.</p>
<p>Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see <code>past_key_values</code> input) to speed up sequential decoding.</p>
</li>
<li>
<p><strong>decoder_hidden_states</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for the output of the embeddings + one for the output of each layer) of shape
<code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>decoder_attentions</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
<li>
<p><strong>cross_attentions</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the decoder\u2019s cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.</p>
</li>
<li>
<p><strong>encoder_last_hidden_state</strong> (<code>jnp.ndarray</code> of shape <code>(batch_size, sequence_length, hidden_size)</code>, <em>optional</em>) \u2014 Sequence of hidden-states at the output of the last layer of the encoder of the model.</p>
</li>
<li>
<p><strong>encoder_hidden_states</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_hidden_states=True</code> is passed or when <code>config.output_hidden_states=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for the output of the embeddings + one for the output of each layer) of shape
<code>(batch_size, sequence_length, hidden_size)</code>.</p>
<p>Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.</p>
</li>
<li>
<p><strong>encoder_attentions</strong> (<code>tuple(jnp.ndarray)</code>, <em>optional</em>, returned when <code>output_attentions=True</code> is passed or when <code>config.output_attentions=True</code>) \u2014 Tuple of <code>jnp.ndarray</code> (one for each layer) of shape <code>(batch_size, num_heads, sequence_length, sequence_length)</code>.</p>
<p>Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.</p>
</li>
</ul>
`,returnType:`
<p><a
  href="/docs/transformers/v4.32.0/en/main_classes/output#transformers.modeling_flax_outputs.FlaxSeq2SeqLMOutput"
>transformers.modeling_flax_outputs.FlaxSeq2SeqLMOutput</a> or <code>tuple(torch.FloatTensor)</code></p>
`}}),Te=new oc({props:{$$slots:{default:[pc]},$$scope:{ctx:V}}}),ke=new Kt({props:{anchor:"transformers.FlaxSpeechEncoderDecoderModel.__call__.example",$$slots:{default:[hc]},$$scope:{ctx:V}}}),yo=new Ve({props:{name:"from_encoder_decoder_pretrained",anchor:"transformers.FlaxSpeechEncoderDecoderModel.from_encoder_decoder_pretrained",parameters:[{name:"encoder_pretrained_model_name_or_path",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"decoder_pretrained_model_name_or_path",val:": typing.Union[str, os.PathLike, NoneType] = None"},{name:"*model_args",val:""},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.FlaxSpeechEncoderDecoderModel.from_encoder_decoder_pretrained.encoder_pretrained_model_name_or_path",description:`<strong>encoder_pretrained_model_name_or_path</strong> (<code>Union[str, os.PathLike]</code>, <em>optional</em>) &#x2014;
Information necessary to initiate the encoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co.
Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or namespaced under a
user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.FlaxPreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
</ul>`,name:"encoder_pretrained_model_name_or_path"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.from_encoder_decoder_pretrained.decoder_pretrained_model_name_or_path",description:`<strong>decoder_pretrained_model_name_or_path</strong> (<code>Union[str, os.PathLike]</code>, <em>optional</em>, defaults to <code>None</code>) &#x2014;
Information necessary to initiate the decoder. Can be either:</p>
<ul>
<li>A string, the <em>model id</em> of a pretrained model hosted inside a model repo on huggingface.co.
Valid model ids can be located at the root-level, like <code>bert-base-uncased</code>, or namespaced under a
user or organization name, like <code>dbmdz/bert-base-german-cased</code>.</li>
<li>A path to a <em>directory</em> containing model weights saved using
<a href="/docs/transformers/v4.32.0/en/main_classes/model#transformers.FlaxPreTrainedModel.save_pretrained">save_pretrained()</a>, e.g., <code>./my_model_directory/</code>.</li>
</ul>`,name:"decoder_pretrained_model_name_or_path"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.from_encoder_decoder_pretrained.model_args",description:`<strong>model_args</strong> (remaining positional arguments, <em>optional</em>) &#x2014;
All remaning positional arguments will be passed to the underlying model&#x2019;s <code>__init__</code> method.`,name:"model_args"},{anchor:"transformers.FlaxSpeechEncoderDecoderModel.from_encoder_decoder_pretrained.kwargs",description:`<strong>kwargs</strong> (remaining dictionary of keyword arguments, <em>optional</em>) &#x2014;
Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
<code>output_attentions=True</code>).</p>
<ul>
<li>To update the encoder configuration, use the prefix <em>encoder_</em> for each configuration parameter.</li>
<li>To update the decoder configuration, use the prefix <em>decoder_</em> for each configuration parameter.</li>
<li>To update the parent model configuration, do not use a prefix for each configuration parameter.</li>
</ul>
<p>Behaves differently depending on whether a <code>config</code> is provided or automatically loaded.`,name:"kwargs"}],source:"https://github.com/huggingface/transformers/blob/v4.32.0/src/transformers/models/speech_encoder_decoder/modeling_flax_speech_encoder_decoder.py#L782"}}),Ee=new Kt({props:{anchor:"transformers.FlaxSpeechEncoderDecoderModel.from_encoder_decoder_pretrained.example",$$slots:{default:[mc]},$$scope:{ctx:V}}}),{c(){m=r("meta"),b=i(),_=r("h1"),f=r("a"),y=r("span"),v(l.$$.fragment),g=i(),I=r("span"),qn=n("Speech Encoder Decoder Models"),on=i(),x=r("p"),An=n("The "),jo=r("a"),Pn=n("SpeechEncoderDecoderModel"),Qn=n(` can be used to initialize a speech-to-text model
with any pretrained speech autoencoding model as the encoder (`),st=r("em"),Hn=n("e.g."),Ln=i(),To=r("a"),On=n("Wav2Vec2"),Kn=n(", "),ko=r("a"),ea=n("Hubert"),oa=n(") and any pretrained autoregressive model as the decoder."),tn=i(),Y=r("p"),ta=n(`The effectiveness of initializing speech-sequence-to-text-sequence models with pretrained checkpoints for speech
recognition and speech translation has `),dt=r("em"),na=n("e.g."),aa=n(" been shown in "),xe=r("a"),ra=n(`Large-Scale Self- and Semi-Supervised Learning for Speech
Translation`),sa=n(` by Changhan Wang, Anne Wu, Juan Pino, Alexei Baevski, Michael Auli,
Alexis Conneau.`),nn=i(),N=r("p"),da=n("An example of how to use a "),Eo=r("a"),la=n("SpeechEncoderDecoderModel"),ca=n(" for inference can be seen in "),Zo=r("a"),ia=n("Speech2Text2"),pa=n("."),an=i(),Q=r("h2"),ce=r("a"),lt=r("span"),v(Re.$$.fragment),ha=i(),Ge=r("span"),ma=n("Randomly initializing "),ct=r("code"),fa=n("SpeechEncoderDecoderModel"),ua=n(" from model configurations."),rn=i(),D=r("p"),Jo=r("a"),ga=n("SpeechEncoderDecoderModel"),_a=n(" can be randomly initialized from an encoder and a decoder config. In the following example, we show how to do this using the default "),Wo=r("a"),ya=n("Wav2Vec2Model"),ba=n(` configuration for the encoder
and the default `),it=r("code"),va=n("BertForCausalLM"),Ma=n(" configuration for the decoder."),sn=i(),v(Ce.$$.fragment),dn=i(),H=r("h2"),ie=r("a"),pt=r("span"),v(Fe.$$.fragment),wa=i(),Be=r("span"),ja=n("Initialising "),ht=r("code"),Ta=n("SpeechEncoderDecoderModel"),ka=n(" from a pretrained encoder and a pretrained decoder."),ln=i(),E=r("p"),So=r("a"),Ea=n("SpeechEncoderDecoderModel"),Za=n(" can be initialized from a pretrained encoder checkpoint and a pretrained decoder checkpoint. Note that any pretrained Transformer-based speech model, "),mt=r("em"),Ja=n("e.g."),Wa=i(),Uo=r("a"),Sa=n("Wav2Vec2"),Ua=n(", "),Vo=r("a"),Va=n("Hubert"),$a=n(" can serve as the encoder and both pretrained auto-encoding models, "),ft=r("em"),xa=n("e.g."),Ra=n(" BERT, pretrained causal language models, "),ut=r("em"),Ga=n("e.g."),Ca=n(" GPT2, as well as the pretrained decoder part of sequence-to-sequence models, "),gt=r("em"),Fa=n("e.g."),Ba=n(` decoder of BART, can be used as the decoder.
Depending on which architecture you choose as the decoder, the cross-attention layers might be randomly initialized.
Initializing `),$o=r("a"),za=n("SpeechEncoderDecoderModel"),Ia=n(" from a pretrained encoder and decoder checkpoint requires the model to be fine-tuned on a downstream task, as has been shown in "),pe=r("a"),Da=n("the "),_t=r("em"),Ya=n("Warm-starting-encoder-decoder blog post"),Na=n(`.
To do so, the `),yt=r("code"),Xa=n("SpeechEncoderDecoderModel"),qa=n(" class provides a "),xo=r("a"),Aa=n("SpeechEncoderDecoderModel.from_encoder_decoder_pretrained()"),Pa=n(" method."),cn=i(),v(ze.$$.fragment),pn=i(),L=r("h2"),he=r("a"),bt=r("span"),v(Ie.$$.fragment),Qa=i(),De=r("span"),Ha=n("Loading an existing "),vt=r("code"),La=n("SpeechEncoderDecoderModel"),Oa=n(" checkpoint and perform inference."),hn=i(),G=r("p"),Ka=n("To load fine-tuned checkpoints of the "),Mt=r("code"),er=n("SpeechEncoderDecoderModel"),or=n(" class, "),Ro=r("a"),tr=n("SpeechEncoderDecoderModel"),nr=n(" provides the "),wt=r("code"),ar=n("from_pretrained(...)"),rr=n(" method just like any other model architecture in Transformers."),mn=i(),me=r("p"),sr=n("To perform inference, one uses the "),jt=r("code"),dr=n("generate"),lr=n(" method, which allows to autoregressively generate text. This method supports various forms of decoding, such as greedy, beam search and multinomial sampling."),fn=i(),v(Ye.$$.fragment),un=i(),O=r("h2"),fe=r("a"),Tt=r("span"),v(Ne.$$.fragment),cr=i(),kt=r("span"),ir=n("Training"),gn=i(),C=r("p"),pr=n(`Once the model is created, it can be fine-tuned similar to BART, T5 or any other encoder-decoder model on a dataset of (speech, text) pairs.
As you can see, only 2 inputs are required for the model in order to compute a loss: `),Et=r("code"),hr=n("input_values"),mr=n(` (which are the
speech inputs) and `),Zt=r("code"),fr=n("labels"),ur=n(" (which are the "),Jt=r("code"),gr=n("input_ids"),_r=n(" of the encoded target sequence)."),_n=i(),v(Xe.$$.fragment),yn=i(),K=r("h2"),ue=r("a"),Wt=r("span"),v(qe.$$.fragment),yr=i(),St=r("span"),br=n("SpeechEncoderDecoderConfig"),bn=i(),$=r("div"),v(Ae.$$.fragment),vr=i(),ge=r("p"),Go=r("a"),Mr=n("SpeechEncoderDecoderConfig"),wr=n(` is the configuration class to store the configuration of a
`),Co=r("a"),jr=n("SpeechEncoderDecoderModel"),Tr=n(`. It is used to instantiate an Encoder Decoder model according to the specified
arguments, defining the encoder and decoder configs.`),kr=i(),ee=r("p"),Er=n("Configuration objects inherit from "),Fo=r("a"),Zr=n("PretrainedConfig"),Jr=n(` and can be used to control the model outputs. Read the
documentation from `),Bo=r("a"),Wr=n("PretrainedConfig"),Sr=n(" for more information."),Ur=i(),v(_e.$$.fragment),Vr=i(),ye=r("div"),v(Pe.$$.fragment),$r=i(),Qe=r("p"),xr=n("Instantiate a "),zo=r("a"),Rr=n("SpeechEncoderDecoderConfig"),Gr=n(` (or a derived class) from a pre-trained encoder model
configuration and decoder model configuration.`),vn=i(),oe=r("h2"),be=r("a"),Ut=r("span"),v(He.$$.fragment),Cr=i(),Vt=r("span"),Fr=n("SpeechEncoderDecoderModel"),Mn=i(),Z=r("div"),v(Le.$$.fragment),Br=i(),te=r("p"),zr=n(`This class can be used to initialize a speech-sequence-to-text-sequence model with any pretrained speech
autoencoding model as the encoder and any pretrained text autoregressive model as the decoder. The encoder is
loaded via `),Io=r("a"),Ir=n("from_pretrained()"),Dr=n(` function and the decoder is loaded via
`),Do=r("a"),Yr=n("from_pretrained()"),Nr=n(` function. Cross-attention layers are automatically added to the decoder
and should be fine-tuned on a downstream generative task, like summarization.`),Xr=i(),Oe=r("p"),qr=n(`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),Ke=r("a"),Ar=n(`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),Pr=n(` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),Qr=i(),eo=r("p"),Hr=n("Additionally, in "),oo=r("a"),Lr=n(`Large-Scale Self- and Semi-Supervised Learning for Speech
Translation`),Or=n(` it is shown how leveraging large pretrained speech models for speech
translation yields a significant performance improvement.`),Kr=i(),$t=r("p"),es=n(`After such an Speech-Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other
models (see the examples for more information).`),os=i(),to=r("p"),ts=n("This model inherits from "),Yo=r("a"),ns=n("PreTrainedModel"),as=n(`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),rs=i(),no=r("p"),ss=n("This model is also a PyTorch "),ao=r("a"),ds=n("torch.nn.Module"),ls=n(` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),cs=i(),X=r("p"),No=r("a"),is=n("SpeechEncoderDecoderModel"),ps=n(` is a generic model class that will be instantiated as a transformer architecture with
one of the base model classes of the library as encoder and another one as decoder when created with the
:meth`),xt=r("em"),hs=n("~transformers.AutoModel.from_pretrained"),ms=n(` class method for the encoder and
:meth`),Rt=r("em"),fs=n("~transformers.AutoModelForCausalLM.from_pretrained"),us=n(" class method for the decoder."),gs=i(),F=r("div"),v(ro.$$.fragment),_s=i(),ne=r("p"),ys=n("The "),Xo=r("a"),bs=n("SpeechEncoderDecoderModel"),vs=n(" forward method, overrides the "),Gt=r("code"),Ms=n("__call__"),ws=n(" special method."),js=i(),v(ve.$$.fragment),Ts=i(),v(Me.$$.fragment),ks=i(),B=r("div"),v(so.$$.fragment),Es=i(),Ct=r("p"),Zs=n(`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),Js=i(),ae=r("p"),Ws=n("The model is set in evaluation mode by default using "),Ft=r("code"),Ss=n("model.eval()"),Us=n(` (Dropout modules are deactivated). To train
the model, you need to first set it back in training mode with `),Bt=r("code"),Vs=n("model.train()"),$s=n("."),xs=i(),v(we.$$.fragment),wn=i(),re=r("h2"),je=r("a"),zt=r("span"),v(lo.$$.fragment),Rs=i(),It=r("span"),Gs=n("FlaxSpeechEncoderDecoderModel"),jn=i(),J=r("div"),v(co.$$.fragment),Cs=i(),se=r("p"),Fs=n(`This class can be used to initialize a speech-sequence-to-text-sequence model with any pretrained speech
autoencoding model as the encoder and any pretrained text autoregressive model as the decoder. The encoder is
loaded via `),qo=r("a"),Bs=n("from_pretrained()"),zs=n(` function and the decoder is loaded via
`),Ao=r("a"),Is=n("from_pretrained()"),Ds=n(` function. Cross-attention layers are automatically added to the decoder
and should be fine-tuned on a downstream generative task, like summarization.`),Ys=i(),io=r("p"),Ns=n(`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),po=r("a"),Xs=n(`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),qs=n(` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),As=i(),ho=r("p"),Ps=n("Additionally, in "),mo=r("a"),Qs=n(`Large-Scale Self- and Semi-Supervised Learning for Speech
Translation`),Hs=n(` it is shown how leveraging large pretrained speech models for speech
translation yields a significant performance improvement.`),Ls=i(),Dt=r("p"),Os=n(`After such an Speech-Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other
models (see the examples for more information).`),Ks=i(),fo=r("p"),ed=n("This model inherits from "),Po=r("a"),od=n("FlaxPreTrainedModel"),td=n(`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),nd=i(),uo=r("p"),ad=n(`This model is also a Flax Linen
`),go=r("a"),rd=n("flax.nn.Module"),sd=n(` subclass. Use it as a
regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.`),dd=i(),q=r("p"),Qo=r("a"),ld=n("FlaxSpeechEncoderDecoderModel"),cd=n(` is a generic model class that will be instantiated as a transformer architecture
with the module (flax.nn.Module) of one of the base model classes of the library as encoder module and another one
as decoder module when created with the :meth`),Yt=r("em"),id=n("~transformers.FlaxAutoModel.from_pretrained"),pd=n(` class method for the
encoder and :meth`),Nt=r("em"),hd=n("~transformers.FlaxAutoModelForCausalLM.from_pretrained"),md=n(" class method for the decoder."),fd=i(),z=r("div"),v(_o.$$.fragment),ud=i(),de=r("p"),gd=n("The "),Ho=r("a"),_d=n("FlaxSpeechEncoderDecoderModel"),yd=n(" forward method, overrides the "),Xt=r("code"),bd=n("__call__"),vd=n(" special method."),Md=i(),v(Te.$$.fragment),wd=i(),v(ke.$$.fragment),jd=i(),A=r("div"),v(yo.$$.fragment),Td=i(),qt=r("p"),kd=n(`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),Ed=i(),v(Ee.$$.fragment),this.h()},l(o){const h=rc('[data-svelte="svelte-1phssyn"]',document.head);m=s(h,"META",{name:!0,content:!0}),h.forEach(t),b=p(o),_=s(o,"H1",{class:!0});var bo=d(_);f=s(bo,"A",{id:!0,class:!0,href:!0});var At=d(f);y=s(At,"SPAN",{});var Pt=d(y);M(l.$$.fragment,Pt),Pt.forEach(t),At.forEach(t),g=p(bo),I=s(bo,"SPAN",{});var Qt=d(I);qn=a(Qt,"Speech Encoder Decoder Models"),Qt.forEach(t),bo.forEach(t),on=p(o),x=s(o,"P",{});var R=d(x);An=a(R,"The "),jo=s(R,"A",{href:!0});var Ht=d(jo);Pn=a(Ht,"SpeechEncoderDecoderModel"),Ht.forEach(t),Qn=a(R,` can be used to initialize a speech-to-text model
with any pretrained speech autoencoding model as the encoder (`),st=s(R,"EM",{});var Lt=d(st);Hn=a(Lt,"e.g."),Lt.forEach(t),Ln=p(R),To=s(R,"A",{href:!0});var Jd=d(To);On=a(Jd,"Wav2Vec2"),Jd.forEach(t),Kn=a(R,", "),ko=s(R,"A",{href:!0});var Wd=d(ko);ea=a(Wd,"Hubert"),Wd.forEach(t),oa=a(R,") and any pretrained autoregressive model as the decoder."),R.forEach(t),tn=p(o),Y=s(o,"P",{});var Lo=d(Y);ta=a(Lo,`The effectiveness of initializing speech-sequence-to-text-sequence models with pretrained checkpoints for speech
recognition and speech translation has `),dt=s(Lo,"EM",{});var Sd=d(dt);na=a(Sd,"e.g."),Sd.forEach(t),aa=a(Lo," been shown in "),xe=s(Lo,"A",{href:!0,rel:!0});var Ud=d(xe);ra=a(Ud,`Large-Scale Self- and Semi-Supervised Learning for Speech
Translation`),Ud.forEach(t),sa=a(Lo,` by Changhan Wang, Anne Wu, Juan Pino, Alexei Baevski, Michael Auli,
Alexis Conneau.`),Lo.forEach(t),nn=p(o),N=s(o,"P",{});var Oo=d(N);da=a(Oo,"An example of how to use a "),Eo=s(Oo,"A",{href:!0});var Vd=d(Eo);la=a(Vd,"SpeechEncoderDecoderModel"),Vd.forEach(t),ca=a(Oo," for inference can be seen in "),Zo=s(Oo,"A",{href:!0});var $d=d(Zo);ia=a($d,"Speech2Text2"),$d.forEach(t),pa=a(Oo,"."),Oo.forEach(t),an=p(o),Q=s(o,"H2",{class:!0});var kn=d(Q);ce=s(kn,"A",{id:!0,class:!0,href:!0});var xd=d(ce);lt=s(xd,"SPAN",{});var Rd=d(lt);M(Re.$$.fragment,Rd),Rd.forEach(t),xd.forEach(t),ha=p(kn),Ge=s(kn,"SPAN",{});var En=d(Ge);ma=a(En,"Randomly initializing "),ct=s(En,"CODE",{});var Gd=d(ct);fa=a(Gd,"SpeechEncoderDecoderModel"),Gd.forEach(t),ua=a(En," from model configurations."),En.forEach(t),kn.forEach(t),rn=p(o),D=s(o,"P",{});var vo=d(D);Jo=s(vo,"A",{href:!0});var Cd=d(Jo);ga=a(Cd,"SpeechEncoderDecoderModel"),Cd.forEach(t),_a=a(vo," can be randomly initialized from an encoder and a decoder config. In the following example, we show how to do this using the default "),Wo=s(vo,"A",{href:!0});var Fd=d(Wo);ya=a(Fd,"Wav2Vec2Model"),Fd.forEach(t),ba=a(vo,` configuration for the encoder
and the default `),it=s(vo,"CODE",{});var Bd=d(it);va=a(Bd,"BertForCausalLM"),Bd.forEach(t),Ma=a(vo," configuration for the decoder."),vo.forEach(t),sn=p(o),M(Ce.$$.fragment,o),dn=p(o),H=s(o,"H2",{class:!0});var Zn=d(H);ie=s(Zn,"A",{id:!0,class:!0,href:!0});var zd=d(ie);pt=s(zd,"SPAN",{});var Id=d(pt);M(Fe.$$.fragment,Id),Id.forEach(t),zd.forEach(t),wa=p(Zn),Be=s(Zn,"SPAN",{});var Jn=d(Be);ja=a(Jn,"Initialising "),ht=s(Jn,"CODE",{});var Dd=d(ht);Ta=a(Dd,"SpeechEncoderDecoderModel"),Dd.forEach(t),ka=a(Jn," from a pretrained encoder and a pretrained decoder."),Jn.forEach(t),Zn.forEach(t),ln=p(o),E=s(o,"P",{});var W=d(E);So=s(W,"A",{href:!0});var Yd=d(So);Ea=a(Yd,"SpeechEncoderDecoderModel"),Yd.forEach(t),Za=a(W," can be initialized from a pretrained encoder checkpoint and a pretrained decoder checkpoint. Note that any pretrained Transformer-based speech model, "),mt=s(W,"EM",{});var Nd=d(mt);Ja=a(Nd,"e.g."),Nd.forEach(t),Wa=p(W),Uo=s(W,"A",{href:!0});var Xd=d(Uo);Sa=a(Xd,"Wav2Vec2"),Xd.forEach(t),Ua=a(W,", "),Vo=s(W,"A",{href:!0});var qd=d(Vo);Va=a(qd,"Hubert"),qd.forEach(t),$a=a(W," can serve as the encoder and both pretrained auto-encoding models, "),ft=s(W,"EM",{});var Ad=d(ft);xa=a(Ad,"e.g."),Ad.forEach(t),Ra=a(W," BERT, pretrained causal language models, "),ut=s(W,"EM",{});var Pd=d(ut);Ga=a(Pd,"e.g."),Pd.forEach(t),Ca=a(W," GPT2, as well as the pretrained decoder part of sequence-to-sequence models, "),gt=s(W,"EM",{});var Qd=d(gt);Fa=a(Qd,"e.g."),Qd.forEach(t),Ba=a(W,` decoder of BART, can be used as the decoder.
Depending on which architecture you choose as the decoder, the cross-attention layers might be randomly initialized.
Initializing `),$o=s(W,"A",{href:!0});var Hd=d($o);za=a(Hd,"SpeechEncoderDecoderModel"),Hd.forEach(t),Ia=a(W," from a pretrained encoder and decoder checkpoint requires the model to be fine-tuned on a downstream task, as has been shown in "),pe=s(W,"A",{href:!0,rel:!0});var Zd=d(pe);Da=a(Zd,"the "),_t=s(Zd,"EM",{});var Ld=d(_t);Ya=a(Ld,"Warm-starting-encoder-decoder blog post"),Ld.forEach(t),Zd.forEach(t),Na=a(W,`.
To do so, the `),yt=s(W,"CODE",{});var Od=d(yt);Xa=a(Od,"SpeechEncoderDecoderModel"),Od.forEach(t),qa=a(W," class provides a "),xo=s(W,"A",{href:!0});var Kd=d(xo);Aa=a(Kd,"SpeechEncoderDecoderModel.from_encoder_decoder_pretrained()"),Kd.forEach(t),Pa=a(W," method."),W.forEach(t),cn=p(o),M(ze.$$.fragment,o),pn=p(o),L=s(o,"H2",{class:!0});var Wn=d(L);he=s(Wn,"A",{id:!0,class:!0,href:!0});var el=d(he);bt=s(el,"SPAN",{});var ol=d(bt);M(Ie.$$.fragment,ol),ol.forEach(t),el.forEach(t),Qa=p(Wn),De=s(Wn,"SPAN",{});var Sn=d(De);Ha=a(Sn,"Loading an existing "),vt=s(Sn,"CODE",{});var tl=d(vt);La=a(tl,"SpeechEncoderDecoderModel"),tl.forEach(t),Oa=a(Sn," checkpoint and perform inference."),Sn.forEach(t),Wn.forEach(t),hn=p(o),G=s(o,"P",{});var Ze=d(G);Ka=a(Ze,"To load fine-tuned checkpoints of the "),Mt=s(Ze,"CODE",{});var nl=d(Mt);er=a(nl,"SpeechEncoderDecoderModel"),nl.forEach(t),or=a(Ze," class, "),Ro=s(Ze,"A",{href:!0});var al=d(Ro);tr=a(al,"SpeechEncoderDecoderModel"),al.forEach(t),nr=a(Ze," provides the "),wt=s(Ze,"CODE",{});var rl=d(wt);ar=a(rl,"from_pretrained(...)"),rl.forEach(t),rr=a(Ze," method just like any other model architecture in Transformers."),Ze.forEach(t),mn=p(o),me=s(o,"P",{});var Un=d(me);sr=a(Un,"To perform inference, one uses the "),jt=s(Un,"CODE",{});var sl=d(jt);dr=a(sl,"generate"),sl.forEach(t),lr=a(Un," method, which allows to autoregressively generate text. This method supports various forms of decoding, such as greedy, beam search and multinomial sampling."),Un.forEach(t),fn=p(o),M(Ye.$$.fragment,o),un=p(o),O=s(o,"H2",{class:!0});var Vn=d(O);fe=s(Vn,"A",{id:!0,class:!0,href:!0});var dl=d(fe);Tt=s(dl,"SPAN",{});var ll=d(Tt);M(Ne.$$.fragment,ll),ll.forEach(t),dl.forEach(t),cr=p(Vn),kt=s(Vn,"SPAN",{});var cl=d(kt);ir=a(cl,"Training"),cl.forEach(t),Vn.forEach(t),gn=p(o),C=s(o,"P",{});var Je=d(C);pr=a(Je,`Once the model is created, it can be fine-tuned similar to BART, T5 or any other encoder-decoder model on a dataset of (speech, text) pairs.
As you can see, only 2 inputs are required for the model in order to compute a loss: `),Et=s(Je,"CODE",{});var il=d(Et);hr=a(il,"input_values"),il.forEach(t),mr=a(Je,` (which are the
speech inputs) and `),Zt=s(Je,"CODE",{});var pl=d(Zt);fr=a(pl,"labels"),pl.forEach(t),ur=a(Je," (which are the "),Jt=s(Je,"CODE",{});var hl=d(Jt);gr=a(hl,"input_ids"),hl.forEach(t),_r=a(Je," of the encoded target sequence)."),Je.forEach(t),_n=p(o),M(Xe.$$.fragment,o),yn=p(o),K=s(o,"H2",{class:!0});var $n=d(K);ue=s($n,"A",{id:!0,class:!0,href:!0});var ml=d(ue);Wt=s(ml,"SPAN",{});var fl=d(Wt);M(qe.$$.fragment,fl),fl.forEach(t),ml.forEach(t),yr=p($n),St=s($n,"SPAN",{});var ul=d(St);br=a(ul,"SpeechEncoderDecoderConfig"),ul.forEach(t),$n.forEach(t),bn=p(o),$=s(o,"DIV",{class:!0});var P=d($);M(Ae.$$.fragment,P),vr=p(P),ge=s(P,"P",{});var Ot=d(ge);Go=s(Ot,"A",{href:!0});var gl=d(Go);Mr=a(gl,"SpeechEncoderDecoderConfig"),gl.forEach(t),wr=a(Ot,` is the configuration class to store the configuration of a
`),Co=s(Ot,"A",{href:!0});var _l=d(Co);jr=a(_l,"SpeechEncoderDecoderModel"),_l.forEach(t),Tr=a(Ot,`. It is used to instantiate an Encoder Decoder model according to the specified
arguments, defining the encoder and decoder configs.`),Ot.forEach(t),kr=p(P),ee=s(P,"P",{});var Ko=d(ee);Er=a(Ko,"Configuration objects inherit from "),Fo=s(Ko,"A",{href:!0});var yl=d(Fo);Zr=a(yl,"PretrainedConfig"),yl.forEach(t),Jr=a(Ko,` and can be used to control the model outputs. Read the
documentation from `),Bo=s(Ko,"A",{href:!0});var bl=d(Bo);Wr=a(bl,"PretrainedConfig"),bl.forEach(t),Sr=a(Ko," for more information."),Ko.forEach(t),Ur=p(P),M(_e.$$.fragment,P),Vr=p(P),ye=s(P,"DIV",{class:!0});var xn=d(ye);M(Pe.$$.fragment,xn),$r=p(xn),Qe=s(xn,"P",{});var Rn=d(Qe);xr=a(Rn,"Instantiate a "),zo=s(Rn,"A",{href:!0});var vl=d(zo);Rr=a(vl,"SpeechEncoderDecoderConfig"),vl.forEach(t),Gr=a(Rn,` (or a derived class) from a pre-trained encoder model
configuration and decoder model configuration.`),Rn.forEach(t),xn.forEach(t),P.forEach(t),vn=p(o),oe=s(o,"H2",{class:!0});var Gn=d(oe);be=s(Gn,"A",{id:!0,class:!0,href:!0});var Ml=d(be);Ut=s(Ml,"SPAN",{});var wl=d(Ut);M(He.$$.fragment,wl),wl.forEach(t),Ml.forEach(t),Cr=p(Gn),Vt=s(Gn,"SPAN",{});var jl=d(Vt);Fr=a(jl,"SpeechEncoderDecoderModel"),jl.forEach(t),Gn.forEach(t),Mn=p(o),Z=s(o,"DIV",{class:!0});var S=d(Z);M(Le.$$.fragment,S),Br=p(S),te=s(S,"P",{});var et=d(te);zr=a(et,`This class can be used to initialize a speech-sequence-to-text-sequence model with any pretrained speech
autoencoding model as the encoder and any pretrained text autoregressive model as the decoder. The encoder is
loaded via `),Io=s(et,"A",{href:!0});var Tl=d(Io);Ir=a(Tl,"from_pretrained()"),Tl.forEach(t),Dr=a(et,` function and the decoder is loaded via
`),Do=s(et,"A",{href:!0});var kl=d(Do);Yr=a(kl,"from_pretrained()"),kl.forEach(t),Nr=a(et,` function. Cross-attention layers are automatically added to the decoder
and should be fine-tuned on a downstream generative task, like summarization.`),et.forEach(t),Xr=p(S),Oe=s(S,"P",{});var Cn=d(Oe);qr=a(Cn,`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),Ke=s(Cn,"A",{href:!0,rel:!0});var El=d(Ke);Ar=a(El,`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),El.forEach(t),Pr=a(Cn,` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),Cn.forEach(t),Qr=p(S),eo=s(S,"P",{});var Fn=d(eo);Hr=a(Fn,"Additionally, in "),oo=s(Fn,"A",{href:!0,rel:!0});var Zl=d(oo);Lr=a(Zl,`Large-Scale Self- and Semi-Supervised Learning for Speech
Translation`),Zl.forEach(t),Or=a(Fn,` it is shown how leveraging large pretrained speech models for speech
translation yields a significant performance improvement.`),Fn.forEach(t),Kr=p(S),$t=s(S,"P",{});var Jl=d($t);es=a(Jl,`After such an Speech-Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other
models (see the examples for more information).`),Jl.forEach(t),os=p(S),to=s(S,"P",{});var Bn=d(to);ts=a(Bn,"This model inherits from "),Yo=s(Bn,"A",{href:!0});var Wl=d(Yo);ns=a(Wl,"PreTrainedModel"),Wl.forEach(t),as=a(Bn,`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),Bn.forEach(t),rs=p(S),no=s(S,"P",{});var zn=d(no);ss=a(zn,"This model is also a PyTorch "),ao=s(zn,"A",{href:!0,rel:!0});var Sl=d(ao);ds=a(Sl,"torch.nn.Module"),Sl.forEach(t),ls=a(zn,` subclass.
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
and behavior.`),zn.forEach(t),cs=p(S),X=s(S,"P",{});var Mo=d(X);No=s(Mo,"A",{href:!0});var Ul=d(No);is=a(Ul,"SpeechEncoderDecoderModel"),Ul.forEach(t),ps=a(Mo,` is a generic model class that will be instantiated as a transformer architecture with
one of the base model classes of the library as encoder and another one as decoder when created with the
:meth`),xt=s(Mo,"EM",{});var Vl=d(xt);hs=a(Vl,"~transformers.AutoModel.from_pretrained"),Vl.forEach(t),ms=a(Mo,` class method for the encoder and
:meth`),Rt=s(Mo,"EM",{});var $l=d(Rt);fs=a($l,"~transformers.AutoModelForCausalLM.from_pretrained"),$l.forEach(t),us=a(Mo," class method for the decoder."),Mo.forEach(t),gs=p(S),F=s(S,"DIV",{class:!0});var We=d(F);M(ro.$$.fragment,We),_s=p(We),ne=s(We,"P",{});var ot=d(ne);ys=a(ot,"The "),Xo=s(ot,"A",{href:!0});var xl=d(Xo);bs=a(xl,"SpeechEncoderDecoderModel"),xl.forEach(t),vs=a(ot," forward method, overrides the "),Gt=s(ot,"CODE",{});var Rl=d(Gt);Ms=a(Rl,"__call__"),Rl.forEach(t),ws=a(ot," special method."),ot.forEach(t),js=p(We),M(ve.$$.fragment,We),Ts=p(We),M(Me.$$.fragment,We),We.forEach(t),ks=p(S),B=s(S,"DIV",{class:!0});var Se=d(B);M(so.$$.fragment,Se),Es=p(Se),Ct=s(Se,"P",{});var Gl=d(Ct);Zs=a(Gl,`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),Gl.forEach(t),Js=p(Se),ae=s(Se,"P",{});var tt=d(ae);Ws=a(tt,"The model is set in evaluation mode by default using "),Ft=s(tt,"CODE",{});var Cl=d(Ft);Ss=a(Cl,"model.eval()"),Cl.forEach(t),Us=a(tt,` (Dropout modules are deactivated). To train
the model, you need to first set it back in training mode with `),Bt=s(tt,"CODE",{});var Fl=d(Bt);Vs=a(Fl,"model.train()"),Fl.forEach(t),$s=a(tt,"."),tt.forEach(t),xs=p(Se),M(we.$$.fragment,Se),Se.forEach(t),S.forEach(t),wn=p(o),re=s(o,"H2",{class:!0});var In=d(re);je=s(In,"A",{id:!0,class:!0,href:!0});var Bl=d(je);zt=s(Bl,"SPAN",{});var zl=d(zt);M(lo.$$.fragment,zl),zl.forEach(t),Bl.forEach(t),Rs=p(In),It=s(In,"SPAN",{});var Il=d(It);Gs=a(Il,"FlaxSpeechEncoderDecoderModel"),Il.forEach(t),In.forEach(t),jn=p(o),J=s(o,"DIV",{class:!0});var U=d(J);M(co.$$.fragment,U),Cs=p(U),se=s(U,"P",{});var nt=d(se);Fs=a(nt,`This class can be used to initialize a speech-sequence-to-text-sequence model with any pretrained speech
autoencoding model as the encoder and any pretrained text autoregressive model as the decoder. The encoder is
loaded via `),qo=s(nt,"A",{href:!0});var Dl=d(qo);Bs=a(Dl,"from_pretrained()"),Dl.forEach(t),zs=a(nt,` function and the decoder is loaded via
`),Ao=s(nt,"A",{href:!0});var Yl=d(Ao);Is=a(Yl,"from_pretrained()"),Yl.forEach(t),Ds=a(nt,` function. Cross-attention layers are automatically added to the decoder
and should be fine-tuned on a downstream generative task, like summarization.`),nt.forEach(t),Ys=p(U),io=s(U,"P",{});var Dn=d(io);Ns=a(Dn,`The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
tasks was shown in `),po=s(Dn,"A",{href:!0,rel:!0});var Nl=d(po);Xs=a(Nl,`Leveraging Pre-trained Checkpoints for Sequence Generation
Tasks`),Nl.forEach(t),qs=a(Dn,` by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.`),Dn.forEach(t),As=p(U),ho=s(U,"P",{});var Yn=d(ho);Ps=a(Yn,"Additionally, in "),mo=s(Yn,"A",{href:!0,rel:!0});var Xl=d(mo);Qs=a(Xl,`Large-Scale Self- and Semi-Supervised Learning for Speech
Translation`),Xl.forEach(t),Hs=a(Yn,` it is shown how leveraging large pretrained speech models for speech
translation yields a significant performance improvement.`),Yn.forEach(t),Ls=p(U),Dt=s(U,"P",{});var ql=d(Dt);Os=a(ql,`After such an Speech-Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other
models (see the examples for more information).`),ql.forEach(t),Ks=p(U),fo=s(U,"P",{});var Nn=d(fo);ed=a(Nn,"This model inherits from "),Po=s(Nn,"A",{href:!0});var Al=d(Po);od=a(Al,"FlaxPreTrainedModel"),Al.forEach(t),td=a(Nn,`. Check the superclass documentation for the generic methods the
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
etc.)`),Nn.forEach(t),nd=p(U),uo=s(U,"P",{});var Xn=d(uo);ad=a(Xn,`This model is also a Flax Linen
`),go=s(Xn,"A",{href:!0,rel:!0});var Pl=d(go);rd=a(Pl,"flax.nn.Module"),Pl.forEach(t),sd=a(Xn,` subclass. Use it as a
regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.`),Xn.forEach(t),dd=p(U),q=s(U,"P",{});var wo=d(q);Qo=s(wo,"A",{href:!0});var Ql=d(Qo);ld=a(Ql,"FlaxSpeechEncoderDecoderModel"),Ql.forEach(t),cd=a(wo,` is a generic model class that will be instantiated as a transformer architecture
with the module (flax.nn.Module) of one of the base model classes of the library as encoder module and another one
as decoder module when created with the :meth`),Yt=s(wo,"EM",{});var Hl=d(Yt);id=a(Hl,"~transformers.FlaxAutoModel.from_pretrained"),Hl.forEach(t),pd=a(wo,` class method for the
encoder and :meth`),Nt=s(wo,"EM",{});var Ll=d(Nt);hd=a(Ll,"~transformers.FlaxAutoModelForCausalLM.from_pretrained"),Ll.forEach(t),md=a(wo," class method for the decoder."),wo.forEach(t),fd=p(U),z=s(U,"DIV",{class:!0});var Ue=d(z);M(_o.$$.fragment,Ue),ud=p(Ue),de=s(Ue,"P",{});var at=d(de);gd=a(at,"The "),Ho=s(at,"A",{href:!0});var Ol=d(Ho);_d=a(Ol,"FlaxSpeechEncoderDecoderModel"),Ol.forEach(t),yd=a(at," forward method, overrides the "),Xt=s(at,"CODE",{});var Kl=d(Xt);bd=a(Kl,"__call__"),Kl.forEach(t),vd=a(at," special method."),at.forEach(t),Md=p(Ue),M(Te.$$.fragment,Ue),wd=p(Ue),M(ke.$$.fragment,Ue),Ue.forEach(t),jd=p(U),A=s(U,"DIV",{class:!0});var rt=d(A);M(yo.$$.fragment,rt),Td=p(rt),qt=s(rt,"P",{});var ec=d(qt);kd=a(ec,`Instantiate an encoder and a decoder from one or two base classes of the library from pretrained model
checkpoints.`),ec.forEach(t),Ed=p(rt),M(Ee.$$.fragment,rt),rt.forEach(t),U.forEach(t),this.h()},h(){c(m,"name","hf:doc:metadata"),c(m,"content",JSON.stringify(uc)),c(f,"id","speech-encoder-decoder-models"),c(f,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(f,"href","#speech-encoder-decoder-models"),c(_,"class","relative group"),c(jo,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel"),c(To,"href","wav2vec2"),c(ko,"href","hubert"),c(xe,"href","https://arxiv.org/abs/2104.06678"),c(xe,"rel","nofollow"),c(Eo,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel"),c(Zo,"href","speech_to_text_2"),c(ce,"id","randomly-initializing-speechencoderdecodermodel-from-model-configurations"),c(ce,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ce,"href","#randomly-initializing-speechencoderdecodermodel-from-model-configurations"),c(Q,"class","relative group"),c(Jo,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel"),c(Wo,"href","/docs/transformers/v4.32.0/en/model_doc/wav2vec2#transformers.Wav2Vec2Model"),c(ie,"id","initialising-speechencoderdecodermodel-from-a-pretrained-encoder-and-a-pretrained-decoder"),c(ie,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ie,"href","#initialising-speechencoderdecodermodel-from-a-pretrained-encoder-and-a-pretrained-decoder"),c(H,"class","relative group"),c(So,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel"),c(Uo,"href","wav2vec2"),c(Vo,"href","hubert"),c($o,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel"),c(pe,"href","https://huggingface.co/blog/warm-starting-encoder-decoder"),c(pe,"rel","nofollow"),c(xo,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel.from_encoder_decoder_pretrained"),c(he,"id","loading-an-existing-speechencoderdecodermodel-checkpoint-and-perform-inference"),c(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(he,"href","#loading-an-existing-speechencoderdecodermodel-checkpoint-and-perform-inference"),c(L,"class","relative group"),c(Ro,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel"),c(fe,"id","training"),c(fe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(fe,"href","#training"),c(O,"class","relative group"),c(ue,"id","transformers.SpeechEncoderDecoderConfig"),c(ue,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(ue,"href","#transformers.SpeechEncoderDecoderConfig"),c(K,"class","relative group"),c(Go,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderConfig"),c(Co,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel"),c(Fo,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),c(Bo,"href","/docs/transformers/v4.32.0/en/main_classes/configuration#transformers.PretrainedConfig"),c(zo,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderConfig"),c(ye,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(be,"id","transformers.SpeechEncoderDecoderModel"),c(be,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(be,"href","#transformers.SpeechEncoderDecoderModel"),c(oe,"class","relative group"),c(Io,"href","/docs/transformers/v4.32.0/en/model_doc/auto#transformers.FlaxAutoModelForVision2Seq.from_pretrained"),c(Do,"href","/docs/transformers/v4.32.0/en/model_doc/auto#transformers.FlaxAutoModelForVision2Seq.from_pretrained"),c(Ke,"href","https://arxiv.org/abs/1907.12461"),c(Ke,"rel","nofollow"),c(oo,"href","https://arxiv.org/abs/2104.06678"),c(oo,"rel","nofollow"),c(Yo,"href","/docs/transformers/v4.32.0/en/main_classes/model#transformers.PreTrainedModel"),c(ao,"href","https://pytorch.org/docs/stable/nn.html#torch.nn.Module"),c(ao,"rel","nofollow"),c(No,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel"),c(Xo,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.SpeechEncoderDecoderModel"),c(F,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(je,"id","transformers.FlaxSpeechEncoderDecoderModel"),c(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(je,"href","#transformers.FlaxSpeechEncoderDecoderModel"),c(re,"class","relative group"),c(qo,"href","/docs/transformers/v4.32.0/en/model_doc/auto#transformers.FlaxAutoModelForVision2Seq.from_pretrained"),c(Ao,"href","/docs/transformers/v4.32.0/en/model_doc/auto#transformers.FlaxAutoModelForVision2Seq.from_pretrained"),c(po,"href","https://arxiv.org/abs/1907.12461"),c(po,"rel","nofollow"),c(mo,"href","https://arxiv.org/abs/2104.06678"),c(mo,"rel","nofollow"),c(Po,"href","/docs/transformers/v4.32.0/en/main_classes/model#transformers.FlaxPreTrainedModel"),c(go,"href","https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html"),c(go,"rel","nofollow"),c(Qo,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.FlaxSpeechEncoderDecoderModel"),c(Ho,"href","/docs/transformers/v4.32.0/en/model_doc/speech-encoder-decoder#transformers.FlaxSpeechEncoderDecoderModel"),c(z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(A,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),c(J,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(o,h){e(document.head,m),u(o,b,h),u(o,_,h),e(_,f),e(f,y),w(l,y,null),e(_,g),e(_,I),e(I,qn),u(o,on,h),u(o,x,h),e(x,An),e(x,jo),e(jo,Pn),e(x,Qn),e(x,st),e(st,Hn),e(x,Ln),e(x,To),e(To,On),e(x,Kn),e(x,ko),e(ko,ea),e(x,oa),u(o,tn,h),u(o,Y,h),e(Y,ta),e(Y,dt),e(dt,na),e(Y,aa),e(Y,xe),e(xe,ra),e(Y,sa),u(o,nn,h),u(o,N,h),e(N,da),e(N,Eo),e(Eo,la),e(N,ca),e(N,Zo),e(Zo,ia),e(N,pa),u(o,an,h),u(o,Q,h),e(Q,ce),e(ce,lt),w(Re,lt,null),e(Q,ha),e(Q,Ge),e(Ge,ma),e(Ge,ct),e(ct,fa),e(Ge,ua),u(o,rn,h),u(o,D,h),e(D,Jo),e(Jo,ga),e(D,_a),e(D,Wo),e(Wo,ya),e(D,ba),e(D,it),e(it,va),e(D,Ma),u(o,sn,h),w(Ce,o,h),u(o,dn,h),u(o,H,h),e(H,ie),e(ie,pt),w(Fe,pt,null),e(H,wa),e(H,Be),e(Be,ja),e(Be,ht),e(ht,Ta),e(Be,ka),u(o,ln,h),u(o,E,h),e(E,So),e(So,Ea),e(E,Za),e(E,mt),e(mt,Ja),e(E,Wa),e(E,Uo),e(Uo,Sa),e(E,Ua),e(E,Vo),e(Vo,Va),e(E,$a),e(E,ft),e(ft,xa),e(E,Ra),e(E,ut),e(ut,Ga),e(E,Ca),e(E,gt),e(gt,Fa),e(E,Ba),e(E,$o),e($o,za),e(E,Ia),e(E,pe),e(pe,Da),e(pe,_t),e(_t,Ya),e(E,Na),e(E,yt),e(yt,Xa),e(E,qa),e(E,xo),e(xo,Aa),e(E,Pa),u(o,cn,h),w(ze,o,h),u(o,pn,h),u(o,L,h),e(L,he),e(he,bt),w(Ie,bt,null),e(L,Qa),e(L,De),e(De,Ha),e(De,vt),e(vt,La),e(De,Oa),u(o,hn,h),u(o,G,h),e(G,Ka),e(G,Mt),e(Mt,er),e(G,or),e(G,Ro),e(Ro,tr),e(G,nr),e(G,wt),e(wt,ar),e(G,rr),u(o,mn,h),u(o,me,h),e(me,sr),e(me,jt),e(jt,dr),e(me,lr),u(o,fn,h),w(Ye,o,h),u(o,un,h),u(o,O,h),e(O,fe),e(fe,Tt),w(Ne,Tt,null),e(O,cr),e(O,kt),e(kt,ir),u(o,gn,h),u(o,C,h),e(C,pr),e(C,Et),e(Et,hr),e(C,mr),e(C,Zt),e(Zt,fr),e(C,ur),e(C,Jt),e(Jt,gr),e(C,_r),u(o,_n,h),w(Xe,o,h),u(o,yn,h),u(o,K,h),e(K,ue),e(ue,Wt),w(qe,Wt,null),e(K,yr),e(K,St),e(St,br),u(o,bn,h),u(o,$,h),w(Ae,$,null),e($,vr),e($,ge),e(ge,Go),e(Go,Mr),e(ge,wr),e(ge,Co),e(Co,jr),e(ge,Tr),e($,kr),e($,ee),e(ee,Er),e(ee,Fo),e(Fo,Zr),e(ee,Jr),e(ee,Bo),e(Bo,Wr),e(ee,Sr),e($,Ur),w(_e,$,null),e($,Vr),e($,ye),w(Pe,ye,null),e(ye,$r),e(ye,Qe),e(Qe,xr),e(Qe,zo),e(zo,Rr),e(Qe,Gr),u(o,vn,h),u(o,oe,h),e(oe,be),e(be,Ut),w(He,Ut,null),e(oe,Cr),e(oe,Vt),e(Vt,Fr),u(o,Mn,h),u(o,Z,h),w(Le,Z,null),e(Z,Br),e(Z,te),e(te,zr),e(te,Io),e(Io,Ir),e(te,Dr),e(te,Do),e(Do,Yr),e(te,Nr),e(Z,Xr),e(Z,Oe),e(Oe,qr),e(Oe,Ke),e(Ke,Ar),e(Oe,Pr),e(Z,Qr),e(Z,eo),e(eo,Hr),e(eo,oo),e(oo,Lr),e(eo,Or),e(Z,Kr),e(Z,$t),e($t,es),e(Z,os),e(Z,to),e(to,ts),e(to,Yo),e(Yo,ns),e(to,as),e(Z,rs),e(Z,no),e(no,ss),e(no,ao),e(ao,ds),e(no,ls),e(Z,cs),e(Z,X),e(X,No),e(No,is),e(X,ps),e(X,xt),e(xt,hs),e(X,ms),e(X,Rt),e(Rt,fs),e(X,us),e(Z,gs),e(Z,F),w(ro,F,null),e(F,_s),e(F,ne),e(ne,ys),e(ne,Xo),e(Xo,bs),e(ne,vs),e(ne,Gt),e(Gt,Ms),e(ne,ws),e(F,js),w(ve,F,null),e(F,Ts),w(Me,F,null),e(Z,ks),e(Z,B),w(so,B,null),e(B,Es),e(B,Ct),e(Ct,Zs),e(B,Js),e(B,ae),e(ae,Ws),e(ae,Ft),e(Ft,Ss),e(ae,Us),e(ae,Bt),e(Bt,Vs),e(ae,$s),e(B,xs),w(we,B,null),u(o,wn,h),u(o,re,h),e(re,je),e(je,zt),w(lo,zt,null),e(re,Rs),e(re,It),e(It,Gs),u(o,jn,h),u(o,J,h),w(co,J,null),e(J,Cs),e(J,se),e(se,Fs),e(se,qo),e(qo,Bs),e(se,zs),e(se,Ao),e(Ao,Is),e(se,Ds),e(J,Ys),e(J,io),e(io,Ns),e(io,po),e(po,Xs),e(io,qs),e(J,As),e(J,ho),e(ho,Ps),e(ho,mo),e(mo,Qs),e(ho,Hs),e(J,Ls),e(J,Dt),e(Dt,Os),e(J,Ks),e(J,fo),e(fo,ed),e(fo,Po),e(Po,od),e(fo,td),e(J,nd),e(J,uo),e(uo,ad),e(uo,go),e(go,rd),e(uo,sd),e(J,dd),e(J,q),e(q,Qo),e(Qo,ld),e(q,cd),e(q,Yt),e(Yt,id),e(q,pd),e(q,Nt),e(Nt,hd),e(q,md),e(J,fd),e(J,z),w(_o,z,null),e(z,ud),e(z,de),e(de,gd),e(de,Ho),e(Ho,_d),e(de,yd),e(de,Xt),e(Xt,bd),e(de,vd),e(z,Md),w(Te,z,null),e(z,wd),w(ke,z,null),e(J,jd),e(J,A),w(yo,A,null),e(A,Td),e(A,qt),e(qt,kd),e(A,Ed),w(Ee,A,null),Tn=!0},p(o,[h]){const bo={};h&2&&(bo.$$scope={dirty:h,ctx:o}),_e.$set(bo);const At={};h&2&&(At.$$scope={dirty:h,ctx:o}),ve.$set(At);const Pt={};h&2&&(Pt.$$scope={dirty:h,ctx:o}),Me.$set(Pt);const Qt={};h&2&&(Qt.$$scope={dirty:h,ctx:o}),we.$set(Qt);const R={};h&2&&(R.$$scope={dirty:h,ctx:o}),Te.$set(R);const Ht={};h&2&&(Ht.$$scope={dirty:h,ctx:o}),ke.$set(Ht);const Lt={};h&2&&(Lt.$$scope={dirty:h,ctx:o}),Ee.$set(Lt)},i(o){Tn||(j(l.$$.fragment,o),j(Re.$$.fragment,o),j(Ce.$$.fragment,o),j(Fe.$$.fragment,o),j(ze.$$.fragment,o),j(Ie.$$.fragment,o),j(Ye.$$.fragment,o),j(Ne.$$.fragment,o),j(Xe.$$.fragment,o),j(qe.$$.fragment,o),j(Ae.$$.fragment,o),j(_e.$$.fragment,o),j(Pe.$$.fragment,o),j(He.$$.fragment,o),j(Le.$$.fragment,o),j(ro.$$.fragment,o),j(ve.$$.fragment,o),j(Me.$$.fragment,o),j(so.$$.fragment,o),j(we.$$.fragment,o),j(lo.$$.fragment,o),j(co.$$.fragment,o),j(_o.$$.fragment,o),j(Te.$$.fragment,o),j(ke.$$.fragment,o),j(yo.$$.fragment,o),j(Ee.$$.fragment,o),Tn=!0)},o(o){T(l.$$.fragment,o),T(Re.$$.fragment,o),T(Ce.$$.fragment,o),T(Fe.$$.fragment,o),T(ze.$$.fragment,o),T(Ie.$$.fragment,o),T(Ye.$$.fragment,o),T(Ne.$$.fragment,o),T(Xe.$$.fragment,o),T(qe.$$.fragment,o),T(Ae.$$.fragment,o),T(_e.$$.fragment,o),T(Pe.$$.fragment,o),T(He.$$.fragment,o),T(Le.$$.fragment,o),T(ro.$$.fragment,o),T(ve.$$.fragment,o),T(Me.$$.fragment,o),T(so.$$.fragment,o),T(we.$$.fragment,o),T(lo.$$.fragment,o),T(co.$$.fragment,o),T(_o.$$.fragment,o),T(Te.$$.fragment,o),T(ke.$$.fragment,o),T(yo.$$.fragment,o),T(Ee.$$.fragment,o),Tn=!1},d(o){t(m),o&&t(b),o&&t(_),k(l),o&&t(on),o&&t(x),o&&t(tn),o&&t(Y),o&&t(nn),o&&t(N),o&&t(an),o&&t(Q),k(Re),o&&t(rn),o&&t(D),o&&t(sn),k(Ce,o),o&&t(dn),o&&t(H),k(Fe),o&&t(ln),o&&t(E),o&&t(cn),k(ze,o),o&&t(pn),o&&t(L),k(Ie),o&&t(hn),o&&t(G),o&&t(mn),o&&t(me),o&&t(fn),k(Ye,o),o&&t(un),o&&t(O),k(Ne),o&&t(gn),o&&t(C),o&&t(_n),k(Xe,o),o&&t(yn),o&&t(K),k(qe),o&&t(bn),o&&t($),k(Ae),k(_e),k(Pe),o&&t(vn),o&&t(oe),k(He),o&&t(Mn),o&&t(Z),k(Le),k(ro),k(ve),k(Me),k(so),k(we),o&&t(wn),o&&t(re),k(lo),o&&t(jn),o&&t(J),k(co),k(_o),k(Te),k(ke),k(yo),k(Ee)}}}const uc={local:"speech-encoder-decoder-models",sections:[{local:"randomly-initializing-speechencoderdecodermodel-from-model-configurations",title:"Randomly initializing `SpeechEncoderDecoderModel` from model configurations."},{local:"initialising-speechencoderdecodermodel-from-a-pretrained-encoder-and-a-pretrained-decoder",title:"Initialising `SpeechEncoderDecoderModel` from a pretrained encoder and a pretrained decoder."},{local:"loading-an-existing-speechencoderdecodermodel-checkpoint-and-perform-inference",title:"Loading an existing `SpeechEncoderDecoderModel` checkpoint and perform inference."},{local:"training",title:"Training"},{local:"transformers.SpeechEncoderDecoderConfig",title:"SpeechEncoderDecoderConfig"},{local:"transformers.SpeechEncoderDecoderModel",title:"SpeechEncoderDecoderModel"},{local:"transformers.FlaxSpeechEncoderDecoderModel",title:"FlaxSpeechEncoderDecoderModel"}],title:"Speech Encoder Decoder Models"};function gc(V){return sc(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class jc extends tc{constructor(m){super();nc(this,m,gc,fc,ac,{})}}export{jc as default,uc as metadata};
