import{S as Zo,i as Wo,s as Io,e as i,k as c,w as d,t as o,M as No,c as r,d as t,m as h,a as p,x as m,h as l,b as u,N as Jn,G as a,g as n,y,q as g,o as w,B as T,v as xo}from"../../chunks/vendor-hf-doc-builder.js";import{T as Eo}from"../../chunks/Tip-hf-doc-builder.js";import{I as Nt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as J}from"../../chunks/CodeBlock-hf-doc-builder.js";function Bo(Xe){let f,S,v,G,C,j,oe,R,A,Z,k,le;return{c(){f=i("p"),S=o(`Whisper can also be \u2018tricked\u2019 into translating from speech in any language X to any language Y. Simply set the task to
`),v=i("code"),G=o('"transcribe"'),C=o(" and the "),j=i("code"),oe=o('"language"'),R=o(` to your target language in the generation key-word arguments,
e.g. for Spanish, one would set:`),A=c(),Z=i("p"),k=i("code"),le=o('generate_kwargs={"task": "transcribe", "language": "es"}')},l(M){f=r(M,"P",{});var U=p(f);S=l(U,`Whisper can also be \u2018tricked\u2019 into translating from speech in any language X to any language Y. Simply set the task to
`),v=r(U,"CODE",{});var Ce=p(v);G=l(Ce,'"transcribe"'),Ce.forEach(t),C=l(U," and the "),j=r(U,"CODE",{});var ie=p(j);oe=l(ie,'"language"'),ie.forEach(t),R=l(U,` to your target language in the generation key-word arguments,
e.g. for Spanish, one would set:`),U.forEach(t),A=h(M),Z=r(M,"P",{});var W=p(Z);k=r(W,"CODE",{});var Y=p(k);le=l(Y,'generate_kwargs={"task": "transcribe", "language": "es"}'),Y.forEach(t),W.forEach(t)},m(M,U){n(M,f,U),a(f,S),a(f,v),a(v,G),a(f,C),a(f,j),a(j,oe),a(f,R),n(M,A,U),n(M,Z,U),a(Z,k),a(k,le)},d(M){M&&t(f),M&&t(A),M&&t(Z)}}}function Vo(Xe){let f;return{c(){f=o(`Here we're using SpeechT5 checkpoint trained specifically for English TTS. Should you wish to translate into a language 
    other than English, either swap the checkpoint for a SpeechT5 TTS model fine-tuned on your language of choice, or 
    use an MMS TTS checkpoint pre-trained in your target langauge.`)},l(S){f=l(S,`Here we're using SpeechT5 checkpoint trained specifically for English TTS. Should you wish to translate into a language 
    other than English, either swap the checkpoint for a SpeechT5 TTS model fine-tuned on your language of choice, or 
    use an MMS TTS checkpoint pre-trained in your target langauge.`)},m(S,v){n(S,f,v)},d(S){S&&t(f)}}}function Xo(Xe){let f,S,v,G,C,j,oe,R,A,Z,k,le,M,U,Ce,ie,W,Y,Sn,xt,I,qa,ht,Ha,Fa,ut,Aa,za,Bt,Re,Pa,Vt,_,Da,ft,Ka,La,dt,Oa,es,mt,ts,as,Xt,re,Ye,kn,Ct,z,ss,yt,ns,os,Rt,P,ls,pe,is,rs,Yt,Qe,ps,Qt,Q,D,gt,ce,cs,wt,hs,qt,N,us,he,fs,ds,ue,ms,ys,Ht,fe,Ft,x,gs,Tt,ws,Ts,de,vs,Ms,At,me,zt,ye,bs,ge,Js,Pt,qe,Ss,Dt,we,Kt,B,ks,vt,Us,js,Mt,$s,Gs,Lt,Te,Ot,K,ea,He,_s,ta,ve,aa,Me,sa,Fe,Es,na,be,oa,Je,la,Ae,Zs,ia,L,Ws,ze,Is,Ns,ra,q,O,bt,Se,xs,Jt,Bs,pa,V,Vs,ke,Xs,Cs,St,Rs,Ys,ca,Pe,Qs,ha,Ue,ua,ee,fa,De,qs,da,je,ma,Ke,Hs,ya,$e,ga,Le,Fs,wa,Ge,Ta,Oe,As,va,_e,Ma,et,zs,ba,H,te,kt,Ee,Ps,Ut,Ds,Ja,b,Ks,Ze,Ls,Os,jt,en,tn,$t,an,sn,Gt,nn,on,_t,ln,rn,Sa,We,ka,tt,pn,Ua,Ie,ja,at,cn,$a,Ne,Ga,st,hn,_a,$,Un,Ea,ae,un,xe,fn,dn,Za,F,se,Et,Be,mn,Zt,yn,Wa,X,gn,Wt,wn,Tn,nt,vn,Mn,Ia;return j=new Nt({}),ce=new Nt({}),fe=new J({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwcGlwZWxpbmUlMEElMEFkZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTNBMCUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIlMEFwaXBlJTIwJTNEJTIwcGlwZWxpbmUoJTBBJTIwJTIwJTIwJTIwJTIyYXV0b21hdGljLXNwZWVjaC1yZWNvZ25pdGlvbiUyMiUyQyUyMG1vZGVsJTNEJTIyb3BlbmFpJTJGd2hpc3Blci1iYXNlJTIyJTJDJTIwZGV2aWNlJTNEZGV2aWNlJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

device = <span class="hljs-string">&quot;cuda:0&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>
pipe = pipeline(
    <span class="hljs-string">&quot;automatic-speech-recognition&quot;</span>, model=<span class="hljs-string">&quot;openai/whisper-base&quot;</span>, device=device
)`}}),me=new J({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJmYWNlYm9vayUyRnZveHBvcHVsaSUyMiUyQyUyMCUyMml0JTIyJTJDJTIwc3BsaXQlM0QlMjJ2YWxpZGF0aW9uJTIyJTJDJTIwc3RyZWFtaW5nJTNEVHJ1ZSklMEFzYW1wbGUlMjAlM0QlMjBuZXh0KGl0ZXIoZGF0YXNldCkp",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

dataset = load_dataset(<span class="hljs-string">&quot;facebook/voxpopuli&quot;</span>, <span class="hljs-string">&quot;it&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>, streaming=<span class="hljs-literal">True</span>)
sample = <span class="hljs-built_in">next</span>(<span class="hljs-built_in">iter</span>(dataset))`}}),we=new J({props:{code:"ZnJvbSUyMElQeXRob24uZGlzcGxheSUyMGltcG9ydCUyMEF1ZGlvJTBBJTBBQXVkaW8oc2FtcGxlJTVCJTIyYXVkaW8lMjIlNUQlNUIlMjJhcnJheSUyMiU1RCUyQyUyMHJhdGUlM0RzYW1wbGUlNUIlMjJhdWRpbyUyMiU1RCU1QiUyMnNhbXBsaW5nX3JhdGUlMjIlNUQp",highlighted:`<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> Audio

Audio(sample[<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>], rate=sample[<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;sampling_rate&quot;</span>])`}}),Te=new J({props:{code:"ZGVmJTIwdHJhbnNsYXRlKGF1ZGlvKSUzQSUwQSUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBwaXBlKGF1ZGlvJTJDJTIwbWF4X25ld190b2tlbnMlM0QyNTYlMkMlMjBnZW5lcmF0ZV9rd2FyZ3MlM0QlN0IlMjJ0YXNrJTIyJTNBJTIwJTIydHJhbnNsYXRlJTIyJTdEKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMG91dHB1dHMlNUIlMjJ0ZXh0JTIyJTVE",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">translate</span>(<span class="hljs-params">audio</span>):
    outputs = pipe(audio, max_new_tokens=<span class="hljs-number">256</span>, generate_kwargs={<span class="hljs-string">&quot;task&quot;</span>: <span class="hljs-string">&quot;translate&quot;</span>})
    <span class="hljs-keyword">return</span> outputs[<span class="hljs-string">&quot;text&quot;</span>]`}}),K=new Eo({props:{$$slots:{default:[Bo]},$$scope:{ctx:Xe}}}),ve=new J({props:{code:"dHJhbnNsYXRlKHNhbXBsZSU1QiUyMmF1ZGlvJTIyJTVELmNvcHkoKSk=",highlighted:'translate(sample[<span class="hljs-string">&quot;audio&quot;</span>].copy())'}}),Me=new J({props:{code:"JyUyMHBzeWNob2xvZ2ljYWwlMjBhbmQlMjBzb2NpYWwuJTIwSSUyMHRoaW5rJTIwdGhhdCUyMGl0JTIwaXMlMjBhJTIwdmVyeSUyMGltcG9ydGFudCUyMHN0ZXAlMjBpbiUyMHRoZSUyMGNvbnN0cnVjdGlvbiUyMG9mJTIwYSUyMGp1cmlkaWNhbCUyMHNwYWNlJTIwb2YlMjBmcmVlZG9tJTJDJTIwY2lyY3VsYXRpb24lMjBhbmQlMjBwcm90ZWN0aW9uJTIwb2YlMjByaWdodHMuJw==",highlighted:'<span class="hljs-comment">&#x27; psychological and social. I think that it is a very important step in the construction of a juridical space of freedom, circulation and protection of rights.&#x27;</span>'}}),be=new J({props:{code:"c2FtcGxlJTVCJTIycmF3X3RleHQlMjIlNUQ=",highlighted:'sample[<span class="hljs-string">&quot;raw_text&quot;</span>]'}}),Je=new J({props:{code:"J1BlbnNvJTIwY2hlJTIwcXVlc3RvJTIwc2lhJTIwdW4lMjBwYXNzbyUyMGluJTIwYXZhbnRpJTIwaW1wb3J0YW50ZSUyMG5lbGxhJTIwY29zdHJ1emlvbmUlMjBkaSUyMHVubyUyMHNwYXppbyUyMGdpdXJpZGljbyUyMGRpJTIwbGliZXJ0JUMzJUEwJTIwZGklMjBjaXJjb2xhemlvbmUlMjBlJTIwZGklMjBwcm90ZXppb25lJTIwZGVpJTIwZGlyaXR0aSUyMHBlciUyMGxlJTIwcGVyc29uZSUyMGluJTIwRXVyb3BhLic=",highlighted:'<span class="hljs-comment">&#x27;Penso che questo sia un passo in avanti importante nella costruzione di uno spazio giuridico di libert\xE0 di circolazione e di protezione dei diritti per le persone in Europa.&#x27;</span>'}}),Se=new Nt({}),Ue=new J({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNwZWVjaFQ1UHJvY2Vzc29yJTJDJTIwU3BlZWNoVDVGb3JUZXh0VG9TcGVlY2glMkMlMjBTcGVlY2hUNUhpZmlHYW4lMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBTcGVlY2hUNVByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIybWljcm9zb2Z0JTJGc3BlZWNodDVfdHRzJTIyKSUwQSUwQW1vZGVsJTIwJTNEJTIwU3BlZWNoVDVGb3JUZXh0VG9TcGVlY2guZnJvbV9wcmV0cmFpbmVkKCUyMm1pY3Jvc29mdCUyRnNwZWVjaHQ1X3R0cyUyMiklMEF2b2NvZGVyJTIwJTNEJTIwU3BlZWNoVDVIaWZpR2FuLmZyb21fcHJldHJhaW5lZCglMjJtaWNyb3NvZnQlMkZzcGVlY2h0NV9oaWZpZ2FuJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan

processor = SpeechT5Processor.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_tts&quot;</span>)

model = SpeechT5ForTextToSpeech.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_tts&quot;</span>)
vocoder = SpeechT5HifiGan.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_hifigan&quot;</span>)`}}),ee=new Eo({props:{$$slots:{default:[Vo]},$$scope:{ctx:Xe}}}),je=new J({props:{code:"bW9kZWwudG8oZGV2aWNlKSUwQXZvY29kZXIudG8oZGV2aWNlKQ==",highlighted:`model.to(device)
vocoder.to(device)`}}),$e=new J({props:{code:"ZW1iZWRkaW5nc19kYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMk1hdHRoaWpzJTJGY211LWFyY3RpYy14dmVjdG9ycyUyMiUyQyUyMHNwbGl0JTNEJTIydmFsaWRhdGlvbiUyMiklMEFzcGVha2VyX2VtYmVkZGluZ3MlMjAlM0QlMjB0b3JjaC50ZW5zb3IoZW1iZWRkaW5nc19kYXRhc2V0JTVCNzMwNiU1RCU1QiUyMnh2ZWN0b3IlMjIlNUQpLnVuc3F1ZWV6ZSgwKQ==",highlighted:`embeddings_dataset = load_dataset(<span class="hljs-string">&quot;Matthijs/cmu-arctic-xvectors&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>)
speaker_embeddings = torch.tensor(embeddings_dataset[<span class="hljs-number">7306</span>][<span class="hljs-string">&quot;xvector&quot;</span>]).unsqueeze(<span class="hljs-number">0</span>)`}}),Ge=new J({props:{code:"ZGVmJTIwc3ludGhlc2lzZSh0ZXh0KSUzQSUwQSUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMHByb2Nlc3Nvcih0ZXh0JTNEdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTIwJTIwJTIwJTIwc3BlZWNoJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGVfc3BlZWNoKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyU1QiUyMmlucHV0X2lkcyUyMiU1RC50byhkZXZpY2UpJTJDJTIwc3BlYWtlcl9lbWJlZGRpbmdzLnRvKGRldmljZSklMkMlMjB2b2NvZGVyJTNEdm9jb2RlciUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBzcGVlY2guY3B1KCk=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">synthesise</span>(<span class="hljs-params">text</span>):
    inputs = processor(text=text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
    speech = model.generate_speech(
        inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].to(device), speaker_embeddings.to(device), vocoder=vocoder
    )
    <span class="hljs-keyword">return</span> speech.cpu()`}}),_e=new J({props:{code:"c3BlZWNoJTIwJTNEJTIwc3ludGhlc2lzZSglMjJIZXklMjB0aGVyZSElMjBUaGlzJTIwaXMlMjBhJTIwdGVzdCElMjIpJTBBJTBBQXVkaW8oc3BlZWNoJTJDJTIwcmF0ZSUzRDE2MDAwKQ==",highlighted:`speech = synthesise(<span class="hljs-string">&quot;Hey there! This is a test!&quot;</span>)

Audio(speech, rate=<span class="hljs-number">16000</span>)`}}),Ee=new Nt({}),We=new J({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBdGFyZ2V0X2R0eXBlJTIwJTNEJTIwbnAuaW50MTYlMEFtYXhfcmFuZ2UlMjAlM0QlMjBucC5paW5mbyh0YXJnZXRfZHR5cGUpLm1heCUwQSUwQSUwQWRlZiUyMHNwZWVjaF90b19zcGVlY2hfdHJhbnNsYXRpb24oYXVkaW8pJTNBJTBBJTIwJTIwJTIwJTIwdHJhbnNsYXRlZF90ZXh0JTIwJTNEJTIwdHJhbnNsYXRlKGF1ZGlvKSUwQSUyMCUyMCUyMCUyMHN5bnRoZXNpc2VkX3NwZWVjaCUyMCUzRCUyMHN5bnRoZXNpc2UodHJhbnNsYXRlZF90ZXh0KSUwQSUyMCUyMCUyMCUyMHN5bnRoZXNpc2VkX3NwZWVjaCUyMCUzRCUyMChzeW50aGVzaXNlZF9zcGVlY2gubnVtcHkoKSUyMColMjBtYXhfcmFuZ2UpLmFzdHlwZShucC5pbnQxNiklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAxNjAwMCUyQyUyMHN5bnRoZXNpc2VkX3NwZWVjaA==",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

target_dtype = np.int16
max_range = np.iinfo(target_dtype).<span class="hljs-built_in">max</span>


<span class="hljs-keyword">def</span> <span class="hljs-title function_">speech_to_speech_translation</span>(<span class="hljs-params">audio</span>):
    translated_text = translate(audio)
    synthesised_speech = synthesise(translated_text)
    synthesised_speech = (synthesised_speech.numpy() * max_range).astype(np.int16)
    <span class="hljs-keyword">return</span> <span class="hljs-number">16000</span>, synthesised_speech`}}),Ie=new J({props:{code:"c2FtcGxpbmdfcmF0ZSUyQyUyMHN5bnRoZXNpc2VkX3NwZWVjaCUyMCUzRCUyMHNwZWVjaF90b19zcGVlY2hfdHJhbnNsYXRpb24oc2FtcGxlJTVCJTIyYXVkaW8lMjIlNUQpJTBBJTBBQXVkaW8oc3ludGhlc2lzZWRfc3BlZWNoJTJDJTIwcmF0ZSUzRHNhbXBsaW5nX3JhdGUp",highlighted:`sampling_rate, synthesised_speech = speech_to_speech_translation(sample[<span class="hljs-string">&quot;audio&quot;</span>])

Audio(synthesised_speech, rate=sampling_rate)`}}),Ne=new J({props:{code:"aW1wb3J0JTIwZ3JhZGlvJTIwYXMlMjBnciUwQSUwQWRlbW8lMjAlM0QlMjBnci5CbG9ja3MoKSUwQSUwQW1pY190cmFuc2xhdGUlMjAlM0QlMjBnci5JbnRlcmZhY2UoJTBBJTIwJTIwJTIwJTIwZm4lM0RzcGVlY2hfdG9fc3BlZWNoX3RyYW5zbGF0aW9uJTJDJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTNEZ3IuQXVkaW8oc291cmNlJTNEJTIybWljcm9waG9uZSUyMiUyQyUyMHR5cGUlM0QlMjJmaWxlcGF0aCUyMiklMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRzJTNEZ3IuQXVkaW8obGFiZWwlM0QlMjJHZW5lcmF0ZWQlMjBTcGVlY2glMjIlMkMlMjB0eXBlJTNEJTIybnVtcHklMjIpJTJDJTBBKSUwQSUwQWZpbGVfdHJhbnNsYXRlJTIwJTNEJTIwZ3IuSW50ZXJmYWNlKCUwQSUyMCUyMCUyMCUyMGZuJTNEc3BlZWNoX3RvX3NwZWVjaF90cmFuc2xhdGlvbiUyQyUwQSUyMCUyMCUyMCUyMGlucHV0cyUzRGdyLkF1ZGlvKHNvdXJjZSUzRCUyMnVwbG9hZCUyMiUyQyUyMHR5cGUlM0QlMjJmaWxlcGF0aCUyMiklMkMlMEElMjAlMjAlMjAlMjBvdXRwdXRzJTNEZ3IuQXVkaW8obGFiZWwlM0QlMjJHZW5lcmF0ZWQlMjBTcGVlY2glMjIlMkMlMjB0eXBlJTNEJTIybnVtcHklMjIpJTJDJTBBKSUwQSUwQXdpdGglMjBkZW1vJTNBJTBBJTIwJTIwJTIwJTIwZ3IuVGFiYmVkSW50ZXJmYWNlKCU1Qm1pY190cmFuc2xhdGUlMkMlMjBmaWxlX3RyYW5zbGF0ZSU1RCUyQyUyMCU1QiUyMk1pY3JvcGhvbmUlMjIlMkMlMjAlMjJBdWRpbyUyMEZpbGUlMjIlNUQpJTBBJTBBZGVtby5sYXVuY2goZGVidWclM0RUcnVlKQ==",highlighted:`<span class="hljs-keyword">import</span> gradio <span class="hljs-keyword">as</span> gr

demo = gr.Blocks()

mic_translate = gr.Interface(
    fn=speech_to_speech_translation,
    inputs=gr.Audio(source=<span class="hljs-string">&quot;microphone&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;filepath&quot;</span>),
    outputs=gr.Audio(label=<span class="hljs-string">&quot;Generated Speech&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;numpy&quot;</span>),
)

file_translate = gr.Interface(
    fn=speech_to_speech_translation,
    inputs=gr.Audio(source=<span class="hljs-string">&quot;upload&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;filepath&quot;</span>),
    outputs=gr.Audio(label=<span class="hljs-string">&quot;Generated Speech&quot;</span>, <span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;numpy&quot;</span>),
)

<span class="hljs-keyword">with</span> demo:
    gr.TabbedInterface([mic_translate, file_translate], [<span class="hljs-string">&quot;Microphone&quot;</span>, <span class="hljs-string">&quot;Audio File&quot;</span>])

demo.launch(debug=<span class="hljs-literal">True</span>)`}}),Be=new Nt({}),{c(){f=i("meta"),S=c(),v=i("h1"),G=i("a"),C=i("span"),d(j.$$.fragment),oe=c(),R=i("span"),A=o("Speech-to-speech translation"),Z=c(),k=i("p"),le=o(`Speech-to-speech translation (STST or S2ST) is a relatively new spoken language processing task. It involves translating
speech from one langauge into speech in a `),M=i("strong"),U=o("different"),Ce=o(" language:"),ie=c(),W=i("div"),Y=i("img"),xt=c(),I=i("p"),qa=o("STST can be viewed as an extension of the traditional machine translation (MT) task: instead of translating "),ht=i("strong"),Ha=o("text"),Fa=o(` from one
language into another, we translate `),ut=i("strong"),Aa=o("speech"),za=o(` from one language into another. STST holds applications in the field of
multilingual communication, enabling speakers in different languages to communicate with one another through the medium
of speech.`),Bt=c(),Re=i("p"),Pa=o(`Suppose you want to communicate with another individual across a langauge barrier. Rather
than writing the information that you want to convey and then translating it to text in the target language, you
can speak it directly and have a STST system convert your spoken speech into the target langauge. The recipient can then
respond by speaking back at the STST system, and you can listen to their response. This is a more natural way of communicating
compared to text-based machine translation.`),Vt=c(),_=i("p"),Da=o("In this chapter, we\u2019ll explore a "),ft=i("em"),Ka=o("cascaded"),La=o(` approach to STST, piecing together the knowledge you\u2019ve acquired in Units
5 and 6 of the course. We\u2019ll use a `),dt=i("em"),Oa=o("speech translation (ST)"),es=o(` system to transcribe the source speech into text in the target
language, then `),mt=i("em"),ts=o("text-to-speech (TTS)"),as=o(" to generate speech in the target language from the translated text:"),Xt=c(),re=i("div"),Ye=i("img"),Ct=c(),z=i("p"),ss=o(`We could also have used a three stage approach, where first we use an automatic speech recognition (ASR) system to
transcribe the source speech into text in the same language, then machine translation to translate the transcribed text
into the target language, and finally text-to-speech to generate speech in the target language. However, adding more
components to the pipeline lends itself to `),yt=i("em"),ns=o("error propagation"),os=o(`, where the errors introduced in one system are compounded
as they flow through the remaining systems, and also increases latency, since inference has to be conducted for more models.`),Rt=c(),P=i("p"),ls=o(`While this cascaded approach to STST is pretty straightforward, it results in very effective STST systems. The three-stage
cascaded system of ASR + MT + TTS was previously used to power many commercial STST products, including `),pe=i("a"),is=o("Google Translate"),rs=o(`.
It\u2019s also a very data and compute efficient way of developing a STST system, since existing speech recognition and
text-to-speech systems can be coupled together to yield a new STST model without any additional training.`),Yt=c(),Qe=i("p"),ps=o(`In the remainder of this Unit, we\u2019ll focus on creating a STST system that translates speech from any language X to speech
in English. The methods covered can be extended to STST systems that translate from any language X to any
langauge Y, but we leave this as an extension to the reader and provide pointers where applicable. We further divide up the
task of STST into its two constituent components: ST and TTS. We\u2019ll finish by piecing them together to build a Gradio
demo to showcase our system.`),Qt=c(),Q=i("h2"),D=i("a"),gt=i("span"),d(ce.$$.fragment),cs=c(),wt=i("span"),hs=o("Speech translation"),qt=c(),N=i("p"),us=o(`We\u2019ll use the Whisper model for our speech translation system, since it\u2019s capable of translating from over 96 languages
to English. Specifically, we\u2019ll load the `),he=i("a"),fs=o("Whisper Base"),ds=o(` checkpoint, which
clocks in at 74M parameters. It\u2019s by no means the most performant Whisper model, with the `),ue=i("a"),ms=o("largest Whisper checkpoint"),ys=o(`
being over 20x larger, but since we\u2019re concatenating two auto-regressive systems together (ST + TTS), we want to ensure
each model can generate relatively quickly so that we get reasonable inference speed:`),Ht=c(),d(fe.$$.fragment),Ft=c(),x=i("p"),gs=o(`Great! To test our STST system, we\u2019ll load an audio sample in a non-English language. Let\u2019s load the first example of the
Italian (`),Tt=i("code"),ws=o("it"),Ts=o(") split of the "),de=i("a"),vs=o("VoxPopuli"),Ms=o(" dataset:"),At=c(),d(me.$$.fragment),zt=c(),ye=i("p"),bs=o("To listen to this sample, we can either play it using the dataset viewer on the Hub: "),ge=i("a"),Js=o("facebook/voxpopuli/viewer"),Pt=c(),qe=i("p"),Ss=o("Or playback using the ipynb audio feature:"),Dt=c(),d(we.$$.fragment),Kt=c(),B=i("p"),ks=o(`Now let\u2019s define a function that takes this audio input and returns the translated text. You\u2019ll remember that we have to
pass the generation key-word argument for the `),vt=i("code"),Us=o('"task"'),js=o(", setting it to "),Mt=i("code"),$s=o('"translate"'),Gs=o(` to ensure that Whisper performs
speech translation and not speech recognition:`),Lt=c(),d(Te.$$.fragment),Ot=c(),d(K.$$.fragment),ea=c(),He=i("p"),_s=o("Great! Let\u2019s quickly check that we get a sensible result from the model:"),ta=c(),d(ve.$$.fragment),aa=c(),d(Me.$$.fragment),sa=c(),Fe=i("p"),Es=o("Alright! If we compare this to the source text:"),na=c(),d(be.$$.fragment),oa=c(),d(Je.$$.fragment),la=c(),Ae=i("p"),Zs=o(`We see that the translation more or less lines up (you can double check this using Google Translate), barring a small
extra few words at the start of the transcription where the speaker was finishing off their previous sentence.`),ia=c(),L=i("p"),Ws=o(`With that, we\u2019ve completed the first half of our cascaded STST pipeline, putting into practice the skills we gained in Unit 5
when we learnt how to use the Whisper model for speech recognition and translation. If you want a refresher on any of the
steps we covered, have a read through the section on `),ze=i("a"),Is=o("Pre-trained models for ASR"),Ns=o(" from Unit 5."),ra=c(),q=i("h2"),O=i("a"),bt=i("span"),d(Se.$$.fragment),xs=c(),Jt=i("span"),Bs=o("Text-to-speech"),pa=c(),V=i("p"),Vs=o(`The second half of our cascaded STST system involves mapping from English text to English speech. For this, we\u2019ll use
the pre-trained `),ke=i("a"),Xs=o("SpeechT5 TTS"),Cs=o(` model for English TTS. \u{1F917} Transformers currently doesn\u2019t
have a TTS `),St=i("code"),Rs=o("pipeline"),Ys=o(`, so we\u2019ll have to use the model directly ourselves. This is no biggie, you\u2019re all experts on using
the model for inference following Unit 6!`),ca=c(),Pe=i("p"),Qs=o("First, let\u2019s load the SpeechT5 processor, model and vocoder from the pre-trained checkpoint:"),ha=c(),d(Ue.$$.fragment),ua=c(),d(ee.$$.fragment),fa=c(),De=i("p"),qs=o("As with the Whisper model, we\u2019ll place the SpeechT5 model and vocoder on our GPU accelerator device if we have one:"),da=c(),d(je.$$.fragment),ma=c(),Ke=i("p"),Hs=o("Great! Let\u2019s load up the speaker embeddings:"),ya=c(),d($e.$$.fragment),ga=c(),Le=i("p"),Fs=o(`We can now write a function that takes a text prompt as input, and generates the corresponding speech. We\u2019ll first pre-process
the text input using the SpeechT5 processor, tokenizing the text to get our input ids. We\u2019ll then pass the input ids and
speaker embeddings to the SpeechT5 model, placing each on the accelerator device if available. Finally, we\u2019ll return the
generated speech, bringing it back to the CPU so that we can play it back in our ipynb notebook:`),wa=c(),d(Ge.$$.fragment),Ta=c(),Oe=i("p"),As=o("Let\u2019s check it works with a dummy text input:"),va=c(),d(_e.$$.fragment),Ma=c(),et=i("p"),zs=o("Sounds good! Now for the exciting part - piecing it all together."),ba=c(),H=i("h2"),te=i("a"),kt=i("span"),d(Ee.$$.fragment),Ps=c(),Ut=i("span"),Ds=o("Creating a STST demo"),Ja=c(),b=i("p"),Ks=o("Before we create a "),Ze=i("a"),Ls=o("Gradio"),Os=o(` demo to showcase our STST system, let\u2019s first do a quick sanity check
to make sure we can concatenate the two models, putting an audio sample in and getting an audio sample out. We\u2019ll do
this by concatenating the two functions we defined in the previous two sub-sections, such that we input the source audio
and retrieve the translated text, then synthesise the translated text to get the translated speech. Finally, we\u2019ll convert
the synthesised speech to an `),jt=i("code"),en=o("int16"),tn=o(` array, which is the output audio file format expected by Gradio. To do this, we
first have to normalise the audio array by the dynamic range of the target dtype (`),$t=i("code"),an=o("int16"),sn=o(`), and then convert from the
default NumPy dtype (`),Gt=i("code"),nn=o("float64"),on=o(") to the target dtype ("),_t=i("code"),ln=o("int16"),rn=o("):"),Sa=c(),d(We.$$.fragment),ka=c(),tt=i("p"),pn=o("Let\u2019s check this concatenated function gives the expected result:"),Ua=c(),d(Ie.$$.fragment),ja=c(),at=i("p"),cn=o(`Perfect! Now we\u2019ll wrap this up into a nice Gradio demo so that we can record our source speech using a microphone input
or file input and playback the system\u2019s prediction:`),$a=c(),d(Ne.$$.fragment),Ga=c(),st=i("p"),hn=o("This will launch a Gradio demo similar to the one running on the Hugging Face Space:"),_a=c(),$=i("iframe"),Ea=c(),ae=i("p"),un=o("You can "),xe=i("a"),fn=o("duplicate"),dn=o(` this demo and adapt
it to use a different Whisper checkpoint, a different TTS checkpoint, or relax the constraint of outputting English
speech and follow the tips provide for translating into a langauge of your choice!`),Za=c(),F=i("h2"),se=i("a"),Et=i("span"),d(Be.$$.fragment),mn=c(),Zt=i("span"),yn=o("Going forwards"),Wa=c(),X=i("p"),gn=o(`While the cascaded system is a compute and data efficient way of building a STST system, it suffers from the issues of
error propagation and additive latency described above. Recent works have explored a `),Wt=i("em"),wn=o("direct"),Tn=o(` approach to STST, one that
does not predict an intermediate text output and instead maps directly from source speech to target speech. These systems
are also capable of retaining the speaking characteristics of the source speaker in the target speech (such a prosody,
pitch and intonation). If you\u2019re interested in finding out more about these systems, check-out the resources listed in
the section on `),nt=i("a"),vn=o("supplemental reading"),Mn=o("."),this.h()},l(e){const s=No('[data-svelte="svelte-1phssyn"]',document.head);f=r(s,"META",{name:!0,content:!0}),s.forEach(t),S=h(e),v=r(e,"H1",{class:!0});var Ve=p(v);G=r(Ve,"A",{id:!0,class:!0,href:!0});var It=p(G);C=r(It,"SPAN",{});var jn=p(C);m(j.$$.fragment,jn),jn.forEach(t),It.forEach(t),oe=h(Ve),R=r(Ve,"SPAN",{});var $n=p(R);A=l($n,"Speech-to-speech translation"),$n.forEach(t),Ve.forEach(t),Z=h(e),k=r(e,"P",{});var Na=p(k);le=l(Na,`Speech-to-speech translation (STST or S2ST) is a relatively new spoken language processing task. It involves translating
speech from one langauge into speech in a `),M=r(Na,"STRONG",{});var Gn=p(M);U=l(Gn,"different"),Gn.forEach(t),Ce=l(Na," language:"),Na.forEach(t),ie=h(e),W=r(e,"DIV",{class:!0});var _n=p(W);Y=r(_n,"IMG",{src:!0,alt:!0}),_n.forEach(t),xt=h(e),I=r(e,"P",{});var ot=p(I);qa=l(ot,"STST can be viewed as an extension of the traditional machine translation (MT) task: instead of translating "),ht=r(ot,"STRONG",{});var En=p(ht);Ha=l(En,"text"),En.forEach(t),Fa=l(ot,` from one
language into another, we translate `),ut=r(ot,"STRONG",{});var Zn=p(ut);Aa=l(Zn,"speech"),Zn.forEach(t),za=l(ot,` from one language into another. STST holds applications in the field of
multilingual communication, enabling speakers in different languages to communicate with one another through the medium
of speech.`),ot.forEach(t),Bt=h(e),Re=r(e,"P",{});var Wn=p(Re);Pa=l(Wn,`Suppose you want to communicate with another individual across a langauge barrier. Rather
than writing the information that you want to convey and then translating it to text in the target language, you
can speak it directly and have a STST system convert your spoken speech into the target langauge. The recipient can then
respond by speaking back at the STST system, and you can listen to their response. This is a more natural way of communicating
compared to text-based machine translation.`),Wn.forEach(t),Vt=h(e),_=r(e,"P",{});var ne=p(_);Da=l(ne,"In this chapter, we\u2019ll explore a "),ft=r(ne,"EM",{});var In=p(ft);Ka=l(In,"cascaded"),In.forEach(t),La=l(ne,` approach to STST, piecing together the knowledge you\u2019ve acquired in Units
5 and 6 of the course. We\u2019ll use a `),dt=r(ne,"EM",{});var Nn=p(dt);Oa=l(Nn,"speech translation (ST)"),Nn.forEach(t),es=l(ne,` system to transcribe the source speech into text in the target
language, then `),mt=r(ne,"EM",{});var xn=p(mt);ts=l(xn,"text-to-speech (TTS)"),xn.forEach(t),as=l(ne," to generate speech in the target language from the translated text:"),ne.forEach(t),Xt=h(e),re=r(e,"DIV",{class:!0});var Bn=p(re);Ye=r(Bn,"IMG",{src:!0,alt:!0}),Bn.forEach(t),Ct=h(e),z=r(e,"P",{});var xa=p(z);ss=l(xa,`We could also have used a three stage approach, where first we use an automatic speech recognition (ASR) system to
transcribe the source speech into text in the same language, then machine translation to translate the transcribed text
into the target language, and finally text-to-speech to generate speech in the target language. However, adding more
components to the pipeline lends itself to `),yt=r(xa,"EM",{});var Vn=p(yt);ns=l(Vn,"error propagation"),Vn.forEach(t),os=l(xa,`, where the errors introduced in one system are compounded
as they flow through the remaining systems, and also increases latency, since inference has to be conducted for more models.`),xa.forEach(t),Rt=h(e),P=r(e,"P",{});var Ba=p(P);ls=l(Ba,`While this cascaded approach to STST is pretty straightforward, it results in very effective STST systems. The three-stage
cascaded system of ASR + MT + TTS was previously used to power many commercial STST products, including `),pe=r(Ba,"A",{href:!0,rel:!0});var Xn=p(pe);is=l(Xn,"Google Translate"),Xn.forEach(t),rs=l(Ba,`.
It\u2019s also a very data and compute efficient way of developing a STST system, since existing speech recognition and
text-to-speech systems can be coupled together to yield a new STST model without any additional training.`),Ba.forEach(t),Yt=h(e),Qe=r(e,"P",{});var Cn=p(Qe);ps=l(Cn,`In the remainder of this Unit, we\u2019ll focus on creating a STST system that translates speech from any language X to speech
in English. The methods covered can be extended to STST systems that translate from any language X to any
langauge Y, but we leave this as an extension to the reader and provide pointers where applicable. We further divide up the
task of STST into its two constituent components: ST and TTS. We\u2019ll finish by piecing them together to build a Gradio
demo to showcase our system.`),Cn.forEach(t),Qt=h(e),Q=r(e,"H2",{class:!0});var Va=p(Q);D=r(Va,"A",{id:!0,class:!0,href:!0});var Rn=p(D);gt=r(Rn,"SPAN",{});var Yn=p(gt);m(ce.$$.fragment,Yn),Yn.forEach(t),Rn.forEach(t),cs=h(Va),wt=r(Va,"SPAN",{});var Qn=p(wt);hs=l(Qn,"Speech translation"),Qn.forEach(t),Va.forEach(t),qt=h(e),N=r(e,"P",{});var lt=p(N);us=l(lt,`We\u2019ll use the Whisper model for our speech translation system, since it\u2019s capable of translating from over 96 languages
to English. Specifically, we\u2019ll load the `),he=r(lt,"A",{href:!0,rel:!0});var qn=p(he);fs=l(qn,"Whisper Base"),qn.forEach(t),ds=l(lt,` checkpoint, which
clocks in at 74M parameters. It\u2019s by no means the most performant Whisper model, with the `),ue=r(lt,"A",{href:!0,rel:!0});var Hn=p(ue);ms=l(Hn,"largest Whisper checkpoint"),Hn.forEach(t),ys=l(lt,`
being over 20x larger, but since we\u2019re concatenating two auto-regressive systems together (ST + TTS), we want to ensure
each model can generate relatively quickly so that we get reasonable inference speed:`),lt.forEach(t),Ht=h(e),m(fe.$$.fragment,e),Ft=h(e),x=r(e,"P",{});var it=p(x);gs=l(it,`Great! To test our STST system, we\u2019ll load an audio sample in a non-English language. Let\u2019s load the first example of the
Italian (`),Tt=r(it,"CODE",{});var Fn=p(Tt);ws=l(Fn,"it"),Fn.forEach(t),Ts=l(it,") split of the "),de=r(it,"A",{href:!0,rel:!0});var An=p(de);vs=l(An,"VoxPopuli"),An.forEach(t),Ms=l(it," dataset:"),it.forEach(t),At=h(e),m(me.$$.fragment,e),zt=h(e),ye=r(e,"P",{});var bn=p(ye);bs=l(bn,"To listen to this sample, we can either play it using the dataset viewer on the Hub: "),ge=r(bn,"A",{href:!0,rel:!0});var zn=p(ge);Js=l(zn,"facebook/voxpopuli/viewer"),zn.forEach(t),bn.forEach(t),Pt=h(e),qe=r(e,"P",{});var Pn=p(qe);Ss=l(Pn,"Or playback using the ipynb audio feature:"),Pn.forEach(t),Dt=h(e),m(we.$$.fragment,e),Kt=h(e),B=r(e,"P",{});var rt=p(B);ks=l(rt,`Now let\u2019s define a function that takes this audio input and returns the translated text. You\u2019ll remember that we have to
pass the generation key-word argument for the `),vt=r(rt,"CODE",{});var Dn=p(vt);Us=l(Dn,'"task"'),Dn.forEach(t),js=l(rt,", setting it to "),Mt=r(rt,"CODE",{});var Kn=p(Mt);$s=l(Kn,'"translate"'),Kn.forEach(t),Gs=l(rt,` to ensure that Whisper performs
speech translation and not speech recognition:`),rt.forEach(t),Lt=h(e),m(Te.$$.fragment,e),Ot=h(e),m(K.$$.fragment,e),ea=h(e),He=r(e,"P",{});var Ln=p(He);_s=l(Ln,"Great! Let\u2019s quickly check that we get a sensible result from the model:"),Ln.forEach(t),ta=h(e),m(ve.$$.fragment,e),aa=h(e),m(Me.$$.fragment,e),sa=h(e),Fe=r(e,"P",{});var On=p(Fe);Es=l(On,"Alright! If we compare this to the source text:"),On.forEach(t),na=h(e),m(be.$$.fragment,e),oa=h(e),m(Je.$$.fragment,e),la=h(e),Ae=r(e,"P",{});var eo=p(Ae);Zs=l(eo,`We see that the translation more or less lines up (you can double check this using Google Translate), barring a small
extra few words at the start of the transcription where the speaker was finishing off their previous sentence.`),eo.forEach(t),ia=h(e),L=r(e,"P",{});var Xa=p(L);Ws=l(Xa,`With that, we\u2019ve completed the first half of our cascaded STST pipeline, putting into practice the skills we gained in Unit 5
when we learnt how to use the Whisper model for speech recognition and translation. If you want a refresher on any of the
steps we covered, have a read through the section on `),ze=r(Xa,"A",{href:!0});var to=p(ze);Is=l(to,"Pre-trained models for ASR"),to.forEach(t),Ns=l(Xa," from Unit 5."),Xa.forEach(t),ra=h(e),q=r(e,"H2",{class:!0});var Ca=p(q);O=r(Ca,"A",{id:!0,class:!0,href:!0});var ao=p(O);bt=r(ao,"SPAN",{});var so=p(bt);m(Se.$$.fragment,so),so.forEach(t),ao.forEach(t),xs=h(Ca),Jt=r(Ca,"SPAN",{});var no=p(Jt);Bs=l(no,"Text-to-speech"),no.forEach(t),Ca.forEach(t),pa=h(e),V=r(e,"P",{});var pt=p(V);Vs=l(pt,`The second half of our cascaded STST system involves mapping from English text to English speech. For this, we\u2019ll use
the pre-trained `),ke=r(pt,"A",{href:!0,rel:!0});var oo=p(ke);Xs=l(oo,"SpeechT5 TTS"),oo.forEach(t),Cs=l(pt,` model for English TTS. \u{1F917} Transformers currently doesn\u2019t
have a TTS `),St=r(pt,"CODE",{});var lo=p(St);Rs=l(lo,"pipeline"),lo.forEach(t),Ys=l(pt,`, so we\u2019ll have to use the model directly ourselves. This is no biggie, you\u2019re all experts on using
the model for inference following Unit 6!`),pt.forEach(t),ca=h(e),Pe=r(e,"P",{});var io=p(Pe);Qs=l(io,"First, let\u2019s load the SpeechT5 processor, model and vocoder from the pre-trained checkpoint:"),io.forEach(t),ha=h(e),m(Ue.$$.fragment,e),ua=h(e),m(ee.$$.fragment,e),fa=h(e),De=r(e,"P",{});var ro=p(De);qs=l(ro,"As with the Whisper model, we\u2019ll place the SpeechT5 model and vocoder on our GPU accelerator device if we have one:"),ro.forEach(t),da=h(e),m(je.$$.fragment,e),ma=h(e),Ke=r(e,"P",{});var po=p(Ke);Hs=l(po,"Great! Let\u2019s load up the speaker embeddings:"),po.forEach(t),ya=h(e),m($e.$$.fragment,e),ga=h(e),Le=r(e,"P",{});var co=p(Le);Fs=l(co,`We can now write a function that takes a text prompt as input, and generates the corresponding speech. We\u2019ll first pre-process
the text input using the SpeechT5 processor, tokenizing the text to get our input ids. We\u2019ll then pass the input ids and
speaker embeddings to the SpeechT5 model, placing each on the accelerator device if available. Finally, we\u2019ll return the
generated speech, bringing it back to the CPU so that we can play it back in our ipynb notebook:`),co.forEach(t),wa=h(e),m(Ge.$$.fragment,e),Ta=h(e),Oe=r(e,"P",{});var ho=p(Oe);As=l(ho,"Let\u2019s check it works with a dummy text input:"),ho.forEach(t),va=h(e),m(_e.$$.fragment,e),Ma=h(e),et=r(e,"P",{});var uo=p(et);zs=l(uo,"Sounds good! Now for the exciting part - piecing it all together."),uo.forEach(t),ba=h(e),H=r(e,"H2",{class:!0});var Ra=p(H);te=r(Ra,"A",{id:!0,class:!0,href:!0});var fo=p(te);kt=r(fo,"SPAN",{});var mo=p(kt);m(Ee.$$.fragment,mo),mo.forEach(t),fo.forEach(t),Ps=h(Ra),Ut=r(Ra,"SPAN",{});var yo=p(Ut);Ds=l(yo,"Creating a STST demo"),yo.forEach(t),Ra.forEach(t),Ja=h(e),b=r(e,"P",{});var E=p(b);Ks=l(E,"Before we create a "),Ze=r(E,"A",{href:!0,rel:!0});var go=p(Ze);Ls=l(go,"Gradio"),go.forEach(t),Os=l(E,` demo to showcase our STST system, let\u2019s first do a quick sanity check
to make sure we can concatenate the two models, putting an audio sample in and getting an audio sample out. We\u2019ll do
this by concatenating the two functions we defined in the previous two sub-sections, such that we input the source audio
and retrieve the translated text, then synthesise the translated text to get the translated speech. Finally, we\u2019ll convert
the synthesised speech to an `),jt=r(E,"CODE",{});var wo=p(jt);en=l(wo,"int16"),wo.forEach(t),tn=l(E,` array, which is the output audio file format expected by Gradio. To do this, we
first have to normalise the audio array by the dynamic range of the target dtype (`),$t=r(E,"CODE",{});var To=p($t);an=l(To,"int16"),To.forEach(t),sn=l(E,`), and then convert from the
default NumPy dtype (`),Gt=r(E,"CODE",{});var vo=p(Gt);nn=l(vo,"float64"),vo.forEach(t),on=l(E,") to the target dtype ("),_t=r(E,"CODE",{});var Mo=p(_t);ln=l(Mo,"int16"),Mo.forEach(t),rn=l(E,"):"),E.forEach(t),Sa=h(e),m(We.$$.fragment,e),ka=h(e),tt=r(e,"P",{});var bo=p(tt);pn=l(bo,"Let\u2019s check this concatenated function gives the expected result:"),bo.forEach(t),Ua=h(e),m(Ie.$$.fragment,e),ja=h(e),at=r(e,"P",{});var Jo=p(at);cn=l(Jo,`Perfect! Now we\u2019ll wrap this up into a nice Gradio demo so that we can record our source speech using a microphone input
or file input and playback the system\u2019s prediction:`),Jo.forEach(t),$a=h(e),m(Ne.$$.fragment,e),Ga=h(e),st=r(e,"P",{});var So=p(st);hn=l(So,"This will launch a Gradio demo similar to the one running on the Hugging Face Space:"),So.forEach(t),_a=h(e),$=r(e,"IFRAME",{src:!0,frameborder:!0,height:!0,title:!0,class:!0,allow:!0,sandbox:!0}),p($).forEach(t),Ea=h(e),ae=r(e,"P",{});var Ya=p(ae);un=l(Ya,"You can "),xe=r(Ya,"A",{href:!0,rel:!0});var ko=p(xe);fn=l(ko,"duplicate"),ko.forEach(t),dn=l(Ya,` this demo and adapt
it to use a different Whisper checkpoint, a different TTS checkpoint, or relax the constraint of outputting English
speech and follow the tips provide for translating into a langauge of your choice!`),Ya.forEach(t),Za=h(e),F=r(e,"H2",{class:!0});var Qa=p(F);se=r(Qa,"A",{id:!0,class:!0,href:!0});var Uo=p(se);Et=r(Uo,"SPAN",{});var jo=p(Et);m(Be.$$.fragment,jo),jo.forEach(t),Uo.forEach(t),mn=h(Qa),Zt=r(Qa,"SPAN",{});var $o=p(Zt);yn=l($o,"Going forwards"),$o.forEach(t),Qa.forEach(t),Wa=h(e),X=r(e,"P",{});var ct=p(X);gn=l(ct,`While the cascaded system is a compute and data efficient way of building a STST system, it suffers from the issues of
error propagation and additive latency described above. Recent works have explored a `),Wt=r(ct,"EM",{});var Go=p(Wt);wn=l(Go,"direct"),Go.forEach(t),Tn=l(ct,` approach to STST, one that
does not predict an intermediate text output and instead maps directly from source speech to target speech. These systems
are also capable of retaining the speaking characteristics of the source speaker in the target speech (such a prosody,
pitch and intonation). If you\u2019re interested in finding out more about these systems, check-out the resources listed in
the section on `),nt=r(ct,"A",{href:!0});var _o=p(nt);vn=l(_o,"supplemental reading"),_o.forEach(t),Mn=l(ct,"."),ct.forEach(t),this.h()},h(){u(f,"name","hf:doc:metadata"),u(f,"content",JSON.stringify(Co)),u(G,"id","speechtospeech-translation"),u(G,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(G,"href","#speechtospeech-translation"),u(v,"class","relative group"),Jn(Y.src,Sn="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st.png")||u(Y,"src",Sn),u(Y,"alt","Diagram of speech to speech translation"),u(W,"class","flex justify-center"),Jn(Ye.src,kn="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/s2st_cascaded.png")||u(Ye,"src",kn),u(Ye,"alt","Diagram of cascaded speech to speech translation"),u(re,"class","flex justify-center"),u(pe,"href","https://ai.googleblog.com/2019/05/introducing-translatotron-end-to-end.html"),u(pe,"rel","nofollow"),u(D,"id","speech-translation"),u(D,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(D,"href","#speech-translation"),u(Q,"class","relative group"),u(he,"href","https://huggingface.co/openai/whisper-base"),u(he,"rel","nofollow"),u(ue,"href","https://huggingface.co/openai/whisper-large-v2"),u(ue,"rel","nofollow"),u(de,"href","https://huggingface.co/datasets/facebook/voxpopuli"),u(de,"rel","nofollow"),u(ge,"href","https://huggingface.co/datasets/facebook/voxpopuli/viewer/it/validation?row=0"),u(ge,"rel","nofollow"),u(ze,"href","../chapter5/asr_models"),u(O,"id","texttospeech"),u(O,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(O,"href","#texttospeech"),u(q,"class","relative group"),u(ke,"href","https://huggingface.co/microsoft/speecht5_tts"),u(ke,"rel","nofollow"),u(te,"id","creating-a-stst-demo"),u(te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(te,"href","#creating-a-stst-demo"),u(H,"class","relative group"),u(Ze,"href","https://gradio.app"),u(Ze,"rel","nofollow"),Jn($.src,Un="https://course-demos-speech-to-speech-translation.hf.space")||u($,"src",Un),u($,"frameborder","0"),u($,"height","450"),u($,"title","Gradio app"),u($,"class","container p-0 flex-grow space-iframe"),u($,"allow","accelerometer; ambient-light-sensor; autoplay; battery; camera; document-domain; encrypted-media; fullscreen; geolocation; gyroscope; layout-animations; legacy-image-formats; magnetometer; microphone; midi; oversized-images; payment; picture-in-picture; publickey-credentials-get; sync-xhr; usb; vr ; wake-lock; xr-spatial-tracking"),u($,"sandbox","allow-forms allow-modals allow-popups allow-popups-to-escape-sandbox allow-same-origin allow-scripts allow-downloads"),u(xe,"href","https://huggingface.co/spaces/course-demos/speech-to-speech-translation?duplicate=true"),u(xe,"rel","nofollow"),u(se,"id","going-forwards"),u(se,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),u(se,"href","#going-forwards"),u(F,"class","relative group"),u(nt,"href","supplemental_reading")},m(e,s){a(document.head,f),n(e,S,s),n(e,v,s),a(v,G),a(G,C),y(j,C,null),a(v,oe),a(v,R),a(R,A),n(e,Z,s),n(e,k,s),a(k,le),a(k,M),a(M,U),a(k,Ce),n(e,ie,s),n(e,W,s),a(W,Y),n(e,xt,s),n(e,I,s),a(I,qa),a(I,ht),a(ht,Ha),a(I,Fa),a(I,ut),a(ut,Aa),a(I,za),n(e,Bt,s),n(e,Re,s),a(Re,Pa),n(e,Vt,s),n(e,_,s),a(_,Da),a(_,ft),a(ft,Ka),a(_,La),a(_,dt),a(dt,Oa),a(_,es),a(_,mt),a(mt,ts),a(_,as),n(e,Xt,s),n(e,re,s),a(re,Ye),n(e,Ct,s),n(e,z,s),a(z,ss),a(z,yt),a(yt,ns),a(z,os),n(e,Rt,s),n(e,P,s),a(P,ls),a(P,pe),a(pe,is),a(P,rs),n(e,Yt,s),n(e,Qe,s),a(Qe,ps),n(e,Qt,s),n(e,Q,s),a(Q,D),a(D,gt),y(ce,gt,null),a(Q,cs),a(Q,wt),a(wt,hs),n(e,qt,s),n(e,N,s),a(N,us),a(N,he),a(he,fs),a(N,ds),a(N,ue),a(ue,ms),a(N,ys),n(e,Ht,s),y(fe,e,s),n(e,Ft,s),n(e,x,s),a(x,gs),a(x,Tt),a(Tt,ws),a(x,Ts),a(x,de),a(de,vs),a(x,Ms),n(e,At,s),y(me,e,s),n(e,zt,s),n(e,ye,s),a(ye,bs),a(ye,ge),a(ge,Js),n(e,Pt,s),n(e,qe,s),a(qe,Ss),n(e,Dt,s),y(we,e,s),n(e,Kt,s),n(e,B,s),a(B,ks),a(B,vt),a(vt,Us),a(B,js),a(B,Mt),a(Mt,$s),a(B,Gs),n(e,Lt,s),y(Te,e,s),n(e,Ot,s),y(K,e,s),n(e,ea,s),n(e,He,s),a(He,_s),n(e,ta,s),y(ve,e,s),n(e,aa,s),y(Me,e,s),n(e,sa,s),n(e,Fe,s),a(Fe,Es),n(e,na,s),y(be,e,s),n(e,oa,s),y(Je,e,s),n(e,la,s),n(e,Ae,s),a(Ae,Zs),n(e,ia,s),n(e,L,s),a(L,Ws),a(L,ze),a(ze,Is),a(L,Ns),n(e,ra,s),n(e,q,s),a(q,O),a(O,bt),y(Se,bt,null),a(q,xs),a(q,Jt),a(Jt,Bs),n(e,pa,s),n(e,V,s),a(V,Vs),a(V,ke),a(ke,Xs),a(V,Cs),a(V,St),a(St,Rs),a(V,Ys),n(e,ca,s),n(e,Pe,s),a(Pe,Qs),n(e,ha,s),y(Ue,e,s),n(e,ua,s),y(ee,e,s),n(e,fa,s),n(e,De,s),a(De,qs),n(e,da,s),y(je,e,s),n(e,ma,s),n(e,Ke,s),a(Ke,Hs),n(e,ya,s),y($e,e,s),n(e,ga,s),n(e,Le,s),a(Le,Fs),n(e,wa,s),y(Ge,e,s),n(e,Ta,s),n(e,Oe,s),a(Oe,As),n(e,va,s),y(_e,e,s),n(e,Ma,s),n(e,et,s),a(et,zs),n(e,ba,s),n(e,H,s),a(H,te),a(te,kt),y(Ee,kt,null),a(H,Ps),a(H,Ut),a(Ut,Ds),n(e,Ja,s),n(e,b,s),a(b,Ks),a(b,Ze),a(Ze,Ls),a(b,Os),a(b,jt),a(jt,en),a(b,tn),a(b,$t),a($t,an),a(b,sn),a(b,Gt),a(Gt,nn),a(b,on),a(b,_t),a(_t,ln),a(b,rn),n(e,Sa,s),y(We,e,s),n(e,ka,s),n(e,tt,s),a(tt,pn),n(e,Ua,s),y(Ie,e,s),n(e,ja,s),n(e,at,s),a(at,cn),n(e,$a,s),y(Ne,e,s),n(e,Ga,s),n(e,st,s),a(st,hn),n(e,_a,s),n(e,$,s),n(e,Ea,s),n(e,ae,s),a(ae,un),a(ae,xe),a(xe,fn),a(ae,dn),n(e,Za,s),n(e,F,s),a(F,se),a(se,Et),y(Be,Et,null),a(F,mn),a(F,Zt),a(Zt,yn),n(e,Wa,s),n(e,X,s),a(X,gn),a(X,Wt),a(Wt,wn),a(X,Tn),a(X,nt),a(nt,vn),a(X,Mn),Ia=!0},p(e,[s]){const Ve={};s&2&&(Ve.$$scope={dirty:s,ctx:e}),K.$set(Ve);const It={};s&2&&(It.$$scope={dirty:s,ctx:e}),ee.$set(It)},i(e){Ia||(g(j.$$.fragment,e),g(ce.$$.fragment,e),g(fe.$$.fragment,e),g(me.$$.fragment,e),g(we.$$.fragment,e),g(Te.$$.fragment,e),g(K.$$.fragment,e),g(ve.$$.fragment,e),g(Me.$$.fragment,e),g(be.$$.fragment,e),g(Je.$$.fragment,e),g(Se.$$.fragment,e),g(Ue.$$.fragment,e),g(ee.$$.fragment,e),g(je.$$.fragment,e),g($e.$$.fragment,e),g(Ge.$$.fragment,e),g(_e.$$.fragment,e),g(Ee.$$.fragment,e),g(We.$$.fragment,e),g(Ie.$$.fragment,e),g(Ne.$$.fragment,e),g(Be.$$.fragment,e),Ia=!0)},o(e){w(j.$$.fragment,e),w(ce.$$.fragment,e),w(fe.$$.fragment,e),w(me.$$.fragment,e),w(we.$$.fragment,e),w(Te.$$.fragment,e),w(K.$$.fragment,e),w(ve.$$.fragment,e),w(Me.$$.fragment,e),w(be.$$.fragment,e),w(Je.$$.fragment,e),w(Se.$$.fragment,e),w(Ue.$$.fragment,e),w(ee.$$.fragment,e),w(je.$$.fragment,e),w($e.$$.fragment,e),w(Ge.$$.fragment,e),w(_e.$$.fragment,e),w(Ee.$$.fragment,e),w(We.$$.fragment,e),w(Ie.$$.fragment,e),w(Ne.$$.fragment,e),w(Be.$$.fragment,e),Ia=!1},d(e){t(f),e&&t(S),e&&t(v),T(j),e&&t(Z),e&&t(k),e&&t(ie),e&&t(W),e&&t(xt),e&&t(I),e&&t(Bt),e&&t(Re),e&&t(Vt),e&&t(_),e&&t(Xt),e&&t(re),e&&t(Ct),e&&t(z),e&&t(Rt),e&&t(P),e&&t(Yt),e&&t(Qe),e&&t(Qt),e&&t(Q),T(ce),e&&t(qt),e&&t(N),e&&t(Ht),T(fe,e),e&&t(Ft),e&&t(x),e&&t(At),T(me,e),e&&t(zt),e&&t(ye),e&&t(Pt),e&&t(qe),e&&t(Dt),T(we,e),e&&t(Kt),e&&t(B),e&&t(Lt),T(Te,e),e&&t(Ot),T(K,e),e&&t(ea),e&&t(He),e&&t(ta),T(ve,e),e&&t(aa),T(Me,e),e&&t(sa),e&&t(Fe),e&&t(na),T(be,e),e&&t(oa),T(Je,e),e&&t(la),e&&t(Ae),e&&t(ia),e&&t(L),e&&t(ra),e&&t(q),T(Se),e&&t(pa),e&&t(V),e&&t(ca),e&&t(Pe),e&&t(ha),T(Ue,e),e&&t(ua),T(ee,e),e&&t(fa),e&&t(De),e&&t(da),T(je,e),e&&t(ma),e&&t(Ke),e&&t(ya),T($e,e),e&&t(ga),e&&t(Le),e&&t(wa),T(Ge,e),e&&t(Ta),e&&t(Oe),e&&t(va),T(_e,e),e&&t(Ma),e&&t(et),e&&t(ba),e&&t(H),T(Ee),e&&t(Ja),e&&t(b),e&&t(Sa),T(We,e),e&&t(ka),e&&t(tt),e&&t(Ua),T(Ie,e),e&&t(ja),e&&t(at),e&&t($a),T(Ne,e),e&&t(Ga),e&&t(st),e&&t(_a),e&&t($),e&&t(Ea),e&&t(ae),e&&t(Za),e&&t(F),T(Be),e&&t(Wa),e&&t(X)}}}const Co={local:"speechtospeech-translation",sections:[{local:"speech-translation",title:"Speech translation"},{local:"texttospeech",title:"Text-to-speech"},{local:"creating-a-stst-demo",title:"Creating a STST demo"},{local:"going-forwards",title:"Going forwards"}],title:"Speech-to-speech translation"};function Ro(Xe){return xo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fo extends Zo{constructor(f){super();Wo(this,f,Ro,Xo,Io,{})}}export{Fo as default,Co as metadata};
