import{S as wd,i as yd,s as bd,e as n,k as c,w as u,t as o,M as gd,c as i,d as t,m as p,a as h,x as f,h as l,b as d,N as Lt,G as s,g as r,y as m,q as w,o as y,B as b,v as Md}from"../../chunks/vendor-hf-doc-builder.js";import{T as md}from"../../chunks/Tip-hf-doc-builder.js";import{I as N}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as v}from"../../chunks/CodeBlock-hf-doc-builder.js";function vd(Qt){let k,T,j,E,I,S,se;return{c(){k=o("If you're using a GPU, you can increase the checkpoint size to use the "),T=n("a"),j=o("Whisper Small English"),E=o(` 
    checkpoint, which will return better transcription accuracy and still be within the required latency threshold. Simply swap the 
    model id to: `),I=n("code"),S=o('"openai/whisper-small.en"'),se=o("."),this.h()},l(g){k=l(g,"If you're using a GPU, you can increase the checkpoint size to use the "),T=i(g,"A",{href:!0});var $=h(T);j=l($,"Whisper Small English"),$.forEach(t),E=l(g,` 
    checkpoint, which will return better transcription accuracy and still be within the required latency threshold. Simply swap the 
    model id to: `),I=i(g,"CODE",{});var Be=h(I);S=l(Be,'"openai/whisper-small.en"'),Be.forEach(t),se=l(g,"."),this.h()},h(){d(T,"href","https://huggingface.co/openai/whisper-small.en")},m(g,$){r(g,k,$),r(g,T,$),s(T,j),r(g,E,$),r(g,I,$),s(I,S),r(g,se,$)},d(g){g&&t(k),g&&t(T),g&&t(E),g&&t(I),g&&t(se)}}}function kd(Qt){let k;return{c(){k=o(`Note that the first time calling this will trigger the model weights to be downloaded, which might take 
    some time depending on your Hub download speed.`)},l(T){k=l(T,`Note that the first time calling this will trigger the model weights to be downloaded, which might take 
    some time depending on your Hub download speed.`)},m(T,j){r(T,k,j)},d(T){T&&t(k)}}}function Td(Qt){let k,T,j,E,I,S,se,g,$,Be,ae,nn,Ts,rn,hn,qa,Rt,cn,La,xe,Js,Yh,Qa,R,oe,js,Ge,pn,Us,dn,Ra,Xt,un,Xa,Ft,fn,Fa,X,le,_s,He,mn,Es,wn,za,zt,yn,Pa,Pt,bn,Da,F,ne,Is,Ve,gn,$s,Mn,Oa,ie,vn,Ws,kn,Tn,Ka,Dt,Jn,eo,z,re,As,Ye,jn,Cs,Un,to,Ot,_n,so,Kt,En,ao,P,he,Zs,qe,In,Ss,$n,oo,M,Wn,es,An,Cn,Le,Zn,Sn,Ns,Nn,Bn,Bs,xn,Gn,xs,Hn,Vn,Gs,Yn,qn,Hs,Ln,Qn,Qe,Rn,Xn,lo,ts,Fn,no,Re,zn,Xe,Pn,io,ss,Dn,ro,Fe,Vs,qh,ho,as,On,co,ze,Ys,Lh,po,ce,Kn,qs,ei,ti,uo,Pe,Ls,Qh,fo,pe,si,De,ai,oi,mo,de,li,Qs,ni,ii,wo,Oe,yo,ue,ri,Rs,hi,ci,bo,Ke,go,U,pi,Xs,di,ui,Fs,fi,mi,zs,wi,yi,Ps,bi,gi,Mo,et,vo,tt,ko,fe,Mi,Ds,vi,ki,To,me,Ti,st,Os,Ji,ji,Jo,_,Ui,Ks,_i,Ei,ea,Ii,$i,ta,Wi,Ai,sa,Ci,Zi,jo,W,Si,aa,Ni,Bi,oa,xi,Gi,la,Hi,Vi,Uo,B,Yi,na,qi,Li,at,ia,Qi,Ri,_o,ot,Eo,A,Xi,ra,Fi,zi,ha,Pi,Di,ca,Oi,Ki,Io,lt,$o,nt,Wo,we,er,pa,tr,sr,Ao,D,ye,da,it,ar,ua,or,Co,x,lr,rt,nr,ir,ht,rr,hr,Zo,ct,So,be,No,G,cr,fa,pr,dr,ma,ur,fr,Bo,C,mr,wa,wr,yr,ya,br,gr,pt,Mr,vr,xo,dt,Go,os,kr,Ho,ut,Vo,ft,Yo,Z,Tr,ba,Jr,jr,ga,Ur,_r,Ma,Er,Ir,qo,O,ge,va,mt,$r,ka,Wr,Lo,Me,Ar,wt,Cr,Zr,Qo,ve,Sr,yt,Nr,Br,Ro,bt,Ta,Rh,Xo,H,xr,gt,Gr,Hr,Mt,Vr,Yr,Fo,vt,Ja,Xh,zo,ke,qr,ja,Lr,Qr,Po,kt,Do,ls,Rr,Oo,Tt,Ko,Jt,el,ns,Xr,tl,K,Te,Ua,jt,Fr,_a,zr,sl,Je,Pr,Ut,Dr,Or,al,_t,ol,is,Kr,ll,Et,nl,V,eh,Ea,th,sh,rs,ah,oh,il,It,rl,hs,lh,hl,$t,cl,cs,nh,pl,ee,je,Ia,Wt,ih,$a,rh,dl,Ue,hh,Wa,ch,ph,ul,At,fl,ps,dh,ml,Y,Aa,Ca,uh,fh,Za,Sa,mh,wh,Na,Ba,yh,wl,ds,bh,yl,te,_e,xa,Ct,gh,Ga,Mh,bl,us,vh,gl,Ee,kh,Zt,Th,Jh,Ml,q,jh,St,Uh,_h,Nt,Eh,Ih,vl,Bt,kl,Ie,$h,Ha,Wh,Ah,Tl,xt,Jl,Gt,Va,Fh,jl,$e,Ul,We,Ch,Ht,Zh,Sh,_l,fs,Nh,El,Vt,Il,ms,Bh,$l,Ae,xh,Yt,Gh,Hh,Wl;return S=new N({}),Ge=new N({}),He=new N({}),Ve=new N({}),Ye=new N({}),qe=new N({}),Oe=new v({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEElMEFkZXZpY2UlMjAlM0QlMjAlMjJjdWRhJTNBMCUyMiUyMGlmJTIwdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSUyMGVsc2UlMjAlMjJjcHUlMjIlMEElMEFjbGFzc2lmaWVyJTIwJTNEJTIwcGlwZWxpbmUoJTBBJTIwJTIwJTIwJTIwJTIyYXVkaW8tY2xhc3NpZmljYXRpb24lMjIlMkMlMjBtb2RlbCUzRCUyMk1JVCUyRmFzdC1maW5ldHVuZWQtc3BlZWNoLWNvbW1hbmRzLXYyJTIyJTJDJTIwZGV2aWNlJTNEZGV2aWNlJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">import</span> torch

device = <span class="hljs-string">&quot;cuda:0&quot;</span> <span class="hljs-keyword">if</span> torch.cuda.is_available() <span class="hljs-keyword">else</span> <span class="hljs-string">&quot;cpu&quot;</span>

classifier = pipeline(
    <span class="hljs-string">&quot;audio-classification&quot;</span>, model=<span class="hljs-string">&quot;MIT/ast-finetuned-speech-commands-v2&quot;</span>, device=device
)`}}),Ke=new v({props:{code:"Y2xhc3NpZmllci5tb2RlbC5jb25maWcuaWQybGFiZWw=",highlighted:"classifier.model.config.id2label"}}),et=new v({props:{code:"Y2xhc3NpZmllci5tb2RlbC5jb25maWcuaWQybGFiZWwlNUIyNyU1RA==",highlighted:'classifier.model.config.id2label[<span class="hljs-number">27</span>]'}}),tt=new v({props:{code:"J21hcnZpbic=",highlighted:'<span class="hljs-symbol">&#x27;marvin</span>&#x27;'}}),ot=new v({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5waXBlbGluZXMuYXVkaW9fdXRpbHMlMjBpbXBvcnQlMjBmZm1wZWdfbWljcm9waG9uZV9saXZlJTBBJTBBJTBBZGVmJTIwbGF1bmNoX2ZuKCUwQSUyMCUyMCUyMCUyMHdha2Vfd29yZCUzRCUyMm1hcnZpbiUyMiUyQyUwQSUyMCUyMCUyMCUyMHByb2JfdGhyZXNob2xkJTNEMC41JTJDJTBBJTIwJTIwJTIwJTIwY2h1bmtfbGVuZ3RoX3MlM0QyLjAlMkMlMEElMjAlMjAlMjAlMjBzdHJlYW1fY2h1bmtfcyUzRDAuMjUlMkMlMEElMjAlMjAlMjAlMjBkZWJ1ZyUzREZhbHNlJTJDJTBBKSUzQSUwQSUyMCUyMCUyMCUyMGlmJTIwd2FrZV93b3JkJTIwbm90JTIwaW4lMjBjbGFzc2lmaWVyLm1vZGVsLmNvbmZpZy5sYWJlbDJpZC5rZXlzKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjByYWlzZSUyMFZhbHVlRXJyb3IoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZiUyMldha2UlMjB3b3JkJTIwJTdCd2FrZV93b3JkJTdEJTIwbm90JTIwaW4lMjBzZXQlMjBvZiUyMHZhbGlkJTIwY2xhc3MlMjBsYWJlbHMlMkMlMjBwaWNrJTIwYSUyMHdha2UlMjB3b3JkJTIwaW4lMjB0aGUlMjBzZXQlMjAlN0JjbGFzc2lmaWVyLm1vZGVsLmNvbmZpZy5sYWJlbDJpZC5rZXlzKCklN0QuJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMHNhbXBsaW5nX3JhdGUlMjAlM0QlMjBjbGFzc2lmaWVyLmZlYXR1cmVfZXh0cmFjdG9yLnNhbXBsaW5nX3JhdGUlMEElMEElMjAlMjAlMjAlMjBtaWMlMjAlM0QlMjBmZm1wZWdfbWljcm9waG9uZV9saXZlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNhbXBsaW5nX3JhdGUlM0RzYW1wbGluZ19yYXRlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2h1bmtfbGVuZ3RoX3MlM0RjaHVua19sZW5ndGhfcyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHN0cmVhbV9jaHVua19zJTNEc3RyZWFtX2NodW5rX3MlMkMlMEElMjAlMjAlMjAlMjApJTBBJTBBJTIwJTIwJTIwJTIwcHJpbnQoJTIyTGlzdGVuaW5nJTIwZm9yJTIwd2FrZSUyMHdvcmQuLi4lMjIpJTBBJTIwJTIwJTIwJTIwZm9yJTIwcHJlZGljdGlvbiUyMGluJTIwY2xhc3NpZmllcihtaWMpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJlZGljdGlvbiUyMCUzRCUyMHByZWRpY3Rpb24lNUIwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBkZWJ1ZyUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KHByZWRpY3Rpb24pJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaWYlMjBwcmVkaWN0aW9uJTVCJTIybGFiZWwlMjIlNUQlMjAlM0QlM0QlMjB3YWtlX3dvcmQlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMHByZWRpY3Rpb24lNUIlMjJzY29yZSUyMiU1RCUyMCUzRSUyMHByb2JfdGhyZXNob2xkJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwVHJ1ZQ==",highlighted:`<span class="hljs-keyword">from</span> transformers.pipelines.audio_utils <span class="hljs-keyword">import</span> ffmpeg_microphone_live


<span class="hljs-keyword">def</span> <span class="hljs-title function_">launch_fn</span>(<span class="hljs-params">
    wake_word=<span class="hljs-string">&quot;marvin&quot;</span>,
    prob_threshold=<span class="hljs-number">0.5</span>,
    chunk_length_s=<span class="hljs-number">2.0</span>,
    stream_chunk_s=<span class="hljs-number">0.25</span>,
    debug=<span class="hljs-literal">False</span>,
</span>):
    <span class="hljs-keyword">if</span> wake_word <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> classifier.model.config.label2id.keys():
        <span class="hljs-keyword">raise</span> ValueError(
            <span class="hljs-string">f&quot;Wake word <span class="hljs-subst">{wake_word}</span> not in set of valid class labels, pick a wake word in the set <span class="hljs-subst">{classifier.model.config.label2id.keys()}</span>.&quot;</span>
        )

    sampling_rate = classifier.feature_extractor.sampling_rate

    mic = ffmpeg_microphone_live(
        sampling_rate=sampling_rate,
        chunk_length_s=chunk_length_s,
        stream_chunk_s=stream_chunk_s,
    )

    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Listening for wake word...&quot;</span>)
    <span class="hljs-keyword">for</span> prediction <span class="hljs-keyword">in</span> classifier(mic):
        prediction = prediction[<span class="hljs-number">0</span>]
        <span class="hljs-keyword">if</span> debug:
            <span class="hljs-built_in">print</span>(prediction)
        <span class="hljs-keyword">if</span> prediction[<span class="hljs-string">&quot;label&quot;</span>] == wake_word:
            <span class="hljs-keyword">if</span> prediction[<span class="hljs-string">&quot;score&quot;</span>] &gt; prob_threshold:
                <span class="hljs-keyword">return</span> <span class="hljs-literal">True</span>`}}),lt=new v({props:{code:"bGF1bmNoX2ZuKGRlYnVnJTNEVHJ1ZSk=",highlighted:'launch_fn(debug=<span class="hljs-literal">True</span>)'}}),nt=new v({props:{code:"TGlzdGVuaW5nJTIwZm9yJTIwd2FrZSUyMHdvcmQuLi4lMEElN0Inc2NvcmUnJTNBJTIwMC4wNTUzMjYxMDc4ODk0MTM4MzQlMkMlMjAnbGFiZWwnJTNBJTIwJ29uZSclN0QlMEElN0Inc2NvcmUnJTNBJTIwMC4wNTk5OTg1NjgxNDc0MjA4OCUyQyUyMCdsYWJlbCclM0ElMjAnb2ZmJyU3RCUwQSU3QidzY29yZSclM0ElMjAwLjEyODI3NDg0MzA5NjczMzElMkMlMjAnbGFiZWwnJTNBJTIwJ2ZpdmUnJTdEJTBBJTdCJ3Njb3JlJyUzQSUyMDAuMDczMTAxMTAzMzA1ODE2NjUlMkMlMjAnbGFiZWwnJTNBJTIwJ2ZvbGxvdyclN0QlMEElN0Inc2NvcmUnJTNBJTIwMC4wNjYzNDgwOTgyMTg0NDEwMSUyQyUyMCdsYWJlbCclM0ElMjAnZm9sbG93JyU3RCUwQSU3QidzY29yZSclM0ElMjAwLjA1OTkyNjQyNDE3NTUwMDg3JTJDJTIwJ2xhYmVsJyUzQSUyMCd0cmVlJyU3RCUwQSU3QidzY29yZSclM0ElMjAwLjA1OTkyNjQyNDE3NTUwMDg3JTJDJTIwJ2xhYmVsJyUzQSUyMCd0cmVlJyU3RCUwQSU3QidzY29yZSclM0ElMjAwLjk5OTkxMzIxNTYzNzIwNyUyQyUyMCdsYWJlbCclM0ElMjAnbWFydmluJyU3RA==",highlighted:`Listening for wake word...
{&#x27;score&#x27;: 0.055326107889413834, &#x27;label&#x27;: &#x27;one&#x27;}
{&#x27;score&#x27;: 0.05999856814742088, &#x27;label&#x27;: &#x27;off&#x27;}
{&#x27;score&#x27;: 0.1282748430967331, &#x27;label&#x27;: &#x27;five&#x27;}
{&#x27;score&#x27;: 0.07310110330581665, &#x27;label&#x27;: &#x27;follow&#x27;}
{&#x27;score&#x27;: 0.06634809821844101, &#x27;label&#x27;: &#x27;follow&#x27;}
{&#x27;score&#x27;: 0.05992642417550087, &#x27;label&#x27;: &#x27;tree&#x27;}
{&#x27;score&#x27;: 0.05992642417550087, &#x27;label&#x27;: &#x27;tree&#x27;}
{&#x27;score&#x27;: 0.999913215637207, &#x27;label&#x27;: &#x27;marvin&#x27;}`}}),it=new N({}),ct=new v({props:{code:"dHJhbnNjcmliZXIlMjAlM0QlMjBwaXBlbGluZSglMEElMjAlMjAlMjAlMjAlMjJhdXRvbWF0aWMtc3BlZWNoLXJlY29nbml0aW9uJTIyJTJDJTIwbW9kZWwlM0QlMjJvcGVuYWklMkZ3aGlzcGVyLWJhc2UuZW4lMjIlMkMlMjBkZXZpY2UlM0RkZXZpY2UlMEEp",highlighted:`transcriber = pipeline(
    <span class="hljs-string">&quot;automatic-speech-recognition&quot;</span>, model=<span class="hljs-string">&quot;openai/whisper-base.en&quot;</span>, device=device
)`}}),be=new md({props:{$$slots:{default:[vd]},$$scope:{ctx:Qt}}}),dt=new v({props:{code:"aW1wb3J0JTIwc3lzJTBBJTBBJTBBZGVmJTIwdHJhbnNjcmliZShjaHVua19sZW5ndGhfcyUzRDUuMCUyQyUyMHN0cmVhbV9jaHVua19zJTNEMS4wKSUzQSUwQSUyMCUyMCUyMCUyMHNhbXBsaW5nX3JhdGUlMjAlM0QlMjB0cmFuc2NyaWJlci5mZWF0dXJlX2V4dHJhY3Rvci5zYW1wbGluZ19yYXRlJTBBJTBBJTIwJTIwJTIwJTIwbWljJTIwJTNEJTIwZmZtcGVnX21pY3JvcGhvbmVfbGl2ZSglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzYW1wbGluZ19yYXRlJTNEc2FtcGxpbmdfcmF0ZSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNodW5rX2xlbmd0aF9zJTNEY2h1bmtfbGVuZ3RoX3MlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBzdHJlYW1fY2h1bmtfcyUzRHN0cmVhbV9jaHVua19zJTJDJTBBJTIwJTIwJTIwJTIwKSUwQSUwQSUyMCUyMCUyMCUyMHByaW50KCUyMlN0YXJ0JTIwc3BlYWtpbmcuLi4lMjIpJTBBJTIwJTIwJTIwJTIwZm9yJTIwaXRlbSUyMGluJTIwdHJhbnNjcmliZXIobWljJTJDJTIwZ2VuZXJhdGVfa3dhcmdzJTNEJTdCJTIybWF4X25ld190b2tlbnMlMjIlM0ElMjAxMjglN0QpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwc3lzLnN0ZG91dC53cml0ZSglMjIlNUMwMzMlNUJLJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHByaW50KGl0ZW0lNUIlMjJ0ZXh0JTIyJTVEJTJDJTIwZW5kJTNEJTIyJTVDciUyMiklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBpZiUyMG5vdCUyMGl0ZW0lNUIlMjJwYXJ0aWFsJTIyJTVEJTVCMCU1RCUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGJyZWFrJTBBJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwaXRlbSU1QiUyMnRleHQlMjIlNUQ=",highlighted:`<span class="hljs-keyword">import</span> sys


<span class="hljs-keyword">def</span> <span class="hljs-title function_">transcribe</span>(<span class="hljs-params">chunk_length_s=<span class="hljs-number">5.0</span>, stream_chunk_s=<span class="hljs-number">1.0</span></span>):
    sampling_rate = transcriber.feature_extractor.sampling_rate

    mic = ffmpeg_microphone_live(
        sampling_rate=sampling_rate,
        chunk_length_s=chunk_length_s,
        stream_chunk_s=stream_chunk_s,
    )

    <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;Start speaking...&quot;</span>)
    <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> transcriber(mic, generate_kwargs={<span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">128</span>}):
        sys.stdout.write(<span class="hljs-string">&quot;\\033[K&quot;</span>)
        <span class="hljs-built_in">print</span>(item[<span class="hljs-string">&quot;text&quot;</span>], end=<span class="hljs-string">&quot;\\r&quot;</span>)
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> item[<span class="hljs-string">&quot;partial&quot;</span>][<span class="hljs-number">0</span>]:
            <span class="hljs-keyword">break</span>

    <span class="hljs-keyword">return</span> item[<span class="hljs-string">&quot;text&quot;</span>]`}}),ut=new v({props:{code:"dHJhbnNjcmliZSgp",highlighted:"transcribe()"}}),ft=new v({props:{code:"U3RhcnQlMjBzcGVha2luZy4uLiUwQSUyMEhleSUyQyUyMHRoaXMlMjBpcyUyMGElMjB0ZXN0JTIwd2l0aCUyMHRoZSUyMHdoaXNwZXIlMjBtb2RlbC4=",highlighted:`Start speaking...
 Hey, this is a test with the whisper model.`}}),mt=new N({}),kt=new v({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmRm9sZGVyJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEElMEElMEFkZWYlMjBxdWVyeSh0ZXh0JTJDJTIwbW9kZWxfaWQlM0QlMjJ0aWl1YWUlMkZmYWxjb24tN2ItaW5zdHJ1Y3QlMjIpJTNBJTBBJTIwJTIwJTIwJTIwYXBpX3VybCUyMCUzRCUyMGYlMjJodHRwcyUzQSUyRiUyRmFwaS1pbmZlcmVuY2UuaHVnZ2luZ2ZhY2UuY28lMkZtb2RlbHMlMkYlN0Jtb2RlbF9pZCU3RCUyMiUwQSUyMCUyMCUyMCUyMGhlYWRlcnMlMjAlM0QlMjAlN0IlMjJBdXRob3JpemF0aW9uJTIyJTNBJTIwZiUyMkJlYXJlciUyMCU3QkhmRm9sZGVyKCkuZ2V0X3Rva2VuKCklN0QlMjIlN0QlMEElMjAlMjAlMjAlMjBwYXlsb2FkJTIwJTNEJTIwJTdCJTIyaW5wdXRzJTIyJTNBJTIwdGV4dCU3RCUwQSUwQSUyMCUyMCUyMCUyMHByaW50KGYlMjJRdWVyeWluZy4uLiUzQSUyMCU3QnRleHQlN0QlMjIpJTBBJTIwJTIwJTIwJTIwcmVzcG9uc2UlMjAlM0QlMjByZXF1ZXN0cy5wb3N0KGFwaV91cmwlMkMlMjBoZWFkZXJzJTNEaGVhZGVycyUyQyUyMGpzb24lM0RwYXlsb2FkKSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJlc3BvbnNlLmpzb24oKSU1QjAlNUQlNUIlMjJnZW5lcmF0ZWRfdGV4dCUyMiU1RCU1Qmxlbih0ZXh0KSUyMCUyQiUyMDElMjAlM0ElNUQ=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfFolder
<span class="hljs-keyword">import</span> requests


<span class="hljs-keyword">def</span> <span class="hljs-title function_">query</span>(<span class="hljs-params">text, model_id=<span class="hljs-string">&quot;tiiuae/falcon-7b-instruct&quot;</span></span>):
    api_url = <span class="hljs-string">f&quot;https://api-inference.huggingface.co/models/<span class="hljs-subst">{model_id}</span>&quot;</span>
    headers = {<span class="hljs-string">&quot;Authorization&quot;</span>: <span class="hljs-string">f&quot;Bearer <span class="hljs-subst">{HfFolder().get_token()}</span>&quot;</span>}
    payload = {<span class="hljs-string">&quot;inputs&quot;</span>: text}

    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Querying...: <span class="hljs-subst">{text}</span>&quot;</span>)
    response = requests.post(api_url, headers=headers, json=payload)
    <span class="hljs-keyword">return</span> response.json()[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;generated_text&quot;</span>][<span class="hljs-built_in">len</span>(text) + <span class="hljs-number">1</span> :]`}}),Tt=new v({props:{code:"cXVlcnkoJTIyV2hhdCUyMGRvZXMlMjBIdWdnaW5nJTIwRmFjZSUyMGRvJTNGJTIyKQ==",highlighted:'query(<span class="hljs-string">&quot;What does Hugging Face do?&quot;</span>)'}}),Jt=new v({props:{code:"J0h1Z2dpbmclMjBGYWNlJTIwaXMlMjBhJTIwY29tcGFueSUyMHRoYXQlMjBwcm92aWRlcyUyMG5hdHVyYWwlMjBsYW5ndWFnZSUyMHByb2Nlc3NpbmclMjBhbmQlMjBtYWNoaW5lJTIwbGVhcm5pbmclMjB0b29scyUyMGZvciUyMGRldmVsb3BlcnMuJTIwVGhleSc=",highlighted:'<span class="hljs-comment">&#x27;Hugging Face is a company that provides natural language processing and machine learning tools for developers. They&#x27;</span>'}}),jt=new N({}),_t=new v({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFNwZWVjaFQ1UHJvY2Vzc29yJTJDJTIwU3BlZWNoVDVGb3JUZXh0VG9TcGVlY2glMkMlMjBTcGVlY2hUNUhpZmlHYW4lMEElMEFwcm9jZXNzb3IlMjAlM0QlMjBTcGVlY2hUNVByb2Nlc3Nvci5mcm9tX3ByZXRyYWluZWQoJTIybWljcm9zb2Z0JTJGc3BlZWNodDVfdHRzJTIyKSUwQSUwQW1vZGVsJTIwJTNEJTIwU3BlZWNoVDVGb3JUZXh0VG9TcGVlY2guZnJvbV9wcmV0cmFpbmVkKCUyMm1pY3Jvc29mdCUyRnNwZWVjaHQ1X3R0cyUyMikudG8oZGV2aWNlKSUwQXZvY29kZXIlMjAlM0QlMjBTcGVlY2hUNUhpZmlHYW4uZnJvbV9wcmV0cmFpbmVkKCUyMm1pY3Jvc29mdCUyRnNwZWVjaHQ1X2hpZmlnYW4lMjIpLnRvKGRldmljZSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan

processor = SpeechT5Processor.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_tts&quot;</span>)

model = SpeechT5ForTextToSpeech.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_tts&quot;</span>).to(device)
vocoder = SpeechT5HifiGan.from_pretrained(<span class="hljs-string">&quot;microsoft/speecht5_hifigan&quot;</span>).to(device)`}}),Et=new v({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZW1iZWRkaW5nc19kYXRhc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMk1hdHRoaWpzJTJGY211LWFyY3RpYy14dmVjdG9ycyUyMiUyQyUyMHNwbGl0JTNEJTIydmFsaWRhdGlvbiUyMiklMEFzcGVha2VyX2VtYmVkZGluZ3MlMjAlM0QlMjB0b3JjaC50ZW5zb3IoZW1iZWRkaW5nc19kYXRhc2V0JTVCNzMwNiU1RCU1QiUyMnh2ZWN0b3IlMjIlNUQpLnVuc3F1ZWV6ZSgwKQ==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

embeddings_dataset = load_dataset(<span class="hljs-string">&quot;Matthijs/cmu-arctic-xvectors&quot;</span>, split=<span class="hljs-string">&quot;validation&quot;</span>)
speaker_embeddings = torch.tensor(embeddings_dataset[<span class="hljs-number">7306</span>][<span class="hljs-string">&quot;xvector&quot;</span>]).unsqueeze(<span class="hljs-number">0</span>)`}}),It=new v({props:{code:"ZGVmJTIwc3ludGhlc2lzZSh0ZXh0KSUzQSUwQSUyMCUyMCUyMCUyMGlucHV0cyUyMCUzRCUyMHByb2Nlc3Nvcih0ZXh0JTNEdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpJTBBJTIwJTIwJTIwJTIwc3BlZWNoJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGVfc3BlZWNoKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlucHV0cyU1QiUyMmlucHV0X2lkcyUyMiU1RC50byhkZXZpY2UpJTJDJTIwc3BlYWtlcl9lbWJlZGRpbmdzLnRvKGRldmljZSklMkMlMjB2b2NvZGVyJTNEdm9jb2RlciUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjByZXR1cm4lMjBzcGVlY2guY3B1KCk=",highlighted:`<span class="hljs-keyword">def</span> <span class="hljs-title function_">synthesise</span>(<span class="hljs-params">text</span>):
    inputs = processor(text=text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
    speech = model.generate_speech(
        inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].to(device), speaker_embeddings.to(device), vocoder=vocoder
    )
    <span class="hljs-keyword">return</span> speech.cpu()`}}),$t=new v({props:{code:"ZnJvbSUyMElQeXRob24uZGlzcGxheSUyMGltcG9ydCUyMEF1ZGlvJTBBJTBBYXVkaW8lMjAlM0QlMjBzeW50aGVzaXNlKCUwQSUyMCUyMCUyMCUyMCUyMkh1Z2dpbmclMjBGYWNlJTIwaXMlMjBhJTIwY29tcGFueSUyMHRoYXQlMjBwcm92aWRlcyUyMG5hdHVyYWwlMjBsYW5ndWFnZSUyMHByb2Nlc3NpbmclMjBhbmQlMjBtYWNoaW5lJTIwbGVhcm5pbmclMjB0b29scyUyMGZvciUyMGRldmVsb3BlcnMuJTIyJTBBKSUwQSUwQUF1ZGlvKGF1ZGlvJTJDJTIwcmF0ZSUzRDE2MDAwKQ==",highlighted:`<span class="hljs-keyword">from</span> IPython.display <span class="hljs-keyword">import</span> Audio

audio = synthesise(
    <span class="hljs-string">&quot;Hugging Face is a company that provides natural language processing and machine learning tools for developers.&quot;</span>
)

Audio(audio, rate=<span class="hljs-number">16000</span>)`}}),Wt=new N({}),At=new v({props:{code:"bGF1bmNoX2ZuKCklMEF0cmFuc2NyaXB0aW9uJTIwJTNEJTIwdHJhbnNjcmliZSgpJTBBcmVzcG9uc2UlMjAlM0QlMjBxdWVyeSh0cmFuc2NyaXB0aW9uKSUwQWF1ZGlvJTIwJTNEJTIwc3ludGhlc2lzZShyZXNwb25zZSklMEElMEFBdWRpbyhhdWRpbyUyQyUyMHJhdGUlM0QxNjAwMCUyQyUyMGF1dG9wbGF5JTNEVHJ1ZSk=",highlighted:`launch_fn()
transcription = transcribe()
response = query(transcription)
audio = synthesise(response)

Audio(audio, rate=<span class="hljs-number">16000</span>, autoplay=<span class="hljs-literal">True</span>)`}}),Ct=new N({}),Bt=new v({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEhmQWdlbnQlMEElMEFhZ2VudCUyMCUzRCUyMEhmQWdlbnQoJTBBJTIwJTIwJTIwJTIwdXJsX2VuZHBvaW50JTNEJTIyaHR0cHMlM0ElMkYlMkZhcGktaW5mZXJlbmNlLmh1Z2dpbmdmYWNlLmNvJTJGbW9kZWxzJTJGYmlnY29kZSUyRnN0YXJjb2RlciUyMiUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> HfAgent

agent = HfAgent(
    url_endpoint=<span class="hljs-string">&quot;https://api-inference.huggingface.co/models/bigcode/starcoder&quot;</span>
)`}}),xt=new v({props:{code:"YWdlbnQucnVuKCUyMkdlbmVyYXRlJTIwYW4lMjBpbWFnZSUyMG9mJTIwYSUyMGNhdCUyMik=",highlighted:'agent.run(<span class="hljs-string">&quot;Generate an image of a cat&quot;</span>)'}}),$e=new md({props:{$$slots:{default:[kd]},$$scope:{ctx:Qt}}}),Vt=new v({props:{code:"bGF1bmNoX2ZuKCklMEF0cmFuc2NyaXB0aW9uJTIwJTNEJTIwdHJhbnNjcmliZSgpJTBBYWdlbnQucnVuKHRyYW5zY3JpcHRpb24p",highlighted:`launch_fn()
transcription = transcribe()
agent.run(transcription)`}}),{c(){k=n("meta"),T=c(),j=n("h1"),E=n("a"),I=n("span"),u(S.$$.fragment),se=c(),g=n("span"),$=o("Creating a voice assistant"),Be=c(),ae=n("p"),nn=o(`In this section, we\u2019ll piece together three models that we\u2019ve already had hands-on experience with to build an end-to-end
voice assistant called `),Ts=n("strong"),rn=o("Marvin"),hn=o(` \u{1F916}. Like Amazon\u2019s Alexa or Apple\u2019s Siri, Marvin is a virtual voice assistant who
responds to a particular \u2018wake word\u2019, then listens out for a spoken query, and finally responds with a spoken answer.`),qa=c(),Rt=n("p"),cn=o("We can break down the voice assistant pipeline into four stages, each of which requires a standalone model:"),La=c(),xe=n("div"),Js=n("img"),Qa=c(),R=n("h3"),oe=n("a"),js=n("span"),u(Ge.$$.fragment),pn=c(),Us=n("span"),dn=o("1. Wake word detection"),Ra=c(),Xt=n("p"),un=o(`Voice assistants are constantly listening to the audio inputs coming through your device\u2019s microphone, however they only
boot into action when a particular \u2018wake word\u2019 or \u2018trigger word\u2019 is spoken.`),Xa=c(),Ft=n("p"),fn=o(`The wake word detection task is handled by a small on-device audio classification model, which is much smaller and lighter
than the speech recognition model, often only several millions of parameters compared to several hundred millions for
speech recognition. Thus, it can be run continuously on your device without draining your battery. Only when the wake
word is detected is the larger speech recognition model launched, and afterwards it is shut down again.`),Fa=c(),X=n("h3"),le=n("a"),_s=n("span"),u(He.$$.fragment),mn=c(),Es=n("span"),wn=o("2. Speech transcription"),za=c(),zt=n("p"),yn=o(`The next stage in the pipeline is transcribing the spoken query to text. In practice, transferring audio files from your
local device to the Cloud is slow due to the large nature of audio files, so it\u2019s more efficient to transcribe them
directly using an automatic speech recognition (ASR) model on-device rather than using a model in the Cloud. The on-device
model might be smaller and thus less accurate than one hosted in the Cloud, but the faster inference speed makes it
worthwhile since we can run speech recognition in near real-time, our spoken audio utterance being transcribed as we say it.`),Pa=c(),Pt=n("p"),bn=o("We\u2019re very familiar with the speech recognition process now, so this should be a piece of cake!"),Da=c(),F=n("h3"),ne=n("a"),Is=n("span"),u(Ve.$$.fragment),gn=c(),$s=n("span"),Mn=o("3. Language model query"),Oa=c(),ie=n("p"),vn=o(`Now that we know what the user asked, we need to generate a response! The best candidate models for this task are
`),Ws=n("em"),kn=o("large language models (LLMs)"),Tn=o(`, since they are effectively able to understand the semantics of the text query and
generate a suitable response.`),Ka=c(),Dt=n("p"),Jn=o(`Since our text query is small (just a few text tokens), and language models large (many billions of parameters), the most
efficient way of running LLM inference is to send our text query from our device to an LLM running in the Cloud,
generate a text response, and return the response back to the device.`),eo=c(),z=n("h3"),re=n("a"),As=n("span"),u(Ye.$$.fragment),jn=c(),Cs=n("span"),Un=o("4. Synthesise speech"),to=c(),Ot=n("p"),_n=o(`Finally, we\u2019ll use a text-to-speech (TTS) model to synthesise the text response as spoken speech. This is done
on-device, but you could feasibly run a TTS model in the Cloud, generating the audio output and transferring it back to
the device.`),so=c(),Kt=n("p"),En=o("Again, we\u2019ve done this several times now, so the process will be very familiar!"),ao=c(),P=n("h2"),he=n("a"),Zs=n("span"),u(qe.$$.fragment),In=c(),Ss=n("span"),$n=o("Wake word detection"),oo=c(),M=n("p"),Wn=o(`The first stage in the voice assistant pipeline is detecting whether the wake word was spoken, and we need to find ourselves
an appropriate pre-trained model for this task! You\u2019ll remember from the section on `),es=n("a"),An=o("pre-trained models for audio classification"),Cn=o(`
that `),Le=n("a"),Zn=o("Speech Commands"),Sn=o(` is a dataset of spoken words designed to
evaluate audio classification models on 15+ simple command words like `),Ns=n("code"),Nn=o('"up"'),Bn=o(", "),Bs=n("code"),xn=o('"down"'),Gn=o(", "),xs=n("code"),Hn=o('"yes"'),Vn=o(" and "),Gs=n("code"),Yn=o('"no"'),qn=o(`, as well as a
`),Hs=n("code"),Ln=o('"silence"'),Qn=o(` label to classify no speech. Take a minute to listen through the samples on the datasets viewer on
the Hub and re-acquaint yourself with the Speech Commands dataset: `),Qe=n("a"),Rn=o("datasets viewer"),Xn=o("."),lo=c(),ts=n("p"),Fn=o(`We can take an audio classification model pre-trained on the Speech Commands dataset and pick one of these simple command
words to be our chosen wake word. Out of the 15+ possible command words, if the model predicts our chosen wake word with the
highest probability, we can be fairly certain that the wake word has been said.`),no=c(),Re=n("p"),zn=o("Let\u2019s head to the Hugging Face Hub and click on the \u201CModels\u201D tab: "),Xe=n("a"),Pn=o("https://huggingface.co/models"),io=c(),ss=n("p"),Dn=o("This is going to bring up all the models on the Hugging Face Hub, sorted by downloads in the past 30 days:"),ro=c(),Fe=n("div"),Vs=n("img"),ho=c(),as=n("p"),On=o(`You\u2019ll notice on the left-hand side that we have a selection of tabs that we can select to filter models by task, library,
dataset, etc. Scroll down and select the task \u201CAudio Classification\u201D from the list of audio tasks:`),co=c(),ze=n("div"),Ys=n("img"),po=c(),ce=n("p"),Kn=o(`We\u2019re now presented with the sub-set of 500+ audio classification models on the Hub. To further refine this selection, we
can filter models by dataset. Click on the tab \u201CDatasets\u201D, and in the search box type \u201Cspeech_commands\u201D. As you begin typing,
you\u2019ll see the selection for `),qs=n("code"),ei=o("speech_commands"),ti=o(` appear underneath the search tab. You can click this button to filter all
audio classification models to those fine-tuned on the Speech Commands dataset:`),uo=c(),Pe=n("div"),Ls=n("img"),fo=c(),pe=n("p"),si=o(`Great! We see that we have six pre-trained models available to us for this specific dataset and task (although there may
be new models added if you\u2019re reading at a later date!). You\u2019ll recognise the first of these models as the `),De=n("a"),ai=o("Audio Spectrogram Transformer checkpoint"),oi=o(`
that we used in Unit 4 example. We\u2019ll use this checkpoint again for our wake word detection task.`),mo=c(),de=n("p"),li=o("Let\u2019s go ahead and load the checkpoint using the "),Qs=n("code"),ni=o("pipeline"),ii=o(" class:"),wo=c(),u(Oe.$$.fragment),yo=c(),ue=n("p"),ri=o("We can check what labels the model was trained on by checking the "),Rs=n("code"),hi=o("id2label"),ci=o(" attribute in the model config:"),bo=c(),u(Ke.$$.fragment),go=c(),U=n("p"),pi=o(`Alright! We see that the model was trained on 35 class labels, including some simple command words that we described above,
as well as some particular objects like `),Xs=n("code"),di=o('"bed"'),ui=o(", "),Fs=n("code"),fi=o('"house"'),mi=o(" and "),zs=n("code"),wi=o('"cat"'),yi=o(`. We see that there is one name in these class labels:
id 27 corresponds to the label `),Ps=n("strong"),bi=o("\u201Cmarvin\u201D"),gi=o(":"),Mo=c(),u(et.$$.fragment),vo=c(),u(tt.$$.fragment),ko=c(),fe=n("p"),Mi=o(`Perfect! We can use this name as our wake word for our voice assistant, similar to how \u201CAlexa\u201D is used for Amazon\u2019s Alexa,
or \u201CHey Siri\u201D is used for Apple\u2019s Siri. Of all the possible labels, if the model predicts `),Ds=n("code"),vi=o('"marvin"'),ki=o(` with the highest class
probability, we can be fairly sure that our chosen wake word has been said.`),To=c(),me=n("p"),Ti=o(`Now we need to define a function that is constantly listening to our device\u2019s microphone input, and continuously
passes the audio to the classification model for inference. To do this, we\u2019ll use a handy helper function that comes
with \u{1F917} Transformers called `),st=n("a"),Os=n("code"),Ji=o("ffmpeg_microphone_live"),ji=o("."),Jo=c(),_=n("p"),Ui=o("This function forwards small chunks of audio of specified length "),Ks=n("code"),_i=o("chunk_length_s"),Ei=o(` to the model to be classified. To ensure that
we get smooth boundaries across chunks of audio, we run a sliding window across our audio with stride `),ea=n("code"),Ii=o("chunk_length_s / 6"),$i=o(`.
So that we don\u2019t have to wait for the entire first chunk to be recorded before we start inferring, we also define a minimal
temporary audio input length `),ta=n("code"),Wi=o("stream_chunk_s"),Ai=o(" that is forwarded to the model before "),sa=n("code"),Ci=o("chunk_length_s"),Zi=o(" time is reached."),jo=c(),W=n("p"),Si=o("The function "),aa=n("code"),Ni=o("ffmpeg_microphone_live"),Bi=o(" returns a "),oa=n("em"),xi=o("generator"),Gi=o(` object, yielding a sequence of audio chunks that can each
be passed to the classification model to make a prediction. We can pass this generator directly to the `),la=n("code"),Hi=o("pipeline"),Vi=o(`,
which in turn returns a sequence of output predictions, one for each chunk of audio input. We can inspect the class
label probabilities for each audio chunk, and stop our wake word detection loop when we detect that the wake word
has been spoken.`),Uo=c(),B=n("p"),Yi=o(`We\u2019ll use a very simple criteria for classifying whether our wake word was spoken: if the class label with the highest
probability was our wake word, and this probability exceeds a threshold `),na=n("code"),qi=o("prob_threshold"),Li=o(`, we declare that the wake word
as having been spoken. Using a probability threshold to gate our classifier this way ensures that the wake word is not
erroneously predicted if the audio input is noise, which is typically when the model is very uncertain and all the class
label probabilities low. You might want to tune this probability threshold, or explore more sophisticated means for
the wake word decision through an `),at=n("a"),ia=n("em"),Qi=o("entropy"),Ri=o(" (or uncertainty) based metric."),_o=c(),u(ot.$$.fragment),Eo=c(),A=n("p"),Xi=o("Let\u2019s give this function a try to see how it works! We\u2019ll set the flag "),ra=n("code"),Fi=o("debug=True"),zi=o(` to print out the prediction for each
chunk of audio. Let the model run for a few seconds to see the kinds of predictions that it makes when there is no speech
input, then clearly say the wake word `),ha=n("code"),Pi=o('"marvin"'),Di=o(" and watch the class label prediction for "),ca=n("code"),Oi=o('"marvin"'),Ki=o(" spike to near 1:"),Io=c(),u(lt.$$.fragment),$o=c(),u(nt.$$.fragment),Wo=c(),we=n("p"),er=o(`Awesome! As we expect, the model generates garbage predictions for the first few seconds. There is no speech input, so the
model makes close to random predictions, but with very low probability. As soon as we say the wake word, the model predicts
`),pa=n("code"),tr=o('"marvin"'),sr=o(` with probability close to 1 and terminates the loop, signalling that the wake word has been detected and that the
ASR system should be activated!`),Ao=c(),D=n("h2"),ye=n("a"),da=n("span"),u(it.$$.fragment),ar=c(),ua=n("span"),or=o("Speech transcription"),Co=c(),x=n("p"),lr=o("Once again, we\u2019ll use the Whisper model for our speech transcription system. Specifically, we\u2019ll load the "),rt=n("a"),nr=o("Whisper Base English"),ir=o(`
checkpoint, since it\u2019s small enough to give good inference speed with reasonable transcription accuracy. We\u2019ll use a trick
to get near real-time transcription by being clever with how we forward our audio inputs to the model. As before, feel
free to use any speech recognition checkpoint on `),ht=n("a"),rr=o("the Hub"),hr=o(`,
including Wav2Vec2, MMS ASR or other Whisper checkpoints:`),Zo=c(),u(ct.$$.fragment),So=c(),u(be.$$.fragment),No=c(),G=n("p"),cr=o("We can now define a function to record our microphone input and transcribe the corresponding text. With the "),fa=n("code"),pr=o("ffmpeg_microphone_live"),dr=o(`
helper function, we can control how \u2018real-time\u2019 our speech recognition model is. Using a smaller `),ma=n("code"),ur=o("stream_chunk_s"),fr=o(` lends
itself to more real-time speech recognition, since we divide our input audio into smaller chunks and transcribe them on
the fly. However, this comes at the expense of poorer accuracy, since there\u2019s less context for the model to infer from.`),Bo=c(),C=n("p"),mr=o("As we\u2019re transcribing the speech, we also need to have an idea of when the user "),wa=n("strong"),wr=o("stops"),yr=o(` speaking, so that we can terminate
the recording. For simplicity, we\u2019ll terminate our microphone recording after the first `),ya=n("code"),br=o("chunk_length_s"),gr=o(` (which is set to
5 seconds by default), but you can experiment with using a `),pt=n("a"),Mr=o("voice activity detection (VAD)"),vr=o(`
model to predict when the user has stopped speaking.`),xo=c(),u(dt.$$.fragment),Go=c(),os=n("p"),kr=o(`Let\u2019s give this a go and see how we get on! Once the microphone is live, start speaking and watch your transcription
appear in semi real-time:`),Ho=c(),u(ut.$$.fragment),Vo=c(),u(ft.$$.fragment),Yo=c(),Z=n("p"),Tr=o("Nice! You can adjust the maximum audio length "),ba=n("code"),Jr=o("chunk_length_s"),jr=o(` based on how fast or slow you speak (increase it if you
felt like you didn\u2019t have enough time to speak, decrease it if you were left waiting at the end), and the
`),ga=n("code"),Ur=o("stream_chunk_s"),_r=o(" for the real-time factor. Just pass these as arguments to the "),Ma=n("code"),Er=o("transcribe"),Ir=o(" function."),qo=c(),O=n("h2"),ge=n("a"),va=n("span"),u(mt.$$.fragment),$r=c(),ka=n("span"),Wr=o("Language model query"),Lo=c(),Me=n("p"),Ar=o(`Now that we have our spoken query transcribed, we want to generate a meaningful response. To do this, we\u2019ll use an LLM
hosted on the Cloud. Specifically, we\u2019ll pick an LLM on the Hugging Face Hub and use the `),wt=n("a"),Cr=o("Inference API"),Zr=o(`
to easily query the model.`),Qo=c(),ve=n("p"),Sr=o("First, let\u2019s head over to the Hugging Face Hub. To find our LLM, we\u2019ll use the "),yt=n("a"),Nr=o("\u{1F917} Open LLM Leaderboard"),Br=o(`,
a Space that ranks LLM models by performance over four generation tasks. We\u2019ll search by \u201Cinstruct\u201D to filter out models
that have been instruction fine-tuned, since these should work better for our querying task:`),Ro=c(),bt=n("div"),Ta=n("img"),Xo=c(),H=n("p"),xr=o("We\u2019ll use the "),gt=n("a"),Gr=o("tiiuae/falcon-7b-instruct"),Hr=o(" checkpoint by "),Mt=n("a"),Vr=o("TII"),Yr=o(`,
a 7B parameter decoder-only LM fine-tuned on a mixture of chat and instruction datasets. You can use any LLM on the Hugging
Face Hub that has the \u201CHosted inference API\u201D enabled, just look out for the widget on the right-side of the model card:`),Fo=c(),vt=n("div"),Ja=n("img"),zo=c(),ke=n("p"),qr=o(`The Inference API allows us to send a HTTP request from our local machine to the LLM hosted on the Hub, and returns the
response as a `),ja=n("code"),Lr=o("json"),Qr=o(` file. All we need to provide is our Hugging Face Hub token (which we retrieve directly from our Hugging Face
Hub folder) and the model id of the LLM we wish to query:`),Po=c(),u(kt.$$.fragment),Do=c(),ls=n("p"),Rr=o("Let\u2019s give it a try with a test input!"),Oo=c(),u(Tt.$$.fragment),Ko=c(),u(Jt.$$.fragment),el=c(),ns=n("p"),Xr=o(`You\u2019ll notice just how fast inference is using the Inference API - we only have to send a small number of text tokens
from our local machine to the hosted model, so the communication cost is very low. The LLM is hosted on GPU accelerators,
so inference runs very quickly. Finally, the generated response is transferred back from the model to our local machine,
again with low communication overhead.`),tl=c(),K=n("h2"),Te=n("a"),Ua=n("span"),u(jt.$$.fragment),Fr=c(),_a=n("span"),zr=o("Synthesise speech"),sl=c(),Je=n("p"),Pr=o("And now we\u2019re ready to get the final spoken output! Once again, we\u2019ll use the Microsoft "),Ut=n("a"),Dr=o("SpeechT5 TTS"),Or=o(`
model for English TTS, but you can use any TTS model of your choice. Let\u2019s go ahead and load the processor and model:`),al=c(),u(_t.$$.fragment),ol=c(),is=n("p"),Kr=o("And also the speaker embeddings:"),ll=c(),u(Et.$$.fragment),nl=c(),V=n("p"),eh=o("We\u2019ll re-use the "),Ea=n("code"),th=o("synthesise"),sh=o(" function that we defined in the previous chapter on "),rs=n("a"),ah=o("Speech-to-speech translation"),oh=o(":"),il=c(),u(It.$$.fragment),rl=c(),hs=n("p"),lh=o("Let\u2019s quickly verify this works as expected:"),hl=c(),u($t.$$.fragment),cl=c(),cs=n("p"),nh=o("Nice job \u{1F44D}"),pl=c(),ee=n("h2"),je=n("a"),Ia=n("span"),u(Wt.$$.fragment),ih=c(),$a=n("span"),rh=o("Marvin \u{1F916}"),dl=c(),Ue=n("p"),hh=o(`Now that we\u2019ve defined a function for each of the four stages of the voice assistant pipeline, all that\u2019s left to do is
piece them together to get our end-to-end voice assistant. We\u2019ll simply concatenate the four stages, starting with
wake word detection (`),Wa=n("code"),ch=o("launch_fn"),ph=o("), speech transcription, querying the LLM, and finally speech synthesis."),ul=c(),u(At.$$.fragment),fl=c(),ps=n("p"),dh=o("Try it out with a few prompts! Here are some examples to get you started:"),ml=c(),Y=n("ul"),Aa=n("li"),Ca=n("em"),uh=o("What is the hottest country in the world?"),fh=c(),Za=n("li"),Sa=n("em"),mh=o("How do Transformer models work?"),wh=c(),Na=n("li"),Ba=n("em"),yh=o("Do you know Spanish?"),wl=c(),ds=n("p"),bh=o(`And with that, we have our end-to-end voice assistant complete, made using the \u{1F917} audio tools you\u2019ve learnt throughout
this course, with a sprinkling of LLM magic at the end. There are several extensions that we could make to improve the
voice assistant. Firstly, the audio classification model classifies 35 different labels. We could use a smaller, more
lightweight binary classification model that only predicts whether the wake word was spoken or not. Secondly, we pre-load
all the models ahead and keep them running on our device. If we wanted to save power, we would only load each model at
the time it was required, and subsequently un-load them afterwards. Thirdly, we\u2019re missing a voice activity detection model
in our transcription function, transcribing for a fixed amount of time, which in some cases is too long, and in others too
short.`),yl=c(),te=n("h2"),_e=n("a"),xa=n("span"),u(Ct.$$.fragment),gh=c(),Ga=n("span"),Mh=o("Generalise to anything \u{1FA84}"),bl=c(),us=n("p"),vh=o(`So far, we\u2019ve seen how we can generate speech outputs with our voice assistant Marvin. To finish, we\u2019ll demonstrate how
we can generalise these speech outputs to text, audio and image.`),gl=c(),Ee=n("p"),kh=o("We\u2019ll use "),Zt=n("a"),Th=o("Transformers Agents"),Jh=o(` to build our assistant.
Transformers Agents provides a natural language API on top of the \u{1F917} Transformers and Diffusers libraries, interpreting
a natural language input using an LLM with carefully crafted prompts, and using a set of curated tools to provide
multimodal outputs.`),Ml=c(),q=n("p"),jh=o("Let\u2019s go ahead and instantiate an agent. There are "),St=n("a"),Uh=o("three LLMs available"),_h=o(`
for Transformers Agents, two of which are open-source and free on the Hugging Face Hub. The third is a model from OpenAI
that requires an OpenAI API key. We\u2019ll use the free `),Nt=n("a"),Eh=o("Bigcode Starcoder"),Ih=o(` model
in this example, but you can also try either of the other LLMs available:`),vl=c(),u(Bt.$$.fragment),kl=c(),Ie=n("p"),$h=o("To use the agent, we simply have to call "),Ha=n("code"),Wh=o("agent.run"),Ah=o(` with our text prompt. As an example, we\u2019ll get it to generate an
image of a cat \u{1F408} (that hopefully looks a bit better than this emoji):`),Tl=c(),u(xt.$$.fragment),Jl=c(),Gt=n("div"),Va=n("img"),jl=c(),u($e.$$.fragment),Ul=c(),We=n("p"),Ch=o("Easy as that! The Agent interpreted our prompt, and used "),Ht=n("a"),Zh=o("Stable Diffusion"),Sh=o(`
under the hood to generate the image, without us having to worry about loading the model, writing the function or executing
the code.`),_l=c(),fs=n("p"),Nh=o(`We can now replace our LLM query function and text synthesis step with our Transformers Agent in our voice assistant,
since the Agent is going to take care of both of these steps for us:`),El=c(),u(Vt.$$.fragment),Il=c(),ms=n("p"),Bh=o(`Try speaking the same prompt \u201CGenerate an image of a cat\u201D and see how the system gets on. If you ask the Agent a simple
question / answer query, the Agent will respond with a text answer. You can encourage it to generate multimodal outputs
by asking it to return an image or speech. For example, you can ask it to: \u201CGenerate an image of a cat, caption it, and
speak the caption\u201D.`),$l=c(),Ae=n("p"),xh=o(`While the Agent is more flexible than our first iteration Marvin \u{1F916} assistant, generalising the voice assistant task in this way
may lead to inferior performance on standard voice assistant queries. To recover performance, you can try using a
more performant LLM checkpoint, such as the one from OpenAI, or define a set of `),Yt=n("a"),Gh=o("custom tools"),Hh=o(`
that are specific to the voice assistant task.`),this.h()},l(e){const a=gd('[data-svelte="svelte-1phssyn"]',document.head);k=i(a,"META",{name:!0,content:!0}),a.forEach(t),T=p(e),j=i(e,"H1",{class:!0});var qt=h(j);E=i(qt,"A",{id:!0,class:!0,href:!0});var Ya=h(E);I=i(Ya,"SPAN",{});var zh=h(I);f(S.$$.fragment,zh),zh.forEach(t),Ya.forEach(t),se=p(qt),g=i(qt,"SPAN",{});var Ph=h(g);$=l(Ph,"Creating a voice assistant"),Ph.forEach(t),qt.forEach(t),Be=p(e),ae=i(e,"P",{});var Al=h(ae);nn=l(Al,`In this section, we\u2019ll piece together three models that we\u2019ve already had hands-on experience with to build an end-to-end
voice assistant called `),Ts=i(Al,"STRONG",{});var Dh=h(Ts);rn=l(Dh,"Marvin"),Dh.forEach(t),hn=l(Al,` \u{1F916}. Like Amazon\u2019s Alexa or Apple\u2019s Siri, Marvin is a virtual voice assistant who
responds to a particular \u2018wake word\u2019, then listens out for a spoken query, and finally responds with a spoken answer.`),Al.forEach(t),qa=p(e),Rt=i(e,"P",{});var Oh=h(Rt);cn=l(Oh,"We can break down the voice assistant pipeline into four stages, each of which requires a standalone model:"),Oh.forEach(t),La=p(e),xe=i(e,"DIV",{class:!0});var Kh=h(xe);Js=i(Kh,"IMG",{src:!0}),Kh.forEach(t),Qa=p(e),R=i(e,"H3",{class:!0});var Cl=h(R);oe=i(Cl,"A",{id:!0,class:!0,href:!0});var ec=h(oe);js=i(ec,"SPAN",{});var tc=h(js);f(Ge.$$.fragment,tc),tc.forEach(t),ec.forEach(t),pn=p(Cl),Us=i(Cl,"SPAN",{});var sc=h(Us);dn=l(sc,"1. Wake word detection"),sc.forEach(t),Cl.forEach(t),Ra=p(e),Xt=i(e,"P",{});var ac=h(Xt);un=l(ac,`Voice assistants are constantly listening to the audio inputs coming through your device\u2019s microphone, however they only
boot into action when a particular \u2018wake word\u2019 or \u2018trigger word\u2019 is spoken.`),ac.forEach(t),Xa=p(e),Ft=i(e,"P",{});var oc=h(Ft);fn=l(oc,`The wake word detection task is handled by a small on-device audio classification model, which is much smaller and lighter
than the speech recognition model, often only several millions of parameters compared to several hundred millions for
speech recognition. Thus, it can be run continuously on your device without draining your battery. Only when the wake
word is detected is the larger speech recognition model launched, and afterwards it is shut down again.`),oc.forEach(t),Fa=p(e),X=i(e,"H3",{class:!0});var Zl=h(X);le=i(Zl,"A",{id:!0,class:!0,href:!0});var lc=h(le);_s=i(lc,"SPAN",{});var nc=h(_s);f(He.$$.fragment,nc),nc.forEach(t),lc.forEach(t),mn=p(Zl),Es=i(Zl,"SPAN",{});var ic=h(Es);wn=l(ic,"2. Speech transcription"),ic.forEach(t),Zl.forEach(t),za=p(e),zt=i(e,"P",{});var rc=h(zt);yn=l(rc,`The next stage in the pipeline is transcribing the spoken query to text. In practice, transferring audio files from your
local device to the Cloud is slow due to the large nature of audio files, so it\u2019s more efficient to transcribe them
directly using an automatic speech recognition (ASR) model on-device rather than using a model in the Cloud. The on-device
model might be smaller and thus less accurate than one hosted in the Cloud, but the faster inference speed makes it
worthwhile since we can run speech recognition in near real-time, our spoken audio utterance being transcribed as we say it.`),rc.forEach(t),Pa=p(e),Pt=i(e,"P",{});var hc=h(Pt);bn=l(hc,"We\u2019re very familiar with the speech recognition process now, so this should be a piece of cake!"),hc.forEach(t),Da=p(e),F=i(e,"H3",{class:!0});var Sl=h(F);ne=i(Sl,"A",{id:!0,class:!0,href:!0});var cc=h(ne);Is=i(cc,"SPAN",{});var pc=h(Is);f(Ve.$$.fragment,pc),pc.forEach(t),cc.forEach(t),gn=p(Sl),$s=i(Sl,"SPAN",{});var dc=h($s);Mn=l(dc,"3. Language model query"),dc.forEach(t),Sl.forEach(t),Oa=p(e),ie=i(e,"P",{});var Nl=h(ie);vn=l(Nl,`Now that we know what the user asked, we need to generate a response! The best candidate models for this task are
`),Ws=i(Nl,"EM",{});var uc=h(Ws);kn=l(uc,"large language models (LLMs)"),uc.forEach(t),Tn=l(Nl,`, since they are effectively able to understand the semantics of the text query and
generate a suitable response.`),Nl.forEach(t),Ka=p(e),Dt=i(e,"P",{});var fc=h(Dt);Jn=l(fc,`Since our text query is small (just a few text tokens), and language models large (many billions of parameters), the most
efficient way of running LLM inference is to send our text query from our device to an LLM running in the Cloud,
generate a text response, and return the response back to the device.`),fc.forEach(t),eo=p(e),z=i(e,"H3",{class:!0});var Bl=h(z);re=i(Bl,"A",{id:!0,class:!0,href:!0});var mc=h(re);As=i(mc,"SPAN",{});var wc=h(As);f(Ye.$$.fragment,wc),wc.forEach(t),mc.forEach(t),jn=p(Bl),Cs=i(Bl,"SPAN",{});var yc=h(Cs);Un=l(yc,"4. Synthesise speech"),yc.forEach(t),Bl.forEach(t),to=p(e),Ot=i(e,"P",{});var bc=h(Ot);_n=l(bc,`Finally, we\u2019ll use a text-to-speech (TTS) model to synthesise the text response as spoken speech. This is done
on-device, but you could feasibly run a TTS model in the Cloud, generating the audio output and transferring it back to
the device.`),bc.forEach(t),so=p(e),Kt=i(e,"P",{});var gc=h(Kt);En=l(gc,"Again, we\u2019ve done this several times now, so the process will be very familiar!"),gc.forEach(t),ao=p(e),P=i(e,"H2",{class:!0});var xl=h(P);he=i(xl,"A",{id:!0,class:!0,href:!0});var Mc=h(he);Zs=i(Mc,"SPAN",{});var vc=h(Zs);f(qe.$$.fragment,vc),vc.forEach(t),Mc.forEach(t),In=p(xl),Ss=i(xl,"SPAN",{});var kc=h(Ss);$n=l(kc,"Wake word detection"),kc.forEach(t),xl.forEach(t),oo=p(e),M=i(e,"P",{});var J=h(M);Wn=l(J,`The first stage in the voice assistant pipeline is detecting whether the wake word was spoken, and we need to find ourselves
an appropriate pre-trained model for this task! You\u2019ll remember from the section on `),es=i(J,"A",{href:!0});var Tc=h(es);An=l(Tc,"pre-trained models for audio classification"),Tc.forEach(t),Cn=l(J,`
that `),Le=i(J,"A",{href:!0,rel:!0});var Jc=h(Le);Zn=l(Jc,"Speech Commands"),Jc.forEach(t),Sn=l(J,` is a dataset of spoken words designed to
evaluate audio classification models on 15+ simple command words like `),Ns=i(J,"CODE",{});var jc=h(Ns);Nn=l(jc,'"up"'),jc.forEach(t),Bn=l(J,", "),Bs=i(J,"CODE",{});var Uc=h(Bs);xn=l(Uc,'"down"'),Uc.forEach(t),Gn=l(J,", "),xs=i(J,"CODE",{});var _c=h(xs);Hn=l(_c,'"yes"'),_c.forEach(t),Vn=l(J," and "),Gs=i(J,"CODE",{});var Ec=h(Gs);Yn=l(Ec,'"no"'),Ec.forEach(t),qn=l(J,`, as well as a
`),Hs=i(J,"CODE",{});var Ic=h(Hs);Ln=l(Ic,'"silence"'),Ic.forEach(t),Qn=l(J,` label to classify no speech. Take a minute to listen through the samples on the datasets viewer on
the Hub and re-acquaint yourself with the Speech Commands dataset: `),Qe=i(J,"A",{href:!0,rel:!0});var $c=h(Qe);Rn=l($c,"datasets viewer"),$c.forEach(t),Xn=l(J,"."),J.forEach(t),lo=p(e),ts=i(e,"P",{});var Wc=h(ts);Fn=l(Wc,`We can take an audio classification model pre-trained on the Speech Commands dataset and pick one of these simple command
words to be our chosen wake word. Out of the 15+ possible command words, if the model predicts our chosen wake word with the
highest probability, we can be fairly certain that the wake word has been said.`),Wc.forEach(t),no=p(e),Re=i(e,"P",{});var Vh=h(Re);zn=l(Vh,"Let\u2019s head to the Hugging Face Hub and click on the \u201CModels\u201D tab: "),Xe=i(Vh,"A",{href:!0,rel:!0});var Ac=h(Xe);Pn=l(Ac,"https://huggingface.co/models"),Ac.forEach(t),Vh.forEach(t),io=p(e),ss=i(e,"P",{});var Cc=h(ss);Dn=l(Cc,"This is going to bring up all the models on the Hugging Face Hub, sorted by downloads in the past 30 days:"),Cc.forEach(t),ro=p(e),Fe=i(e,"DIV",{class:!0});var Zc=h(Fe);Vs=i(Zc,"IMG",{src:!0}),Zc.forEach(t),ho=p(e),as=i(e,"P",{});var Sc=h(as);On=l(Sc,`You\u2019ll notice on the left-hand side that we have a selection of tabs that we can select to filter models by task, library,
dataset, etc. Scroll down and select the task \u201CAudio Classification\u201D from the list of audio tasks:`),Sc.forEach(t),co=p(e),ze=i(e,"DIV",{class:!0});var Nc=h(ze);Ys=i(Nc,"IMG",{src:!0}),Nc.forEach(t),po=p(e),ce=i(e,"P",{});var Gl=h(ce);Kn=l(Gl,`We\u2019re now presented with the sub-set of 500+ audio classification models on the Hub. To further refine this selection, we
can filter models by dataset. Click on the tab \u201CDatasets\u201D, and in the search box type \u201Cspeech_commands\u201D. As you begin typing,
you\u2019ll see the selection for `),qs=i(Gl,"CODE",{});var Bc=h(qs);ei=l(Bc,"speech_commands"),Bc.forEach(t),ti=l(Gl,` appear underneath the search tab. You can click this button to filter all
audio classification models to those fine-tuned on the Speech Commands dataset:`),Gl.forEach(t),uo=p(e),Pe=i(e,"DIV",{class:!0});var xc=h(Pe);Ls=i(xc,"IMG",{src:!0}),xc.forEach(t),fo=p(e),pe=i(e,"P",{});var Hl=h(pe);si=l(Hl,`Great! We see that we have six pre-trained models available to us for this specific dataset and task (although there may
be new models added if you\u2019re reading at a later date!). You\u2019ll recognise the first of these models as the `),De=i(Hl,"A",{href:!0,rel:!0});var Gc=h(De);ai=l(Gc,"Audio Spectrogram Transformer checkpoint"),Gc.forEach(t),oi=l(Hl,`
that we used in Unit 4 example. We\u2019ll use this checkpoint again for our wake word detection task.`),Hl.forEach(t),mo=p(e),de=i(e,"P",{});var Vl=h(de);li=l(Vl,"Let\u2019s go ahead and load the checkpoint using the "),Qs=i(Vl,"CODE",{});var Hc=h(Qs);ni=l(Hc,"pipeline"),Hc.forEach(t),ii=l(Vl," class:"),Vl.forEach(t),wo=p(e),f(Oe.$$.fragment,e),yo=p(e),ue=i(e,"P",{});var Yl=h(ue);ri=l(Yl,"We can check what labels the model was trained on by checking the "),Rs=i(Yl,"CODE",{});var Vc=h(Rs);hi=l(Vc,"id2label"),Vc.forEach(t),ci=l(Yl," attribute in the model config:"),Yl.forEach(t),bo=p(e),f(Ke.$$.fragment,e),go=p(e),U=i(e,"P",{});var L=h(U);pi=l(L,`Alright! We see that the model was trained on 35 class labels, including some simple command words that we described above,
as well as some particular objects like `),Xs=i(L,"CODE",{});var Yc=h(Xs);di=l(Yc,'"bed"'),Yc.forEach(t),ui=l(L,", "),Fs=i(L,"CODE",{});var qc=h(Fs);fi=l(qc,'"house"'),qc.forEach(t),mi=l(L," and "),zs=i(L,"CODE",{});var Lc=h(zs);wi=l(Lc,'"cat"'),Lc.forEach(t),yi=l(L,`. We see that there is one name in these class labels:
id 27 corresponds to the label `),Ps=i(L,"STRONG",{});var Qc=h(Ps);bi=l(Qc,"\u201Cmarvin\u201D"),Qc.forEach(t),gi=l(L,":"),L.forEach(t),Mo=p(e),f(et.$$.fragment,e),vo=p(e),f(tt.$$.fragment,e),ko=p(e),fe=i(e,"P",{});var ql=h(fe);Mi=l(ql,`Perfect! We can use this name as our wake word for our voice assistant, similar to how \u201CAlexa\u201D is used for Amazon\u2019s Alexa,
or \u201CHey Siri\u201D is used for Apple\u2019s Siri. Of all the possible labels, if the model predicts `),Ds=i(ql,"CODE",{});var Rc=h(Ds);vi=l(Rc,'"marvin"'),Rc.forEach(t),ki=l(ql,` with the highest class
probability, we can be fairly sure that our chosen wake word has been said.`),ql.forEach(t),To=p(e),me=i(e,"P",{});var Ll=h(me);Ti=l(Ll,`Now we need to define a function that is constantly listening to our device\u2019s microphone input, and continuously
passes the audio to the classification model for inference. To do this, we\u2019ll use a handy helper function that comes
with \u{1F917} Transformers called `),st=i(Ll,"A",{href:!0,rel:!0});var Xc=h(st);Os=i(Xc,"CODE",{});var Fc=h(Os);Ji=l(Fc,"ffmpeg_microphone_live"),Fc.forEach(t),Xc.forEach(t),ji=l(Ll,"."),Ll.forEach(t),Jo=p(e),_=i(e,"P",{});var Q=h(_);Ui=l(Q,"This function forwards small chunks of audio of specified length "),Ks=i(Q,"CODE",{});var zc=h(Ks);_i=l(zc,"chunk_length_s"),zc.forEach(t),Ei=l(Q,` to the model to be classified. To ensure that
we get smooth boundaries across chunks of audio, we run a sliding window across our audio with stride `),ea=i(Q,"CODE",{});var Pc=h(ea);Ii=l(Pc,"chunk_length_s / 6"),Pc.forEach(t),$i=l(Q,`.
So that we don\u2019t have to wait for the entire first chunk to be recorded before we start inferring, we also define a minimal
temporary audio input length `),ta=i(Q,"CODE",{});var Dc=h(ta);Wi=l(Dc,"stream_chunk_s"),Dc.forEach(t),Ai=l(Q," that is forwarded to the model before "),sa=i(Q,"CODE",{});var Oc=h(sa);Ci=l(Oc,"chunk_length_s"),Oc.forEach(t),Zi=l(Q," time is reached."),Q.forEach(t),jo=p(e),W=i(e,"P",{});var Ce=h(W);Si=l(Ce,"The function "),aa=i(Ce,"CODE",{});var Kc=h(aa);Ni=l(Kc,"ffmpeg_microphone_live"),Kc.forEach(t),Bi=l(Ce," returns a "),oa=i(Ce,"EM",{});var ep=h(oa);xi=l(ep,"generator"),ep.forEach(t),Gi=l(Ce,` object, yielding a sequence of audio chunks that can each
be passed to the classification model to make a prediction. We can pass this generator directly to the `),la=i(Ce,"CODE",{});var tp=h(la);Hi=l(tp,"pipeline"),tp.forEach(t),Vi=l(Ce,`,
which in turn returns a sequence of output predictions, one for each chunk of audio input. We can inspect the class
label probabilities for each audio chunk, and stop our wake word detection loop when we detect that the wake word
has been spoken.`),Ce.forEach(t),Uo=p(e),B=i(e,"P",{});var ws=h(B);Yi=l(ws,`We\u2019ll use a very simple criteria for classifying whether our wake word was spoken: if the class label with the highest
probability was our wake word, and this probability exceeds a threshold `),na=i(ws,"CODE",{});var sp=h(na);qi=l(sp,"prob_threshold"),sp.forEach(t),Li=l(ws,`, we declare that the wake word
as having been spoken. Using a probability threshold to gate our classifier this way ensures that the wake word is not
erroneously predicted if the audio input is noise, which is typically when the model is very uncertain and all the class
label probabilities low. You might want to tune this probability threshold, or explore more sophisticated means for
the wake word decision through an `),at=i(ws,"A",{href:!0,rel:!0});var ap=h(at);ia=i(ap,"EM",{});var op=h(ia);Qi=l(op,"entropy"),op.forEach(t),ap.forEach(t),Ri=l(ws," (or uncertainty) based metric."),ws.forEach(t),_o=p(e),f(ot.$$.fragment,e),Eo=p(e),A=i(e,"P",{});var Ze=h(A);Xi=l(Ze,"Let\u2019s give this function a try to see how it works! We\u2019ll set the flag "),ra=i(Ze,"CODE",{});var lp=h(ra);Fi=l(lp,"debug=True"),lp.forEach(t),zi=l(Ze,` to print out the prediction for each
chunk of audio. Let the model run for a few seconds to see the kinds of predictions that it makes when there is no speech
input, then clearly say the wake word `),ha=i(Ze,"CODE",{});var np=h(ha);Pi=l(np,'"marvin"'),np.forEach(t),Di=l(Ze," and watch the class label prediction for "),ca=i(Ze,"CODE",{});var ip=h(ca);Oi=l(ip,'"marvin"'),ip.forEach(t),Ki=l(Ze," spike to near 1:"),Ze.forEach(t),Io=p(e),f(lt.$$.fragment,e),$o=p(e),f(nt.$$.fragment,e),Wo=p(e),we=i(e,"P",{});var Ql=h(we);er=l(Ql,`Awesome! As we expect, the model generates garbage predictions for the first few seconds. There is no speech input, so the
model makes close to random predictions, but with very low probability. As soon as we say the wake word, the model predicts
`),pa=i(Ql,"CODE",{});var rp=h(pa);tr=l(rp,'"marvin"'),rp.forEach(t),sr=l(Ql,` with probability close to 1 and terminates the loop, signalling that the wake word has been detected and that the
ASR system should be activated!`),Ql.forEach(t),Ao=p(e),D=i(e,"H2",{class:!0});var Rl=h(D);ye=i(Rl,"A",{id:!0,class:!0,href:!0});var hp=h(ye);da=i(hp,"SPAN",{});var cp=h(da);f(it.$$.fragment,cp),cp.forEach(t),hp.forEach(t),ar=p(Rl),ua=i(Rl,"SPAN",{});var pp=h(ua);or=l(pp,"Speech transcription"),pp.forEach(t),Rl.forEach(t),Co=p(e),x=i(e,"P",{});var ys=h(x);lr=l(ys,"Once again, we\u2019ll use the Whisper model for our speech transcription system. Specifically, we\u2019ll load the "),rt=i(ys,"A",{href:!0,rel:!0});var dp=h(rt);nr=l(dp,"Whisper Base English"),dp.forEach(t),ir=l(ys,`
checkpoint, since it\u2019s small enough to give good inference speed with reasonable transcription accuracy. We\u2019ll use a trick
to get near real-time transcription by being clever with how we forward our audio inputs to the model. As before, feel
free to use any speech recognition checkpoint on `),ht=i(ys,"A",{href:!0,rel:!0});var up=h(ht);rr=l(up,"the Hub"),up.forEach(t),hr=l(ys,`,
including Wav2Vec2, MMS ASR or other Whisper checkpoints:`),ys.forEach(t),Zo=p(e),f(ct.$$.fragment,e),So=p(e),f(be.$$.fragment,e),No=p(e),G=i(e,"P",{});var bs=h(G);cr=l(bs,"We can now define a function to record our microphone input and transcribe the corresponding text. With the "),fa=i(bs,"CODE",{});var fp=h(fa);pr=l(fp,"ffmpeg_microphone_live"),fp.forEach(t),dr=l(bs,`
helper function, we can control how \u2018real-time\u2019 our speech recognition model is. Using a smaller `),ma=i(bs,"CODE",{});var mp=h(ma);ur=l(mp,"stream_chunk_s"),mp.forEach(t),fr=l(bs,` lends
itself to more real-time speech recognition, since we divide our input audio into smaller chunks and transcribe them on
the fly. However, this comes at the expense of poorer accuracy, since there\u2019s less context for the model to infer from.`),bs.forEach(t),Bo=p(e),C=i(e,"P",{});var Se=h(C);mr=l(Se,"As we\u2019re transcribing the speech, we also need to have an idea of when the user "),wa=i(Se,"STRONG",{});var wp=h(wa);wr=l(wp,"stops"),wp.forEach(t),yr=l(Se,` speaking, so that we can terminate
the recording. For simplicity, we\u2019ll terminate our microphone recording after the first `),ya=i(Se,"CODE",{});var yp=h(ya);br=l(yp,"chunk_length_s"),yp.forEach(t),gr=l(Se,` (which is set to
5 seconds by default), but you can experiment with using a `),pt=i(Se,"A",{href:!0,rel:!0});var bp=h(pt);Mr=l(bp,"voice activity detection (VAD)"),bp.forEach(t),vr=l(Se,`
model to predict when the user has stopped speaking.`),Se.forEach(t),xo=p(e),f(dt.$$.fragment,e),Go=p(e),os=i(e,"P",{});var gp=h(os);kr=l(gp,`Let\u2019s give this a go and see how we get on! Once the microphone is live, start speaking and watch your transcription
appear in semi real-time:`),gp.forEach(t),Ho=p(e),f(ut.$$.fragment,e),Vo=p(e),f(ft.$$.fragment,e),Yo=p(e),Z=i(e,"P",{});var Ne=h(Z);Tr=l(Ne,"Nice! You can adjust the maximum audio length "),ba=i(Ne,"CODE",{});var Mp=h(ba);Jr=l(Mp,"chunk_length_s"),Mp.forEach(t),jr=l(Ne,` based on how fast or slow you speak (increase it if you
felt like you didn\u2019t have enough time to speak, decrease it if you were left waiting at the end), and the
`),ga=i(Ne,"CODE",{});var vp=h(ga);Ur=l(vp,"stream_chunk_s"),vp.forEach(t),_r=l(Ne," for the real-time factor. Just pass these as arguments to the "),Ma=i(Ne,"CODE",{});var kp=h(Ma);Er=l(kp,"transcribe"),kp.forEach(t),Ir=l(Ne," function."),Ne.forEach(t),qo=p(e),O=i(e,"H2",{class:!0});var Xl=h(O);ge=i(Xl,"A",{id:!0,class:!0,href:!0});var Tp=h(ge);va=i(Tp,"SPAN",{});var Jp=h(va);f(mt.$$.fragment,Jp),Jp.forEach(t),Tp.forEach(t),$r=p(Xl),ka=i(Xl,"SPAN",{});var jp=h(ka);Wr=l(jp,"Language model query"),jp.forEach(t),Xl.forEach(t),Lo=p(e),Me=i(e,"P",{});var Fl=h(Me);Ar=l(Fl,`Now that we have our spoken query transcribed, we want to generate a meaningful response. To do this, we\u2019ll use an LLM
hosted on the Cloud. Specifically, we\u2019ll pick an LLM on the Hugging Face Hub and use the `),wt=i(Fl,"A",{href:!0,rel:!0});var Up=h(wt);Cr=l(Up,"Inference API"),Up.forEach(t),Zr=l(Fl,`
to easily query the model.`),Fl.forEach(t),Qo=p(e),ve=i(e,"P",{});var zl=h(ve);Sr=l(zl,"First, let\u2019s head over to the Hugging Face Hub. To find our LLM, we\u2019ll use the "),yt=i(zl,"A",{href:!0,rel:!0});var _p=h(yt);Nr=l(_p,"\u{1F917} Open LLM Leaderboard"),_p.forEach(t),Br=l(zl,`,
a Space that ranks LLM models by performance over four generation tasks. We\u2019ll search by \u201Cinstruct\u201D to filter out models
that have been instruction fine-tuned, since these should work better for our querying task:`),zl.forEach(t),Ro=p(e),bt=i(e,"DIV",{class:!0});var Ep=h(bt);Ta=i(Ep,"IMG",{src:!0}),Ep.forEach(t),Xo=p(e),H=i(e,"P",{});var gs=h(H);xr=l(gs,"We\u2019ll use the "),gt=i(gs,"A",{href:!0,rel:!0});var Ip=h(gt);Gr=l(Ip,"tiiuae/falcon-7b-instruct"),Ip.forEach(t),Hr=l(gs," checkpoint by "),Mt=i(gs,"A",{href:!0,rel:!0});var $p=h(Mt);Vr=l($p,"TII"),$p.forEach(t),Yr=l(gs,`,
a 7B parameter decoder-only LM fine-tuned on a mixture of chat and instruction datasets. You can use any LLM on the Hugging
Face Hub that has the \u201CHosted inference API\u201D enabled, just look out for the widget on the right-side of the model card:`),gs.forEach(t),Fo=p(e),vt=i(e,"DIV",{class:!0});var Wp=h(vt);Ja=i(Wp,"IMG",{src:!0}),Wp.forEach(t),zo=p(e),ke=i(e,"P",{});var Pl=h(ke);qr=l(Pl,`The Inference API allows us to send a HTTP request from our local machine to the LLM hosted on the Hub, and returns the
response as a `),ja=i(Pl,"CODE",{});var Ap=h(ja);Lr=l(Ap,"json"),Ap.forEach(t),Qr=l(Pl,` file. All we need to provide is our Hugging Face Hub token (which we retrieve directly from our Hugging Face
Hub folder) and the model id of the LLM we wish to query:`),Pl.forEach(t),Po=p(e),f(kt.$$.fragment,e),Do=p(e),ls=i(e,"P",{});var Cp=h(ls);Rr=l(Cp,"Let\u2019s give it a try with a test input!"),Cp.forEach(t),Oo=p(e),f(Tt.$$.fragment,e),Ko=p(e),f(Jt.$$.fragment,e),el=p(e),ns=i(e,"P",{});var Zp=h(ns);Xr=l(Zp,`You\u2019ll notice just how fast inference is using the Inference API - we only have to send a small number of text tokens
from our local machine to the hosted model, so the communication cost is very low. The LLM is hosted on GPU accelerators,
so inference runs very quickly. Finally, the generated response is transferred back from the model to our local machine,
again with low communication overhead.`),Zp.forEach(t),tl=p(e),K=i(e,"H2",{class:!0});var Dl=h(K);Te=i(Dl,"A",{id:!0,class:!0,href:!0});var Sp=h(Te);Ua=i(Sp,"SPAN",{});var Np=h(Ua);f(jt.$$.fragment,Np),Np.forEach(t),Sp.forEach(t),Fr=p(Dl),_a=i(Dl,"SPAN",{});var Bp=h(_a);zr=l(Bp,"Synthesise speech"),Bp.forEach(t),Dl.forEach(t),sl=p(e),Je=i(e,"P",{});var Ol=h(Je);Pr=l(Ol,"And now we\u2019re ready to get the final spoken output! Once again, we\u2019ll use the Microsoft "),Ut=i(Ol,"A",{href:!0,rel:!0});var xp=h(Ut);Dr=l(xp,"SpeechT5 TTS"),xp.forEach(t),Or=l(Ol,`
model for English TTS, but you can use any TTS model of your choice. Let\u2019s go ahead and load the processor and model:`),Ol.forEach(t),al=p(e),f(_t.$$.fragment,e),ol=p(e),is=i(e,"P",{});var Gp=h(is);Kr=l(Gp,"And also the speaker embeddings:"),Gp.forEach(t),ll=p(e),f(Et.$$.fragment,e),nl=p(e),V=i(e,"P",{});var Ms=h(V);eh=l(Ms,"We\u2019ll re-use the "),Ea=i(Ms,"CODE",{});var Hp=h(Ea);th=l(Hp,"synthesise"),Hp.forEach(t),sh=l(Ms," function that we defined in the previous chapter on "),rs=i(Ms,"A",{href:!0});var Vp=h(rs);ah=l(Vp,"Speech-to-speech translation"),Vp.forEach(t),oh=l(Ms,":"),Ms.forEach(t),il=p(e),f(It.$$.fragment,e),rl=p(e),hs=i(e,"P",{});var Yp=h(hs);lh=l(Yp,"Let\u2019s quickly verify this works as expected:"),Yp.forEach(t),hl=p(e),f($t.$$.fragment,e),cl=p(e),cs=i(e,"P",{});var qp=h(cs);nh=l(qp,"Nice job \u{1F44D}"),qp.forEach(t),pl=p(e),ee=i(e,"H2",{class:!0});var Kl=h(ee);je=i(Kl,"A",{id:!0,class:!0,href:!0});var Lp=h(je);Ia=i(Lp,"SPAN",{});var Qp=h(Ia);f(Wt.$$.fragment,Qp),Qp.forEach(t),Lp.forEach(t),ih=p(Kl),$a=i(Kl,"SPAN",{});var Rp=h($a);rh=l(Rp,"Marvin \u{1F916}"),Rp.forEach(t),Kl.forEach(t),dl=p(e),Ue=i(e,"P",{});var en=h(Ue);hh=l(en,`Now that we\u2019ve defined a function for each of the four stages of the voice assistant pipeline, all that\u2019s left to do is
piece them together to get our end-to-end voice assistant. We\u2019ll simply concatenate the four stages, starting with
wake word detection (`),Wa=i(en,"CODE",{});var Xp=h(Wa);ch=l(Xp,"launch_fn"),Xp.forEach(t),ph=l(en,"), speech transcription, querying the LLM, and finally speech synthesis."),en.forEach(t),ul=p(e),f(At.$$.fragment,e),fl=p(e),ps=i(e,"P",{});var Fp=h(ps);dh=l(Fp,"Try it out with a few prompts! Here are some examples to get you started:"),Fp.forEach(t),ml=p(e),Y=i(e,"UL",{});var vs=h(Y);Aa=i(vs,"LI",{});var zp=h(Aa);Ca=i(zp,"EM",{});var Pp=h(Ca);uh=l(Pp,"What is the hottest country in the world?"),Pp.forEach(t),zp.forEach(t),fh=p(vs),Za=i(vs,"LI",{});var Dp=h(Za);Sa=i(Dp,"EM",{});var Op=h(Sa);mh=l(Op,"How do Transformer models work?"),Op.forEach(t),Dp.forEach(t),wh=p(vs),Na=i(vs,"LI",{});var Kp=h(Na);Ba=i(Kp,"EM",{});var ed=h(Ba);yh=l(ed,"Do you know Spanish?"),ed.forEach(t),Kp.forEach(t),vs.forEach(t),wl=p(e),ds=i(e,"P",{});var td=h(ds);bh=l(td,`And with that, we have our end-to-end voice assistant complete, made using the \u{1F917} audio tools you\u2019ve learnt throughout
this course, with a sprinkling of LLM magic at the end. There are several extensions that we could make to improve the
voice assistant. Firstly, the audio classification model classifies 35 different labels. We could use a smaller, more
lightweight binary classification model that only predicts whether the wake word was spoken or not. Secondly, we pre-load
all the models ahead and keep them running on our device. If we wanted to save power, we would only load each model at
the time it was required, and subsequently un-load them afterwards. Thirdly, we\u2019re missing a voice activity detection model
in our transcription function, transcribing for a fixed amount of time, which in some cases is too long, and in others too
short.`),td.forEach(t),yl=p(e),te=i(e,"H2",{class:!0});var tn=h(te);_e=i(tn,"A",{id:!0,class:!0,href:!0});var sd=h(_e);xa=i(sd,"SPAN",{});var ad=h(xa);f(Ct.$$.fragment,ad),ad.forEach(t),sd.forEach(t),gh=p(tn),Ga=i(tn,"SPAN",{});var od=h(Ga);Mh=l(od,"Generalise to anything \u{1FA84}"),od.forEach(t),tn.forEach(t),bl=p(e),us=i(e,"P",{});var ld=h(us);vh=l(ld,`So far, we\u2019ve seen how we can generate speech outputs with our voice assistant Marvin. To finish, we\u2019ll demonstrate how
we can generalise these speech outputs to text, audio and image.`),ld.forEach(t),gl=p(e),Ee=i(e,"P",{});var sn=h(Ee);kh=l(sn,"We\u2019ll use "),Zt=i(sn,"A",{href:!0,rel:!0});var nd=h(Zt);Th=l(nd,"Transformers Agents"),nd.forEach(t),Jh=l(sn,` to build our assistant.
Transformers Agents provides a natural language API on top of the \u{1F917} Transformers and Diffusers libraries, interpreting
a natural language input using an LLM with carefully crafted prompts, and using a set of curated tools to provide
multimodal outputs.`),sn.forEach(t),Ml=p(e),q=i(e,"P",{});var ks=h(q);jh=l(ks,"Let\u2019s go ahead and instantiate an agent. There are "),St=i(ks,"A",{href:!0,rel:!0});var id=h(St);Uh=l(id,"three LLMs available"),id.forEach(t),_h=l(ks,`
for Transformers Agents, two of which are open-source and free on the Hugging Face Hub. The third is a model from OpenAI
that requires an OpenAI API key. We\u2019ll use the free `),Nt=i(ks,"A",{href:!0,rel:!0});var rd=h(Nt);Eh=l(rd,"Bigcode Starcoder"),rd.forEach(t),Ih=l(ks,` model
in this example, but you can also try either of the other LLMs available:`),ks.forEach(t),vl=p(e),f(Bt.$$.fragment,e),kl=p(e),Ie=i(e,"P",{});var an=h(Ie);$h=l(an,"To use the agent, we simply have to call "),Ha=i(an,"CODE",{});var hd=h(Ha);Wh=l(hd,"agent.run"),hd.forEach(t),Ah=l(an,` with our text prompt. As an example, we\u2019ll get it to generate an
image of a cat \u{1F408} (that hopefully looks a bit better than this emoji):`),an.forEach(t),Tl=p(e),f(xt.$$.fragment,e),Jl=p(e),Gt=i(e,"DIV",{class:!0});var cd=h(Gt);Va=i(cd,"IMG",{src:!0}),cd.forEach(t),jl=p(e),f($e.$$.fragment,e),Ul=p(e),We=i(e,"P",{});var on=h(We);Ch=l(on,"Easy as that! The Agent interpreted our prompt, and used "),Ht=i(on,"A",{href:!0,rel:!0});var pd=h(Ht);Zh=l(pd,"Stable Diffusion"),pd.forEach(t),Sh=l(on,`
under the hood to generate the image, without us having to worry about loading the model, writing the function or executing
the code.`),on.forEach(t),_l=p(e),fs=i(e,"P",{});var dd=h(fs);Nh=l(dd,`We can now replace our LLM query function and text synthesis step with our Transformers Agent in our voice assistant,
since the Agent is going to take care of both of these steps for us:`),dd.forEach(t),El=p(e),f(Vt.$$.fragment,e),Il=p(e),ms=i(e,"P",{});var ud=h(ms);Bh=l(ud,`Try speaking the same prompt \u201CGenerate an image of a cat\u201D and see how the system gets on. If you ask the Agent a simple
question / answer query, the Agent will respond with a text answer. You can encourage it to generate multimodal outputs
by asking it to return an image or speech. For example, you can ask it to: \u201CGenerate an image of a cat, caption it, and
speak the caption\u201D.`),ud.forEach(t),$l=p(e),Ae=i(e,"P",{});var ln=h(Ae);xh=l(ln,`While the Agent is more flexible than our first iteration Marvin \u{1F916} assistant, generalising the voice assistant task in this way
may lead to inferior performance on standard voice assistant queries. To recover performance, you can try using a
more performant LLM checkpoint, such as the one from OpenAI, or define a set of `),Yt=i(ln,"A",{href:!0,rel:!0});var fd=h(Yt);Gh=l(fd,"custom tools"),fd.forEach(t),Hh=l(ln,`
that are specific to the voice assistant task.`),ln.forEach(t),this.h()},h(){d(k,"name","hf:doc:metadata"),d(k,"content",JSON.stringify(Jd)),d(E,"id","creating-a-voice-assistant"),d(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(E,"href","#creating-a-voice-assistant"),d(j,"class","relative group"),Lt(Js.src,Yh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/voice_assistant.png")||d(Js,"src",Yh),d(xe,"class","flex justify-center"),d(oe,"id","1-wake-word-detection"),d(oe,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(oe,"href","#1-wake-word-detection"),d(R,"class","relative group"),d(le,"id","2-speech-transcription"),d(le,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(le,"href","#2-speech-transcription"),d(X,"class","relative group"),d(ne,"id","3-language-model-query"),d(ne,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ne,"href","#3-language-model-query"),d(F,"class","relative group"),d(re,"id","4-synthesise-speech"),d(re,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(re,"href","#4-synthesise-speech"),d(z,"class","relative group"),d(he,"id","wake-word-detection"),d(he,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(he,"href","#wake-word-detection"),d(P,"class","relative group"),d(es,"href","../chapter4/classification_models"),d(Le,"href","https://huggingface.co/datasets/speech_commands"),d(Le,"rel","nofollow"),d(Qe,"href","https://huggingface.co/datasets/speech_commands/viewer/v0.01/train"),d(Qe,"rel","nofollow"),d(Xe,"href","https://huggingface.co/models"),d(Xe,"rel","nofollow"),Lt(Vs.src,qh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/all_models.png")||d(Vs,"src",qh),d(Fe,"class","flex justify-center"),Lt(Ys.src,Lh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/by_audio_classification.png")||d(Ys,"src",Lh),d(ze,"class","flex justify-center"),Lt(Ls.src,Qh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/by_speech_commands.png")||d(Ls,"src",Qh),d(Pe,"class","flex justify-center"),d(De,"href","https://huggingface.co/MIT/ast-finetuned-speech-commands-v2"),d(De,"rel","nofollow"),d(st,"href","https://github.com/huggingface/transformers/blob/fb78769b9c053876ed7ae152ee995b0439a4462a/src/transformers/pipelines/audio_utils.py#L98"),d(st,"rel","nofollow"),d(at,"href","https://en.wikipedia.org/wiki/Entropy_(information_theory)"),d(at,"rel","nofollow"),d(ye,"id","speech-transcription"),d(ye,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ye,"href","#speech-transcription"),d(D,"class","relative group"),d(rt,"href","https://huggingface.co/openai/whisper-base.en"),d(rt,"rel","nofollow"),d(ht,"href","https://huggingface.co/models?pipeline_tag=automatic-speech-recognition&library=transformers&sort=trending"),d(ht,"rel","nofollow"),d(pt,"href","https://huggingface.co/models?pipeline_tag=voice-activity-detection&sort=trending"),d(pt,"rel","nofollow"),d(ge,"id","language-model-query"),d(ge,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(ge,"href","#language-model-query"),d(O,"class","relative group"),d(wt,"href","https://huggingface.co/inference-api"),d(wt,"rel","nofollow"),d(yt,"href","https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard"),d(yt,"rel","nofollow"),Lt(Ta.src,Rh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/llm_leaderboard.png")||d(Ta,"src",Rh),d(bt,"class","flex justify-center"),d(gt,"href","https://huggingface.co/tiiuae/falcon-7b-instruct"),d(gt,"rel","nofollow"),d(Mt,"href","https://www.tii.ae/"),d(Mt,"rel","nofollow"),Lt(Ja.src,Xh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/inference_api.png")||d(Ja,"src",Xh),d(vt,"class","flex justify-center"),d(Te,"id","synthesise-speech"),d(Te,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(Te,"href","#synthesise-speech"),d(K,"class","relative group"),d(Ut,"href","https://huggingface.co/microsoft/speecht5_tts"),d(Ut,"rel","nofollow"),d(rs,"href","speech-to-speech"),d(je,"id","marvin"),d(je,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(je,"href","#marvin"),d(ee,"class","relative group"),d(_e,"id","generalise-to-anything"),d(_e,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),d(_e,"href","#generalise-to-anything"),d(te,"class","relative group"),d(Zt,"href","https://huggingface.co/docs/transformers/transformers_agents"),d(Zt,"rel","nofollow"),d(St,"href","https://huggingface.co/docs/transformers/transformers_agents#quickstart"),d(St,"rel","nofollow"),d(Nt,"href","https://huggingface.co/bigcode/starcoder"),d(Nt,"rel","nofollow"),Lt(Va.src,Fh="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/generated_cat.png")||d(Va,"src",Fh),d(Gt,"class","flex justify-center"),d(Ht,"href","https://huggingface.co/docs/diffusers/using-diffusers/conditional_image_generation"),d(Ht,"rel","nofollow"),d(Yt,"href","https://huggingface.co/docs/transformers/transformers_agents#custom-tools"),d(Yt,"rel","nofollow")},m(e,a){s(document.head,k),r(e,T,a),r(e,j,a),s(j,E),s(E,I),m(S,I,null),s(j,se),s(j,g),s(g,$),r(e,Be,a),r(e,ae,a),s(ae,nn),s(ae,Ts),s(Ts,rn),s(ae,hn),r(e,qa,a),r(e,Rt,a),s(Rt,cn),r(e,La,a),r(e,xe,a),s(xe,Js),r(e,Qa,a),r(e,R,a),s(R,oe),s(oe,js),m(Ge,js,null),s(R,pn),s(R,Us),s(Us,dn),r(e,Ra,a),r(e,Xt,a),s(Xt,un),r(e,Xa,a),r(e,Ft,a),s(Ft,fn),r(e,Fa,a),r(e,X,a),s(X,le),s(le,_s),m(He,_s,null),s(X,mn),s(X,Es),s(Es,wn),r(e,za,a),r(e,zt,a),s(zt,yn),r(e,Pa,a),r(e,Pt,a),s(Pt,bn),r(e,Da,a),r(e,F,a),s(F,ne),s(ne,Is),m(Ve,Is,null),s(F,gn),s(F,$s),s($s,Mn),r(e,Oa,a),r(e,ie,a),s(ie,vn),s(ie,Ws),s(Ws,kn),s(ie,Tn),r(e,Ka,a),r(e,Dt,a),s(Dt,Jn),r(e,eo,a),r(e,z,a),s(z,re),s(re,As),m(Ye,As,null),s(z,jn),s(z,Cs),s(Cs,Un),r(e,to,a),r(e,Ot,a),s(Ot,_n),r(e,so,a),r(e,Kt,a),s(Kt,En),r(e,ao,a),r(e,P,a),s(P,he),s(he,Zs),m(qe,Zs,null),s(P,In),s(P,Ss),s(Ss,$n),r(e,oo,a),r(e,M,a),s(M,Wn),s(M,es),s(es,An),s(M,Cn),s(M,Le),s(Le,Zn),s(M,Sn),s(M,Ns),s(Ns,Nn),s(M,Bn),s(M,Bs),s(Bs,xn),s(M,Gn),s(M,xs),s(xs,Hn),s(M,Vn),s(M,Gs),s(Gs,Yn),s(M,qn),s(M,Hs),s(Hs,Ln),s(M,Qn),s(M,Qe),s(Qe,Rn),s(M,Xn),r(e,lo,a),r(e,ts,a),s(ts,Fn),r(e,no,a),r(e,Re,a),s(Re,zn),s(Re,Xe),s(Xe,Pn),r(e,io,a),r(e,ss,a),s(ss,Dn),r(e,ro,a),r(e,Fe,a),s(Fe,Vs),r(e,ho,a),r(e,as,a),s(as,On),r(e,co,a),r(e,ze,a),s(ze,Ys),r(e,po,a),r(e,ce,a),s(ce,Kn),s(ce,qs),s(qs,ei),s(ce,ti),r(e,uo,a),r(e,Pe,a),s(Pe,Ls),r(e,fo,a),r(e,pe,a),s(pe,si),s(pe,De),s(De,ai),s(pe,oi),r(e,mo,a),r(e,de,a),s(de,li),s(de,Qs),s(Qs,ni),s(de,ii),r(e,wo,a),m(Oe,e,a),r(e,yo,a),r(e,ue,a),s(ue,ri),s(ue,Rs),s(Rs,hi),s(ue,ci),r(e,bo,a),m(Ke,e,a),r(e,go,a),r(e,U,a),s(U,pi),s(U,Xs),s(Xs,di),s(U,ui),s(U,Fs),s(Fs,fi),s(U,mi),s(U,zs),s(zs,wi),s(U,yi),s(U,Ps),s(Ps,bi),s(U,gi),r(e,Mo,a),m(et,e,a),r(e,vo,a),m(tt,e,a),r(e,ko,a),r(e,fe,a),s(fe,Mi),s(fe,Ds),s(Ds,vi),s(fe,ki),r(e,To,a),r(e,me,a),s(me,Ti),s(me,st),s(st,Os),s(Os,Ji),s(me,ji),r(e,Jo,a),r(e,_,a),s(_,Ui),s(_,Ks),s(Ks,_i),s(_,Ei),s(_,ea),s(ea,Ii),s(_,$i),s(_,ta),s(ta,Wi),s(_,Ai),s(_,sa),s(sa,Ci),s(_,Zi),r(e,jo,a),r(e,W,a),s(W,Si),s(W,aa),s(aa,Ni),s(W,Bi),s(W,oa),s(oa,xi),s(W,Gi),s(W,la),s(la,Hi),s(W,Vi),r(e,Uo,a),r(e,B,a),s(B,Yi),s(B,na),s(na,qi),s(B,Li),s(B,at),s(at,ia),s(ia,Qi),s(B,Ri),r(e,_o,a),m(ot,e,a),r(e,Eo,a),r(e,A,a),s(A,Xi),s(A,ra),s(ra,Fi),s(A,zi),s(A,ha),s(ha,Pi),s(A,Di),s(A,ca),s(ca,Oi),s(A,Ki),r(e,Io,a),m(lt,e,a),r(e,$o,a),m(nt,e,a),r(e,Wo,a),r(e,we,a),s(we,er),s(we,pa),s(pa,tr),s(we,sr),r(e,Ao,a),r(e,D,a),s(D,ye),s(ye,da),m(it,da,null),s(D,ar),s(D,ua),s(ua,or),r(e,Co,a),r(e,x,a),s(x,lr),s(x,rt),s(rt,nr),s(x,ir),s(x,ht),s(ht,rr),s(x,hr),r(e,Zo,a),m(ct,e,a),r(e,So,a),m(be,e,a),r(e,No,a),r(e,G,a),s(G,cr),s(G,fa),s(fa,pr),s(G,dr),s(G,ma),s(ma,ur),s(G,fr),r(e,Bo,a),r(e,C,a),s(C,mr),s(C,wa),s(wa,wr),s(C,yr),s(C,ya),s(ya,br),s(C,gr),s(C,pt),s(pt,Mr),s(C,vr),r(e,xo,a),m(dt,e,a),r(e,Go,a),r(e,os,a),s(os,kr),r(e,Ho,a),m(ut,e,a),r(e,Vo,a),m(ft,e,a),r(e,Yo,a),r(e,Z,a),s(Z,Tr),s(Z,ba),s(ba,Jr),s(Z,jr),s(Z,ga),s(ga,Ur),s(Z,_r),s(Z,Ma),s(Ma,Er),s(Z,Ir),r(e,qo,a),r(e,O,a),s(O,ge),s(ge,va),m(mt,va,null),s(O,$r),s(O,ka),s(ka,Wr),r(e,Lo,a),r(e,Me,a),s(Me,Ar),s(Me,wt),s(wt,Cr),s(Me,Zr),r(e,Qo,a),r(e,ve,a),s(ve,Sr),s(ve,yt),s(yt,Nr),s(ve,Br),r(e,Ro,a),r(e,bt,a),s(bt,Ta),r(e,Xo,a),r(e,H,a),s(H,xr),s(H,gt),s(gt,Gr),s(H,Hr),s(H,Mt),s(Mt,Vr),s(H,Yr),r(e,Fo,a),r(e,vt,a),s(vt,Ja),r(e,zo,a),r(e,ke,a),s(ke,qr),s(ke,ja),s(ja,Lr),s(ke,Qr),r(e,Po,a),m(kt,e,a),r(e,Do,a),r(e,ls,a),s(ls,Rr),r(e,Oo,a),m(Tt,e,a),r(e,Ko,a),m(Jt,e,a),r(e,el,a),r(e,ns,a),s(ns,Xr),r(e,tl,a),r(e,K,a),s(K,Te),s(Te,Ua),m(jt,Ua,null),s(K,Fr),s(K,_a),s(_a,zr),r(e,sl,a),r(e,Je,a),s(Je,Pr),s(Je,Ut),s(Ut,Dr),s(Je,Or),r(e,al,a),m(_t,e,a),r(e,ol,a),r(e,is,a),s(is,Kr),r(e,ll,a),m(Et,e,a),r(e,nl,a),r(e,V,a),s(V,eh),s(V,Ea),s(Ea,th),s(V,sh),s(V,rs),s(rs,ah),s(V,oh),r(e,il,a),m(It,e,a),r(e,rl,a),r(e,hs,a),s(hs,lh),r(e,hl,a),m($t,e,a),r(e,cl,a),r(e,cs,a),s(cs,nh),r(e,pl,a),r(e,ee,a),s(ee,je),s(je,Ia),m(Wt,Ia,null),s(ee,ih),s(ee,$a),s($a,rh),r(e,dl,a),r(e,Ue,a),s(Ue,hh),s(Ue,Wa),s(Wa,ch),s(Ue,ph),r(e,ul,a),m(At,e,a),r(e,fl,a),r(e,ps,a),s(ps,dh),r(e,ml,a),r(e,Y,a),s(Y,Aa),s(Aa,Ca),s(Ca,uh),s(Y,fh),s(Y,Za),s(Za,Sa),s(Sa,mh),s(Y,wh),s(Y,Na),s(Na,Ba),s(Ba,yh),r(e,wl,a),r(e,ds,a),s(ds,bh),r(e,yl,a),r(e,te,a),s(te,_e),s(_e,xa),m(Ct,xa,null),s(te,gh),s(te,Ga),s(Ga,Mh),r(e,bl,a),r(e,us,a),s(us,vh),r(e,gl,a),r(e,Ee,a),s(Ee,kh),s(Ee,Zt),s(Zt,Th),s(Ee,Jh),r(e,Ml,a),r(e,q,a),s(q,jh),s(q,St),s(St,Uh),s(q,_h),s(q,Nt),s(Nt,Eh),s(q,Ih),r(e,vl,a),m(Bt,e,a),r(e,kl,a),r(e,Ie,a),s(Ie,$h),s(Ie,Ha),s(Ha,Wh),s(Ie,Ah),r(e,Tl,a),m(xt,e,a),r(e,Jl,a),r(e,Gt,a),s(Gt,Va),r(e,jl,a),m($e,e,a),r(e,Ul,a),r(e,We,a),s(We,Ch),s(We,Ht),s(Ht,Zh),s(We,Sh),r(e,_l,a),r(e,fs,a),s(fs,Nh),r(e,El,a),m(Vt,e,a),r(e,Il,a),r(e,ms,a),s(ms,Bh),r(e,$l,a),r(e,Ae,a),s(Ae,xh),s(Ae,Yt),s(Yt,Gh),s(Ae,Hh),Wl=!0},p(e,[a]){const qt={};a&2&&(qt.$$scope={dirty:a,ctx:e}),be.$set(qt);const Ya={};a&2&&(Ya.$$scope={dirty:a,ctx:e}),$e.$set(Ya)},i(e){Wl||(w(S.$$.fragment,e),w(Ge.$$.fragment,e),w(He.$$.fragment,e),w(Ve.$$.fragment,e),w(Ye.$$.fragment,e),w(qe.$$.fragment,e),w(Oe.$$.fragment,e),w(Ke.$$.fragment,e),w(et.$$.fragment,e),w(tt.$$.fragment,e),w(ot.$$.fragment,e),w(lt.$$.fragment,e),w(nt.$$.fragment,e),w(it.$$.fragment,e),w(ct.$$.fragment,e),w(be.$$.fragment,e),w(dt.$$.fragment,e),w(ut.$$.fragment,e),w(ft.$$.fragment,e),w(mt.$$.fragment,e),w(kt.$$.fragment,e),w(Tt.$$.fragment,e),w(Jt.$$.fragment,e),w(jt.$$.fragment,e),w(_t.$$.fragment,e),w(Et.$$.fragment,e),w(It.$$.fragment,e),w($t.$$.fragment,e),w(Wt.$$.fragment,e),w(At.$$.fragment,e),w(Ct.$$.fragment,e),w(Bt.$$.fragment,e),w(xt.$$.fragment,e),w($e.$$.fragment,e),w(Vt.$$.fragment,e),Wl=!0)},o(e){y(S.$$.fragment,e),y(Ge.$$.fragment,e),y(He.$$.fragment,e),y(Ve.$$.fragment,e),y(Ye.$$.fragment,e),y(qe.$$.fragment,e),y(Oe.$$.fragment,e),y(Ke.$$.fragment,e),y(et.$$.fragment,e),y(tt.$$.fragment,e),y(ot.$$.fragment,e),y(lt.$$.fragment,e),y(nt.$$.fragment,e),y(it.$$.fragment,e),y(ct.$$.fragment,e),y(be.$$.fragment,e),y(dt.$$.fragment,e),y(ut.$$.fragment,e),y(ft.$$.fragment,e),y(mt.$$.fragment,e),y(kt.$$.fragment,e),y(Tt.$$.fragment,e),y(Jt.$$.fragment,e),y(jt.$$.fragment,e),y(_t.$$.fragment,e),y(Et.$$.fragment,e),y(It.$$.fragment,e),y($t.$$.fragment,e),y(Wt.$$.fragment,e),y(At.$$.fragment,e),y(Ct.$$.fragment,e),y(Bt.$$.fragment,e),y(xt.$$.fragment,e),y($e.$$.fragment,e),y(Vt.$$.fragment,e),Wl=!1},d(e){t(k),e&&t(T),e&&t(j),b(S),e&&t(Be),e&&t(ae),e&&t(qa),e&&t(Rt),e&&t(La),e&&t(xe),e&&t(Qa),e&&t(R),b(Ge),e&&t(Ra),e&&t(Xt),e&&t(Xa),e&&t(Ft),e&&t(Fa),e&&t(X),b(He),e&&t(za),e&&t(zt),e&&t(Pa),e&&t(Pt),e&&t(Da),e&&t(F),b(Ve),e&&t(Oa),e&&t(ie),e&&t(Ka),e&&t(Dt),e&&t(eo),e&&t(z),b(Ye),e&&t(to),e&&t(Ot),e&&t(so),e&&t(Kt),e&&t(ao),e&&t(P),b(qe),e&&t(oo),e&&t(M),e&&t(lo),e&&t(ts),e&&t(no),e&&t(Re),e&&t(io),e&&t(ss),e&&t(ro),e&&t(Fe),e&&t(ho),e&&t(as),e&&t(co),e&&t(ze),e&&t(po),e&&t(ce),e&&t(uo),e&&t(Pe),e&&t(fo),e&&t(pe),e&&t(mo),e&&t(de),e&&t(wo),b(Oe,e),e&&t(yo),e&&t(ue),e&&t(bo),b(Ke,e),e&&t(go),e&&t(U),e&&t(Mo),b(et,e),e&&t(vo),b(tt,e),e&&t(ko),e&&t(fe),e&&t(To),e&&t(me),e&&t(Jo),e&&t(_),e&&t(jo),e&&t(W),e&&t(Uo),e&&t(B),e&&t(_o),b(ot,e),e&&t(Eo),e&&t(A),e&&t(Io),b(lt,e),e&&t($o),b(nt,e),e&&t(Wo),e&&t(we),e&&t(Ao),e&&t(D),b(it),e&&t(Co),e&&t(x),e&&t(Zo),b(ct,e),e&&t(So),b(be,e),e&&t(No),e&&t(G),e&&t(Bo),e&&t(C),e&&t(xo),b(dt,e),e&&t(Go),e&&t(os),e&&t(Ho),b(ut,e),e&&t(Vo),b(ft,e),e&&t(Yo),e&&t(Z),e&&t(qo),e&&t(O),b(mt),e&&t(Lo),e&&t(Me),e&&t(Qo),e&&t(ve),e&&t(Ro),e&&t(bt),e&&t(Xo),e&&t(H),e&&t(Fo),e&&t(vt),e&&t(zo),e&&t(ke),e&&t(Po),b(kt,e),e&&t(Do),e&&t(ls),e&&t(Oo),b(Tt,e),e&&t(Ko),b(Jt,e),e&&t(el),e&&t(ns),e&&t(tl),e&&t(K),b(jt),e&&t(sl),e&&t(Je),e&&t(al),b(_t,e),e&&t(ol),e&&t(is),e&&t(ll),b(Et,e),e&&t(nl),e&&t(V),e&&t(il),b(It,e),e&&t(rl),e&&t(hs),e&&t(hl),b($t,e),e&&t(cl),e&&t(cs),e&&t(pl),e&&t(ee),b(Wt),e&&t(dl),e&&t(Ue),e&&t(ul),b(At,e),e&&t(fl),e&&t(ps),e&&t(ml),e&&t(Y),e&&t(wl),e&&t(ds),e&&t(yl),e&&t(te),b(Ct),e&&t(bl),e&&t(us),e&&t(gl),e&&t(Ee),e&&t(Ml),e&&t(q),e&&t(vl),b(Bt,e),e&&t(kl),e&&t(Ie),e&&t(Tl),b(xt,e),e&&t(Jl),e&&t(Gt),e&&t(jl),b($e,e),e&&t(Ul),e&&t(We),e&&t(_l),e&&t(fs),e&&t(El),b(Vt,e),e&&t(Il),e&&t(ms),e&&t($l),e&&t(Ae)}}}const Jd={local:"creating-a-voice-assistant",sections:[{local:null,sections:[{local:"1-wake-word-detection",title:"1. Wake word detection"},{local:"2-speech-transcription",title:"2. Speech transcription"},{local:"3-language-model-query",title:"3. Language model query"},{local:"4-synthesise-speech",title:"4. Synthesise speech"}],title:null},{local:"wake-word-detection",title:"Wake word detection"},{local:"speech-transcription",title:"Speech transcription"},{local:"language-model-query",title:"Language model query"},{local:"synthesise-speech",title:"Synthesise speech"},{local:"marvin",title:"Marvin \u{1F916}"},{local:"generalise-to-anything",title:"Generalise to anything \u{1FA84}"}],title:"Creating a voice assistant"};function jd(Qt){return Md(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $d extends wd{constructor(k){super();yd(this,k,jd,Td,bd,{})}}export{$d as default,Jd as metadata};
