import{S as zf,i as Xf,s as Qf,e as a,k as i,w as q,t as n,M as Kf,c as o,d as e,m as d,a as r,x as O,h as l,b as h,N as tu,G as t,g as f,y as Z,L as eu,q as z,o as X,B as Q,v as au}from"../../chunks/vendor-hf-doc-builder.js";import{I as bt}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as Zf}from"../../chunks/CodeBlock-hf-doc-builder.js";function ou(Id){let K,Bo,tt,it,ce,Tt,Hr,fe,Nr,jo,zt,Pr,Ro,Xt,Mr,Lo,et,dt,ue,kt,Ir,pe,Ur,Jo,at,ht,me,Dt,Wr,ve,Yr,Go,ot,ct,ge,Ct,Vr,we,Fr,qo,Qt,Br,Oo,rt,ft,_e,At,jr,Ee,Rr,Zo,ut,ye,Lr,Jr,be,Gr,zo,Kt,qr,Xo,$t,Qo,te,Or,Ko,St,tr,st,pt,Te,xt,Zr,ke,zr,er,nt,mt,De,Ht,Xr,Ce,Qr,ar,ee,Kr,or,vt,Ae,u,$e,ts,es,Se,as,os,xe,rs,ss,He,ns,ls,Ne,is,ds,Pe,hs,cs,Me,fs,us,Ie,ps,ms,p,m,Ue,Nt,vs,gs,We,ws,_s,Ye,Es,ys,Ve,bs,Ts,Fe,ks,Ds,Be,Cs,As,je,$s,Ss,Re,xs,Hs,v,Le,Pt,Ns,Ps,Je,Ms,Is,Ge,Us,Ws,qe,Ys,Vs,Oe,Fs,Bs,Ze,js,Rs,ze,Ls,Js,Xe,Gs,qs,g,Qe,Mt,Os,Zs,Ke,zs,Xs,ta,Qs,Ks,ea,tn,en,aa,an,on,oa,rn,sn,ra,nn,ln,sa,dn,hn,w,na,It,cn,fn,la,un,pn,ia,mn,vn,da,gn,wn,ha,_n,En,ca,yn,bn,fa,Tn,kn,ua,Dn,Cn,_,pa,Ut,An,$n,ma,Sn,xn,va,Hn,Nn,ga,Pn,Mn,wa,In,Un,_a,Wn,Yn,Ea,Vn,Fn,ya,Bn,jn,E,ba,Wt,Rn,Ln,Ta,Jn,Gn,ka,qn,On,Da,Zn,zn,Ca,Xn,Qn,Aa,Kn,tl,$a,el,al,Sa,ol,rl,y,xa,Yt,sl,nl,Ha,ll,il,Na,dl,hl,Pa,cl,fl,Ma,ul,pl,Ia,ml,vl,Ua,gl,wl,Wa,_l,El,b,Ya,Vt,yl,bl,Va,Tl,kl,Fa,Dl,Cl,Ba,Al,$l,ja,Sl,xl,Ra,Hl,Nl,La,Pl,Ml,Ja,Il,rr,ae,Ul,sr,gt,Ga,T,qa,Wl,Yl,Oa,Vl,Fl,Za,Bl,jl,za,Rl,Ll,Xa,Jl,Gl,Qa,ql,Ol,Ka,Zl,zl,to,Xl,Ql,R,k,eo,Ft,Kl,ti,ao,ei,ai,oo,oi,ri,ro,si,ni,so,li,ii,no,di,hi,lo,ci,fi,io,ui,pi,D,ho,Bt,mi,vi,co,gi,wi,fo,_i,Ei,uo,yi,bi,po,Ti,ki,mo,Di,Ci,vo,Ai,$i,go,Si,xi,C,wo,jt,Hi,Ni,_o,Pi,Mi,Eo,Ii,Ui,yo,Wi,Yi,bo,Vi,Fi,To,Bi,ji,ko,Ri,Li,Do,Ji,Gi,A,Co,Rt,qi,Oi,Ao,Zi,zi,$o,Xi,Qi,So,Ki,td,xo,ed,ad,Ho,od,rd,No,sd,nd,Po,ld,nr,L,id,Lt,dd,hd,Jt,cd,fd,lr,oe,ud,ir,lt,wt,Mo,Gt,pd,Io,md,dr,re,vd,hr,_t,gd,qt,wd,_d,cr,J,Ed,Ot,Uo,yd,bd,Wo,Td,kd,fr,G,Dd,Yo,Cd,Ad,Vo,$d,Sd,ur,Zt,se,Ud,pr,ne,xd,mr,le,Hd,vr,Et,Nd,Fo,Pd,Md,gr;return Tt=new bt({}),kt=new bt({}),Dt=new bt({}),Ct=new bt({}),At=new bt({}),$t=new Zf({props:{code:"JUUyJTgwJTlDQ29uc2lkZXIlMjB0aGUlMjB0YXNrJTIwb2YlMjB0cmFpbmluZyUyMGElMjBtb2RlbCUyMG9uJTIwYSUyMHNwZWVjaCUyMHJlY29nbml0aW9uJTIwZGF0YXNldCVFMiU4MCU5RA==",highlighted:'\u201CConsider <span class="hljs-keyword">the</span> task <span class="hljs-keyword">of</span> training <span class="hljs-keyword">a</span> model <span class="hljs-keyword">on</span> <span class="hljs-title">a</span> <span class="hljs-title">speech</span> <span class="hljs-title">recognition</span> <span class="hljs-title">dataset</span>\u201D'}}),St=new Zf({props:{code:"JUUyJTgwJTlDTGV0JUUyJTgwJTk5cyUyMHVoaCUyMGxldCdzJTIwdGFrZSUyMGElMjBsb29rJTIwYXQlMjBob3clMjB5b3UnZCUyMGdvJTIwYWJvdXQlMjB0cmFpbmluZyUyMGElMjBtb2RlbCUyMG9uJTIwdWhtJTIwYSUyMHNwLSUyMHNwZWVjaCUyMHJlY29nbml0aW9uJTIwZGF0YXNldCVFMiU4MCU5RA==",highlighted:'\u201C<span class="hljs-keyword">Let</span>\u2019s uhh <span class="hljs-keyword">let</span><span class="hljs-comment">&#x27;s take a look at how you&#x27;d go about training a model on uhm a sp- speech recognition dataset\u201D</span>'}}),xt=new bt({}),Ht=new bt({}),Gt=new bt({}),{c(){K=a("meta"),Bo=i(),tt=a("h1"),it=a("a"),ce=a("span"),q(Tt.$$.fragment),Hr=i(),fe=a("span"),Nr=n("Choosing a dataset"),jo=i(),zt=a("p"),Pr=n(`As with any machine learning problem, our model is only as good as the data that we train it on. Speech recognition
datasets vary considerably in how they are curated and the domains that they cover. To pick the right dataset, we need
to match our criteria with the features that a dataset offers.`),Ro=i(),Xt=a("p"),Mr=n("Before we pick a dataset, we first need to understand the key defining features."),Lo=i(),et=a("h2"),dt=a("a"),ue=a("span"),q(kt.$$.fragment),Ir=i(),pe=a("span"),Ur=n("Features of speech datasets"),Jo=i(),at=a("h3"),ht=a("a"),me=a("span"),q(Dt.$$.fragment),Wr=i(),ve=a("span"),Yr=n("1. Number of hours"),Go=n(`

Simply put, the number of training hours indicates how large the dataset is. It\u2019s analogous to the number of training
examples in an NLP dataset. However, bigger datasets aren\u2019t necessarily better. If we want a model that generalises well,
we want a **diverse** dataset with lots of different speakers, domains and speaking styles.
`),ot=a("h3"),ct=a("a"),ge=a("span"),q(Ct.$$.fragment),Vr=i(),we=a("span"),Fr=n("2. Domain"),qo=n(`

The domain entails where the data was sourced from, whether it be audiobooks, podcasts, YouTube or financial meetings.
Each domain has a different distribution of data. For example, audiobooks are recorded in high-quality studio conditions
(with no background noise) and text that is taken from written literature. Whereas for YouTube, the audio likely contains
more background noise and a more informal style of speech.
`),Qt=a("p"),Br=n(`We need to match our domain to the conditions we anticipate at inference time. For instance, if we train our model on
audiobooks, we can\u2019t expect it to perform well in noisy environments.`),Oo=i(),rt=a("h3"),ft=a("a"),_e=a("span"),q(At.$$.fragment),jr=i(),Ee=a("span"),Rr=n("3. Speaking style"),Zo=n(`

The speaking style falls into one of two categories:
`),ut=a("ul"),ye=a("li"),Lr=n("Narrated: read from a script"),Jr=i(),be=a("li"),Gr=n("Spontaneous: un-scripted, conversational speech"),zo=i(),Kt=a("p"),qr=n(`The audio and text data reflect the style of speaking. Since narrated text is scripted, it tends to be spoken articulately
and without any errors:`),Xo=i(),q($t.$$.fragment),Qo=i(),te=a("p"),Or=n(`Whereas for spontaneous speech, we can expect a more colloquial style of speech, with the inclusion of repetitions,
hesitations and false-starts:`),Ko=i(),q(St.$$.fragment),tr=i(),st=a("h3"),pt=a("a"),Te=a("span"),q(xt.$$.fragment),Zr=i(),ke=a("span"),zr=n("4. Transcription style"),er=n(`

The transcription style refers to whether the target text has punctuation, casing or both. If we want a system to generate
fully formatted text that could be used for a publication or meeting transcription, we require training data with punctuation
and casing. If we just require the spoken words in an un-formatted structure, neither punctuation nor casing are necessary.
In this case, we can either pick a dataset without punctuation or casing, or pick one that has punctuation and casing and
then subsequently remove them from the target text through pre-processing.
`),nt=a("h2"),mt=a("a"),De=a("span"),q(Ht.$$.fragment),Xr=i(),Ce=a("span"),Qr=n("A summary of datasets on the Hub"),ar=i(),ee=a("p"),Kr=n("Here is a summary of the most popular English speech recognition datasets on the Hugging Face Hub:"),or=i(),vt=a("table"),Ae=a("thead"),u=a("tr"),$e=a("th"),ts=n("Dataset"),es=i(),Se=a("th"),as=n("Train Hours"),os=i(),xe=a("th"),rs=n("Domain"),ss=i(),He=a("th"),ns=n("Speaking Style"),ls=i(),Ne=a("th"),is=n("Casing"),ds=i(),Pe=a("th"),hs=n("Punctuation"),cs=i(),Me=a("th"),fs=n("License"),us=i(),Ie=a("th"),ps=n("Recommended Use"),ms=i(),p=a("tbody"),m=a("tr"),Ue=a("td"),Nt=a("a"),vs=n("LibriSpeech"),gs=i(),We=a("td"),ws=n("960"),_s=i(),Ye=a("td"),Es=n("Audiobook"),ys=i(),Ve=a("td"),bs=n("Narrated"),Ts=i(),Fe=a("td"),ks=n("\u274C"),Ds=i(),Be=a("td"),Cs=n("\u274C"),As=i(),je=a("td"),$s=n("CC-BY-4.0"),Ss=i(),Re=a("td"),xs=n("Academic benchmarks"),Hs=i(),v=a("tr"),Le=a("td"),Pt=a("a"),Ns=n("Common Voice 11"),Ps=i(),Je=a("td"),Ms=n("3000"),Is=i(),Ge=a("td"),Us=n("Wikipedia"),Ws=i(),qe=a("td"),Ys=n("Narrated"),Vs=i(),Oe=a("td"),Fs=n("\u2705"),Bs=i(),Ze=a("td"),js=n("\u2705"),Rs=i(),ze=a("td"),Ls=n("CC0-1.0"),Js=i(),Xe=a("td"),Gs=n("Non-native speakers"),qs=i(),g=a("tr"),Qe=a("td"),Mt=a("a"),Os=n("VoxPopuli"),Zs=i(),Ke=a("td"),zs=n("540"),Xs=i(),ta=a("td"),Qs=n("European Parliament"),Ks=i(),ea=a("td"),tn=n("Oratory"),en=i(),aa=a("td"),an=n("\u274C"),on=i(),oa=a("td"),rn=n("\u2705"),sn=i(),ra=a("td"),nn=n("CC0"),ln=i(),sa=a("td"),dn=n("Non-native speakers"),hn=i(),w=a("tr"),na=a("td"),It=a("a"),cn=n("TED-LIUM"),fn=i(),la=a("td"),un=n("450"),pn=i(),ia=a("td"),mn=n("TED talks"),vn=i(),da=a("td"),gn=n("Oratory"),wn=i(),ha=a("td"),_n=n("\u274C"),En=i(),ca=a("td"),yn=n("\u274C"),bn=i(),fa=a("td"),Tn=n("CC-BY-NC-ND 3.0"),kn=i(),ua=a("td"),Dn=n("Technical topics"),Cn=i(),_=a("tr"),pa=a("td"),Ut=a("a"),An=n("GigaSpeech"),$n=i(),ma=a("td"),Sn=n("10000"),xn=i(),va=a("td"),Hn=n("Audiobook, podcast, YouTube"),Nn=i(),ga=a("td"),Pn=n("Narrated, spontaneous"),Mn=i(),wa=a("td"),In=n("\u274C"),Un=i(),_a=a("td"),Wn=n("\u2705"),Yn=i(),Ea=a("td"),Vn=n("apache-2.0"),Fn=i(),ya=a("td"),Bn=n("Robustness over multiple domains"),jn=i(),E=a("tr"),ba=a("td"),Wt=a("a"),Rn=n("SPGISpeech"),Ln=i(),Ta=a("td"),Jn=n("5000"),Gn=i(),ka=a("td"),qn=n("Financial meetings"),On=i(),Da=a("td"),Zn=n("Oratory, spontaneous"),zn=i(),Ca=a("td"),Xn=n("\u2705"),Qn=i(),Aa=a("td"),Kn=n("\u2705"),tl=i(),$a=a("td"),el=n("User Agreement"),al=i(),Sa=a("td"),ol=n("Fully formatted transcriptions"),rl=i(),y=a("tr"),xa=a("td"),Yt=a("a"),sl=n("Earnings-22"),nl=i(),Ha=a("td"),ll=n("119"),il=i(),Na=a("td"),dl=n("Financial meetings"),hl=i(),Pa=a("td"),cl=n("Oratory, spontaneous"),fl=i(),Ma=a("td"),ul=n("\u2705"),pl=i(),Ia=a("td"),ml=n("\u2705"),vl=i(),Ua=a("td"),gl=n("CC-BY-SA-4.0"),wl=i(),Wa=a("td"),_l=n("Diversity of accents"),El=i(),b=a("tr"),Ya=a("td"),Vt=a("a"),yl=n("AMI"),bl=i(),Va=a("td"),Tl=n("100"),kl=i(),Fa=a("td"),Dl=n("Meetings"),Cl=i(),Ba=a("td"),Al=n("Spontaneous"),$l=i(),ja=a("td"),Sl=n("\u2705"),xl=i(),Ra=a("td"),Hl=n("\u2705"),Nl=i(),La=a("td"),Pl=n("CC-BY-4.0"),Ml=i(),Ja=a("td"),Il=n("Noisy speech conditions"),rr=i(),ae=a("p"),Ul=n(`This table serves as a reference for selecting a dataset based on your criterion. Below is an equivalent table for
multilingual speech recognition. Note that we omit the train hours column, since this varies depending on the language
for each dataset, and replace it with the number of languages per dataset:`),sr=i(),gt=a("table"),Ga=a("thead"),T=a("tr"),qa=a("th"),Wl=n("Dataset"),Yl=i(),Oa=a("th"),Vl=n("Languages"),Fl=i(),Za=a("th"),Bl=n("Domain"),jl=i(),za=a("th"),Rl=n("Speaking Style"),Ll=i(),Xa=a("th"),Jl=n("Casing"),Gl=i(),Qa=a("th"),ql=n("Punctuation"),Ol=i(),Ka=a("th"),Zl=n("License"),zl=i(),to=a("th"),Xl=n("Recommended Usage"),Ql=i(),R=a("tbody"),k=a("tr"),eo=a("td"),Ft=a("a"),Kl=n("Multilingual LibriSpeech"),ti=i(),ao=a("td"),ei=n("6"),ai=i(),oo=a("td"),oi=n("Audiobooks"),ri=i(),ro=a("td"),si=n("Narrated"),ni=i(),so=a("td"),li=n("\u274C"),ii=i(),no=a("td"),di=n("\u274C"),hi=i(),lo=a("td"),ci=n("CC-BY-4.0"),fi=i(),io=a("td"),ui=n("Academic benchmarks"),pi=i(),D=a("tr"),ho=a("td"),Bt=a("a"),mi=n("Common Voice 13"),vi=i(),co=a("td"),gi=n("108"),wi=i(),fo=a("td"),_i=n("Wikipedia text & crowd-sourced speech"),Ei=i(),uo=a("td"),yi=n("Narrated"),bi=i(),po=a("td"),Ti=n("\u2705"),ki=i(),mo=a("td"),Di=n("\u2705"),Ci=i(),vo=a("td"),Ai=n("CC0-1.0"),$i=i(),go=a("td"),Si=n("Diverse speaker set"),xi=i(),C=a("tr"),wo=a("td"),jt=a("a"),Hi=n("VoxPopuli"),Ni=i(),_o=a("td"),Pi=n("15"),Mi=i(),Eo=a("td"),Ii=n("European Parliament recordings"),Ui=i(),yo=a("td"),Wi=n("Spontaneous"),Yi=i(),bo=a("td"),Vi=n("\u274C"),Fi=i(),To=a("td"),Bi=n("\u2705"),ji=i(),ko=a("td"),Ri=n("CC0"),Li=i(),Do=a("td"),Ji=n("European languages"),Gi=i(),A=a("tr"),Co=a("td"),Rt=a("a"),qi=n("FLEURS"),Oi=i(),Ao=a("td"),Zi=n("101"),zi=i(),$o=a("td"),Xi=n("European Parliament recordings"),Qi=i(),So=a("td"),Ki=n("Spontaneous"),td=i(),xo=a("td"),ed=n("\u274C"),ad=i(),Ho=a("td"),od=n("\u274C"),rd=i(),No=a("td"),sd=n("CC-BY-4.0"),nd=i(),Po=a("td"),ld=n("Multilingual evaluation"),nr=i(),L=a("p"),id=n("For a detailed breakdown of the audio datasets covered in both tables, refer to the blog post "),Lt=a("a"),dd=n("A Complete Guide to Audio Datasets"),hd=n(`.
While there are over 180 speech recognition datasets on the Hub, it may be possible that there isn\u2019t a dataset that matches
your needs. In this case, it\u2019s also possible to use your own audio data with \u{1F917} Datasets. To create a custom audio dataset,
refer to the guide `),Jt=a("a"),cd=n("Create an audio dataset"),fd=n(`. When creating a custom
audio dataset, consider sharing the final dataset on the Hub so that others in the community can benefit from your
efforts - the audio community is inclusive and wide-ranging, and others will appreciate your work as you do theirs.`),lr=i(),oe=a("p"),ud=n(`Alright! Now that we\u2019ve gone through all the criterion for selecting an ASR dataset, let\u2019s pick one for the purpose of this tutorial.
We know that Whisper already does a pretty good job at transcribing data in high-resource languages (such as English and Spanish), so
we\u2019ll focus ourselves on low-resource multilingual transcription. We want to retain Whisper\u2019s ability to predict punctuation and casing,
so it seems from the second table that Common Voice 13 is a great candidate dataset!`),ir=i(),lt=a("h2"),wt=a("a"),Mo=a("span"),q(Gt.$$.fragment),pd=i(),Io=a("span"),md=n("Common Voice 13"),dr=i(),re=a("p"),vd=n(`Common Voice 13 is a crowd-sourced dataset where speakers record text from Wikipedia in various languages. It forms part of
the Common Voice series, a collection of Common Voice datasets released by Mozilla Foundation. At the time of writing,
Common Voice 13 is the latest edition of the dataset, with the most languages and hours per language out of any release to date.`),hr=i(),_t=a("p"),gd=n(`We can get the full list of languages for the Common Voice 13 dataset by checking-out the dataset page on the Hub:
`),qt=a("a"),wd=n("mozilla-foundation/common_voice_13_0"),_d=n(`.
The first time you view this page, you\u2019ll be asked to accept the terms of use. After that, you\u2019ll be given full access to the dataset.`),cr=i(),J=a("p"),Ed=n(`Once we\u2019ve provided authentication to use the dataset, we\u2019ll be presented with the dataset preview. The dataset preview
shows us the first 100 samples of the dataset for each language. What\u2019s more, it\u2019s loaded up with audio samples ready for us
to listen to in real time. For this Unit, we\u2019ll select `),Ot=a("a"),Uo=a("em"),yd=n("Dhivehi"),bd=n(`
(or `),Wo=a("em"),Td=n("Maldivian"),kd=n(`), an Indo-Aryan language spoken in the South Asian island country of the Maldives. While we\u2019re selecting
Dhivehi for this tutorial, the steps covered here apply to any one of the 108 languages in the Common Voice 13 dataset, and
more generally to any one of the 180+ audio datasets on the Hugging Face Hub, so there\u2019s no restriction on language or dialect.`),fr=i(),G=a("p"),Dd=n("We can select the Dhivehi subset of Common Voice 13 by setting the subset to "),Yo=a("code"),Cd=n("dv"),Ad=n(" using the dropdown menu ("),Vo=a("code"),$d=n("dv"),Sd=n(` being the language
identifier code for Dhivehi):`),ur=i(),Zt=a("div"),se=a("img"),pr=i(),ne=a("p"),xd=n(`If we hit the play button on the first sample, we can listen to the audio and see the corresponding text. Have a scroll
through the samples for the train and test sets to get a better feel for the audio and text data that we\u2019re dealing with.
You can tell from the intonation and style that the recordings are taken from narrated speech. You\u2019ll also likely notice
the large variation in speakers and recording quality, a common trait of crowd-sourced data.`),mr=i(),le=a("p"),Hd=n(`The Dataset Preview is a brilliant way of experiencing audio datasets before committing to using them. You can pick any
dataset on the Hub, scroll through the samples and listen to the audio for the different subsets and splits, gauging whether
it\u2019s the right dataset for your needs. Once you\u2019ve selected a dataset, it\u2019s trivial to load the data so that you can
start using it.`),vr=i(),Et=a("p"),Nd=n(`Now, I personally don\u2019t speak Dhivehi, and expect the vast majority of readers not to either! To know if our fine-tuned model
is any good, we\u2019ll need a rigorous way of `),Fo=a("em"),Pd=n("evaluating"),Md=n(` it on unseen data and measuring its transcription accuracy.
We\u2019ll cover exactly this in the next section!`),this.h()},l(s){const c=Kf('[data-svelte="svelte-1phssyn"]',document.head);K=o(c,"META",{name:!0,content:!0}),c.forEach(e),Bo=d(s),tt=o(s,"H1",{class:!0});var wr=r(tt);it=o(wr,"A",{id:!0,class:!0,href:!0});var Wd=r(it);ce=o(Wd,"SPAN",{});var Yd=r(ce);O(Tt.$$.fragment,Yd),Yd.forEach(e),Wd.forEach(e),Hr=d(wr),fe=o(wr,"SPAN",{});var Vd=r(fe);Nr=l(Vd,"Choosing a dataset"),Vd.forEach(e),wr.forEach(e),jo=d(s),zt=o(s,"P",{});var Fd=r(zt);Pr=l(Fd,`As with any machine learning problem, our model is only as good as the data that we train it on. Speech recognition
datasets vary considerably in how they are curated and the domains that they cover. To pick the right dataset, we need
to match our criteria with the features that a dataset offers.`),Fd.forEach(e),Ro=d(s),Xt=o(s,"P",{});var Bd=r(Xt);Mr=l(Bd,"Before we pick a dataset, we first need to understand the key defining features."),Bd.forEach(e),Lo=d(s),et=o(s,"H2",{class:!0});var _r=r(et);dt=o(_r,"A",{id:!0,class:!0,href:!0});var jd=r(dt);ue=o(jd,"SPAN",{});var Rd=r(ue);O(kt.$$.fragment,Rd),Rd.forEach(e),jd.forEach(e),Ir=d(_r),pe=o(_r,"SPAN",{});var Ld=r(pe);Ur=l(Ld,"Features of speech datasets"),Ld.forEach(e),_r.forEach(e),Jo=d(s),at=o(s,"H3",{class:!0});var Er=r(at);ht=o(Er,"A",{id:!0,class:!0,href:!0});var Jd=r(ht);me=o(Jd,"SPAN",{});var Gd=r(me);O(Dt.$$.fragment,Gd),Gd.forEach(e),Jd.forEach(e),Wr=d(Er),ve=o(Er,"SPAN",{});var qd=r(ve);Yr=l(qd,"1. Number of hours"),qd.forEach(e),Er.forEach(e),Go=l(s,`

Simply put, the number of training hours indicates how large the dataset is. It\u2019s analogous to the number of training
examples in an NLP dataset. However, bigger datasets aren\u2019t necessarily better. If we want a model that generalises well,
we want a **diverse** dataset with lots of different speakers, domains and speaking styles.
`),ot=o(s,"H3",{class:!0});var yr=r(ot);ct=o(yr,"A",{id:!0,class:!0,href:!0});var Od=r(ct);ge=o(Od,"SPAN",{});var Zd=r(ge);O(Ct.$$.fragment,Zd),Zd.forEach(e),Od.forEach(e),Vr=d(yr),we=o(yr,"SPAN",{});var zd=r(we);Fr=l(zd,"2. Domain"),zd.forEach(e),yr.forEach(e),qo=l(s,`

The domain entails where the data was sourced from, whether it be audiobooks, podcasts, YouTube or financial meetings.
Each domain has a different distribution of data. For example, audiobooks are recorded in high-quality studio conditions
(with no background noise) and text that is taken from written literature. Whereas for YouTube, the audio likely contains
more background noise and a more informal style of speech.
`),Qt=o(s,"P",{});var Xd=r(Qt);Br=l(Xd,`We need to match our domain to the conditions we anticipate at inference time. For instance, if we train our model on
audiobooks, we can\u2019t expect it to perform well in noisy environments.`),Xd.forEach(e),Oo=d(s),rt=o(s,"H3",{class:!0});var br=r(rt);ft=o(br,"A",{id:!0,class:!0,href:!0});var Qd=r(ft);_e=o(Qd,"SPAN",{});var Kd=r(_e);O(At.$$.fragment,Kd),Kd.forEach(e),Qd.forEach(e),jr=d(br),Ee=o(br,"SPAN",{});var th=r(Ee);Rr=l(th,"3. Speaking style"),th.forEach(e),br.forEach(e),Zo=l(s,`

The speaking style falls into one of two categories:
`),ut=o(s,"UL",{});var Tr=r(ut);ye=o(Tr,"LI",{});var eh=r(ye);Lr=l(eh,"Narrated: read from a script"),eh.forEach(e),Jr=d(Tr),be=o(Tr,"LI",{});var ah=r(be);Gr=l(ah,"Spontaneous: un-scripted, conversational speech"),ah.forEach(e),Tr.forEach(e),zo=d(s),Kt=o(s,"P",{});var oh=r(Kt);qr=l(oh,`The audio and text data reflect the style of speaking. Since narrated text is scripted, it tends to be spoken articulately
and without any errors:`),oh.forEach(e),Xo=d(s),O($t.$$.fragment,s),Qo=d(s),te=o(s,"P",{});var rh=r(te);Or=l(rh,`Whereas for spontaneous speech, we can expect a more colloquial style of speech, with the inclusion of repetitions,
hesitations and false-starts:`),rh.forEach(e),Ko=d(s),O(St.$$.fragment,s),tr=d(s),st=o(s,"H3",{class:!0});var kr=r(st);pt=o(kr,"A",{id:!0,class:!0,href:!0});var sh=r(pt);Te=o(sh,"SPAN",{});var nh=r(Te);O(xt.$$.fragment,nh),nh.forEach(e),sh.forEach(e),Zr=d(kr),ke=o(kr,"SPAN",{});var lh=r(ke);zr=l(lh,"4. Transcription style"),lh.forEach(e),kr.forEach(e),er=l(s,`

The transcription style refers to whether the target text has punctuation, casing or both. If we want a system to generate
fully formatted text that could be used for a publication or meeting transcription, we require training data with punctuation
and casing. If we just require the spoken words in an un-formatted structure, neither punctuation nor casing are necessary.
In this case, we can either pick a dataset without punctuation or casing, or pick one that has punctuation and casing and
then subsequently remove them from the target text through pre-processing.
`),nt=o(s,"H2",{class:!0});var Dr=r(nt);mt=o(Dr,"A",{id:!0,class:!0,href:!0});var ih=r(mt);De=o(ih,"SPAN",{});var dh=r(De);O(Ht.$$.fragment,dh),dh.forEach(e),ih.forEach(e),Xr=d(Dr),Ce=o(Dr,"SPAN",{});var hh=r(Ce);Qr=l(hh,"A summary of datasets on the Hub"),hh.forEach(e),Dr.forEach(e),ar=d(s),ee=o(s,"P",{});var ch=r(ee);Kr=l(ch,"Here is a summary of the most popular English speech recognition datasets on the Hugging Face Hub:"),ch.forEach(e),or=d(s),vt=o(s,"TABLE",{});var Cr=r(vt);Ae=o(Cr,"THEAD",{});var fh=r(Ae);u=o(fh,"TR",{});var $=r(u);$e=o($,"TH",{});var uh=r($e);ts=l(uh,"Dataset"),uh.forEach(e),es=d($),Se=o($,"TH",{});var ph=r(Se);as=l(ph,"Train Hours"),ph.forEach(e),os=d($),xe=o($,"TH",{});var mh=r(xe);rs=l(mh,"Domain"),mh.forEach(e),ss=d($),He=o($,"TH",{});var vh=r(He);ns=l(vh,"Speaking Style"),vh.forEach(e),ls=d($),Ne=o($,"TH",{});var gh=r(Ne);is=l(gh,"Casing"),gh.forEach(e),ds=d($),Pe=o($,"TH",{});var wh=r(Pe);hs=l(wh,"Punctuation"),wh.forEach(e),cs=d($),Me=o($,"TH",{});var _h=r(Me);fs=l(_h,"License"),_h.forEach(e),us=d($),Ie=o($,"TH",{});var Eh=r(Ie);ps=l(Eh,"Recommended Use"),Eh.forEach(e),$.forEach(e),fh.forEach(e),ms=d(Cr),p=o(Cr,"TBODY",{});var S=r(p);m=o(S,"TR",{});var x=r(m);Ue=o(x,"TD",{});var yh=r(Ue);Nt=o(yh,"A",{href:!0,rel:!0});var bh=r(Nt);vs=l(bh,"LibriSpeech"),bh.forEach(e),yh.forEach(e),gs=d(x),We=o(x,"TD",{});var Th=r(We);ws=l(Th,"960"),Th.forEach(e),_s=d(x),Ye=o(x,"TD",{});var kh=r(Ye);Es=l(kh,"Audiobook"),kh.forEach(e),ys=d(x),Ve=o(x,"TD",{});var Dh=r(Ve);bs=l(Dh,"Narrated"),Dh.forEach(e),Ts=d(x),Fe=o(x,"TD",{});var Ch=r(Fe);ks=l(Ch,"\u274C"),Ch.forEach(e),Ds=d(x),Be=o(x,"TD",{});var Ah=r(Be);Cs=l(Ah,"\u274C"),Ah.forEach(e),As=d(x),je=o(x,"TD",{});var $h=r(je);$s=l($h,"CC-BY-4.0"),$h.forEach(e),Ss=d(x),Re=o(x,"TD",{});var Sh=r(Re);xs=l(Sh,"Academic benchmarks"),Sh.forEach(e),x.forEach(e),Hs=d(S),v=o(S,"TR",{});var H=r(v);Le=o(H,"TD",{});var xh=r(Le);Pt=o(xh,"A",{href:!0,rel:!0});var Hh=r(Pt);Ns=l(Hh,"Common Voice 11"),Hh.forEach(e),xh.forEach(e),Ps=d(H),Je=o(H,"TD",{});var Nh=r(Je);Ms=l(Nh,"3000"),Nh.forEach(e),Is=d(H),Ge=o(H,"TD",{});var Ph=r(Ge);Us=l(Ph,"Wikipedia"),Ph.forEach(e),Ws=d(H),qe=o(H,"TD",{});var Mh=r(qe);Ys=l(Mh,"Narrated"),Mh.forEach(e),Vs=d(H),Oe=o(H,"TD",{});var Ih=r(Oe);Fs=l(Ih,"\u2705"),Ih.forEach(e),Bs=d(H),Ze=o(H,"TD",{});var Uh=r(Ze);js=l(Uh,"\u2705"),Uh.forEach(e),Rs=d(H),ze=o(H,"TD",{});var Wh=r(ze);Ls=l(Wh,"CC0-1.0"),Wh.forEach(e),Js=d(H),Xe=o(H,"TD",{});var Yh=r(Xe);Gs=l(Yh,"Non-native speakers"),Yh.forEach(e),H.forEach(e),qs=d(S),g=o(S,"TR",{});var N=r(g);Qe=o(N,"TD",{});var Vh=r(Qe);Mt=o(Vh,"A",{href:!0,rel:!0});var Fh=r(Mt);Os=l(Fh,"VoxPopuli"),Fh.forEach(e),Vh.forEach(e),Zs=d(N),Ke=o(N,"TD",{});var Bh=r(Ke);zs=l(Bh,"540"),Bh.forEach(e),Xs=d(N),ta=o(N,"TD",{});var jh=r(ta);Qs=l(jh,"European Parliament"),jh.forEach(e),Ks=d(N),ea=o(N,"TD",{});var Rh=r(ea);tn=l(Rh,"Oratory"),Rh.forEach(e),en=d(N),aa=o(N,"TD",{});var Lh=r(aa);an=l(Lh,"\u274C"),Lh.forEach(e),on=d(N),oa=o(N,"TD",{});var Jh=r(oa);rn=l(Jh,"\u2705"),Jh.forEach(e),sn=d(N),ra=o(N,"TD",{});var Gh=r(ra);nn=l(Gh,"CC0"),Gh.forEach(e),ln=d(N),sa=o(N,"TD",{});var qh=r(sa);dn=l(qh,"Non-native speakers"),qh.forEach(e),N.forEach(e),hn=d(S),w=o(S,"TR",{});var P=r(w);na=o(P,"TD",{});var Oh=r(na);It=o(Oh,"A",{href:!0,rel:!0});var Zh=r(It);cn=l(Zh,"TED-LIUM"),Zh.forEach(e),Oh.forEach(e),fn=d(P),la=o(P,"TD",{});var zh=r(la);un=l(zh,"450"),zh.forEach(e),pn=d(P),ia=o(P,"TD",{});var Xh=r(ia);mn=l(Xh,"TED talks"),Xh.forEach(e),vn=d(P),da=o(P,"TD",{});var Qh=r(da);gn=l(Qh,"Oratory"),Qh.forEach(e),wn=d(P),ha=o(P,"TD",{});var Kh=r(ha);_n=l(Kh,"\u274C"),Kh.forEach(e),En=d(P),ca=o(P,"TD",{});var tc=r(ca);yn=l(tc,"\u274C"),tc.forEach(e),bn=d(P),fa=o(P,"TD",{});var ec=r(fa);Tn=l(ec,"CC-BY-NC-ND 3.0"),ec.forEach(e),kn=d(P),ua=o(P,"TD",{});var ac=r(ua);Dn=l(ac,"Technical topics"),ac.forEach(e),P.forEach(e),Cn=d(S),_=o(S,"TR",{});var M=r(_);pa=o(M,"TD",{});var oc=r(pa);Ut=o(oc,"A",{href:!0,rel:!0});var rc=r(Ut);An=l(rc,"GigaSpeech"),rc.forEach(e),oc.forEach(e),$n=d(M),ma=o(M,"TD",{});var sc=r(ma);Sn=l(sc,"10000"),sc.forEach(e),xn=d(M),va=o(M,"TD",{});var nc=r(va);Hn=l(nc,"Audiobook, podcast, YouTube"),nc.forEach(e),Nn=d(M),ga=o(M,"TD",{});var lc=r(ga);Pn=l(lc,"Narrated, spontaneous"),lc.forEach(e),Mn=d(M),wa=o(M,"TD",{});var ic=r(wa);In=l(ic,"\u274C"),ic.forEach(e),Un=d(M),_a=o(M,"TD",{});var dc=r(_a);Wn=l(dc,"\u2705"),dc.forEach(e),Yn=d(M),Ea=o(M,"TD",{});var hc=r(Ea);Vn=l(hc,"apache-2.0"),hc.forEach(e),Fn=d(M),ya=o(M,"TD",{});var cc=r(ya);Bn=l(cc,"Robustness over multiple domains"),cc.forEach(e),M.forEach(e),jn=d(S),E=o(S,"TR",{});var I=r(E);ba=o(I,"TD",{});var fc=r(ba);Wt=o(fc,"A",{href:!0,rel:!0});var uc=r(Wt);Rn=l(uc,"SPGISpeech"),uc.forEach(e),fc.forEach(e),Ln=d(I),Ta=o(I,"TD",{});var pc=r(Ta);Jn=l(pc,"5000"),pc.forEach(e),Gn=d(I),ka=o(I,"TD",{});var mc=r(ka);qn=l(mc,"Financial meetings"),mc.forEach(e),On=d(I),Da=o(I,"TD",{});var vc=r(Da);Zn=l(vc,"Oratory, spontaneous"),vc.forEach(e),zn=d(I),Ca=o(I,"TD",{});var gc=r(Ca);Xn=l(gc,"\u2705"),gc.forEach(e),Qn=d(I),Aa=o(I,"TD",{});var wc=r(Aa);Kn=l(wc,"\u2705"),wc.forEach(e),tl=d(I),$a=o(I,"TD",{});var _c=r($a);el=l(_c,"User Agreement"),_c.forEach(e),al=d(I),Sa=o(I,"TD",{});var Ec=r(Sa);ol=l(Ec,"Fully formatted transcriptions"),Ec.forEach(e),I.forEach(e),rl=d(S),y=o(S,"TR",{});var U=r(y);xa=o(U,"TD",{});var yc=r(xa);Yt=o(yc,"A",{href:!0,rel:!0});var bc=r(Yt);sl=l(bc,"Earnings-22"),bc.forEach(e),yc.forEach(e),nl=d(U),Ha=o(U,"TD",{});var Tc=r(Ha);ll=l(Tc,"119"),Tc.forEach(e),il=d(U),Na=o(U,"TD",{});var kc=r(Na);dl=l(kc,"Financial meetings"),kc.forEach(e),hl=d(U),Pa=o(U,"TD",{});var Dc=r(Pa);cl=l(Dc,"Oratory, spontaneous"),Dc.forEach(e),fl=d(U),Ma=o(U,"TD",{});var Cc=r(Ma);ul=l(Cc,"\u2705"),Cc.forEach(e),pl=d(U),Ia=o(U,"TD",{});var Ac=r(Ia);ml=l(Ac,"\u2705"),Ac.forEach(e),vl=d(U),Ua=o(U,"TD",{});var $c=r(Ua);gl=l($c,"CC-BY-SA-4.0"),$c.forEach(e),wl=d(U),Wa=o(U,"TD",{});var Sc=r(Wa);_l=l(Sc,"Diversity of accents"),Sc.forEach(e),U.forEach(e),El=d(S),b=o(S,"TR",{});var W=r(b);Ya=o(W,"TD",{});var xc=r(Ya);Vt=o(xc,"A",{href:!0,rel:!0});var Hc=r(Vt);yl=l(Hc,"AMI"),Hc.forEach(e),xc.forEach(e),bl=d(W),Va=o(W,"TD",{});var Nc=r(Va);Tl=l(Nc,"100"),Nc.forEach(e),kl=d(W),Fa=o(W,"TD",{});var Pc=r(Fa);Dl=l(Pc,"Meetings"),Pc.forEach(e),Cl=d(W),Ba=o(W,"TD",{});var Mc=r(Ba);Al=l(Mc,"Spontaneous"),Mc.forEach(e),$l=d(W),ja=o(W,"TD",{});var Ic=r(ja);Sl=l(Ic,"\u2705"),Ic.forEach(e),xl=d(W),Ra=o(W,"TD",{});var Uc=r(Ra);Hl=l(Uc,"\u2705"),Uc.forEach(e),Nl=d(W),La=o(W,"TD",{});var Wc=r(La);Pl=l(Wc,"CC-BY-4.0"),Wc.forEach(e),Ml=d(W),Ja=o(W,"TD",{});var Yc=r(Ja);Il=l(Yc,"Noisy speech conditions"),Yc.forEach(e),W.forEach(e),S.forEach(e),Cr.forEach(e),rr=d(s),ae=o(s,"P",{});var Vc=r(ae);Ul=l(Vc,`This table serves as a reference for selecting a dataset based on your criterion. Below is an equivalent table for
multilingual speech recognition. Note that we omit the train hours column, since this varies depending on the language
for each dataset, and replace it with the number of languages per dataset:`),Vc.forEach(e),sr=d(s),gt=o(s,"TABLE",{});var Ar=r(gt);Ga=o(Ar,"THEAD",{});var Fc=r(Ga);T=o(Fc,"TR",{});var Y=r(T);qa=o(Y,"TH",{});var Bc=r(qa);Wl=l(Bc,"Dataset"),Bc.forEach(e),Yl=d(Y),Oa=o(Y,"TH",{});var jc=r(Oa);Vl=l(jc,"Languages"),jc.forEach(e),Fl=d(Y),Za=o(Y,"TH",{});var Rc=r(Za);Bl=l(Rc,"Domain"),Rc.forEach(e),jl=d(Y),za=o(Y,"TH",{});var Lc=r(za);Rl=l(Lc,"Speaking Style"),Lc.forEach(e),Ll=d(Y),Xa=o(Y,"TH",{});var Jc=r(Xa);Jl=l(Jc,"Casing"),Jc.forEach(e),Gl=d(Y),Qa=o(Y,"TH",{});var Gc=r(Qa);ql=l(Gc,"Punctuation"),Gc.forEach(e),Ol=d(Y),Ka=o(Y,"TH",{});var qc=r(Ka);Zl=l(qc,"License"),qc.forEach(e),zl=d(Y),to=o(Y,"TH",{});var Oc=r(to);Xl=l(Oc,"Recommended Usage"),Oc.forEach(e),Y.forEach(e),Fc.forEach(e),Ql=d(Ar),R=o(Ar,"TBODY",{});var yt=r(R);k=o(yt,"TR",{});var V=r(k);eo=o(V,"TD",{});var Zc=r(eo);Ft=o(Zc,"A",{href:!0,rel:!0});var zc=r(Ft);Kl=l(zc,"Multilingual LibriSpeech"),zc.forEach(e),Zc.forEach(e),ti=d(V),ao=o(V,"TD",{});var Xc=r(ao);ei=l(Xc,"6"),Xc.forEach(e),ai=d(V),oo=o(V,"TD",{});var Qc=r(oo);oi=l(Qc,"Audiobooks"),Qc.forEach(e),ri=d(V),ro=o(V,"TD",{});var Kc=r(ro);si=l(Kc,"Narrated"),Kc.forEach(e),ni=d(V),so=o(V,"TD",{});var tf=r(so);li=l(tf,"\u274C"),tf.forEach(e),ii=d(V),no=o(V,"TD",{});var ef=r(no);di=l(ef,"\u274C"),ef.forEach(e),hi=d(V),lo=o(V,"TD",{});var af=r(lo);ci=l(af,"CC-BY-4.0"),af.forEach(e),fi=d(V),io=o(V,"TD",{});var of=r(io);ui=l(of,"Academic benchmarks"),of.forEach(e),V.forEach(e),pi=d(yt),D=o(yt,"TR",{});var F=r(D);ho=o(F,"TD",{});var rf=r(ho);Bt=o(rf,"A",{href:!0,rel:!0});var sf=r(Bt);mi=l(sf,"Common Voice 13"),sf.forEach(e),rf.forEach(e),vi=d(F),co=o(F,"TD",{});var nf=r(co);gi=l(nf,"108"),nf.forEach(e),wi=d(F),fo=o(F,"TD",{});var lf=r(fo);_i=l(lf,"Wikipedia text & crowd-sourced speech"),lf.forEach(e),Ei=d(F),uo=o(F,"TD",{});var df=r(uo);yi=l(df,"Narrated"),df.forEach(e),bi=d(F),po=o(F,"TD",{});var hf=r(po);Ti=l(hf,"\u2705"),hf.forEach(e),ki=d(F),mo=o(F,"TD",{});var cf=r(mo);Di=l(cf,"\u2705"),cf.forEach(e),Ci=d(F),vo=o(F,"TD",{});var ff=r(vo);Ai=l(ff,"CC0-1.0"),ff.forEach(e),$i=d(F),go=o(F,"TD",{});var uf=r(go);Si=l(uf,"Diverse speaker set"),uf.forEach(e),F.forEach(e),xi=d(yt),C=o(yt,"TR",{});var B=r(C);wo=o(B,"TD",{});var pf=r(wo);jt=o(pf,"A",{href:!0,rel:!0});var mf=r(jt);Hi=l(mf,"VoxPopuli"),mf.forEach(e),pf.forEach(e),Ni=d(B),_o=o(B,"TD",{});var vf=r(_o);Pi=l(vf,"15"),vf.forEach(e),Mi=d(B),Eo=o(B,"TD",{});var gf=r(Eo);Ii=l(gf,"European Parliament recordings"),gf.forEach(e),Ui=d(B),yo=o(B,"TD",{});var wf=r(yo);Wi=l(wf,"Spontaneous"),wf.forEach(e),Yi=d(B),bo=o(B,"TD",{});var _f=r(bo);Vi=l(_f,"\u274C"),_f.forEach(e),Fi=d(B),To=o(B,"TD",{});var Ef=r(To);Bi=l(Ef,"\u2705"),Ef.forEach(e),ji=d(B),ko=o(B,"TD",{});var yf=r(ko);Ri=l(yf,"CC0"),yf.forEach(e),Li=d(B),Do=o(B,"TD",{});var bf=r(Do);Ji=l(bf,"European languages"),bf.forEach(e),B.forEach(e),Gi=d(yt),A=o(yt,"TR",{});var j=r(A);Co=o(j,"TD",{});var Tf=r(Co);Rt=o(Tf,"A",{href:!0,rel:!0});var kf=r(Rt);qi=l(kf,"FLEURS"),kf.forEach(e),Tf.forEach(e),Oi=d(j),Ao=o(j,"TD",{});var Df=r(Ao);Zi=l(Df,"101"),Df.forEach(e),zi=d(j),$o=o(j,"TD",{});var Cf=r($o);Xi=l(Cf,"European Parliament recordings"),Cf.forEach(e),Qi=d(j),So=o(j,"TD",{});var Af=r(So);Ki=l(Af,"Spontaneous"),Af.forEach(e),td=d(j),xo=o(j,"TD",{});var $f=r(xo);ed=l($f,"\u274C"),$f.forEach(e),ad=d(j),Ho=o(j,"TD",{});var Sf=r(Ho);od=l(Sf,"\u274C"),Sf.forEach(e),rd=d(j),No=o(j,"TD",{});var xf=r(No);sd=l(xf,"CC-BY-4.0"),xf.forEach(e),nd=d(j),Po=o(j,"TD",{});var Hf=r(Po);ld=l(Hf,"Multilingual evaluation"),Hf.forEach(e),j.forEach(e),yt.forEach(e),Ar.forEach(e),nr=d(s),L=o(s,"P",{});var ie=r(L);id=l(ie,"For a detailed breakdown of the audio datasets covered in both tables, refer to the blog post "),Lt=o(ie,"A",{href:!0,rel:!0});var Nf=r(Lt);dd=l(Nf,"A Complete Guide to Audio Datasets"),Nf.forEach(e),hd=l(ie,`.
While there are over 180 speech recognition datasets on the Hub, it may be possible that there isn\u2019t a dataset that matches
your needs. In this case, it\u2019s also possible to use your own audio data with \u{1F917} Datasets. To create a custom audio dataset,
refer to the guide `),Jt=o(ie,"A",{href:!0,rel:!0});var Pf=r(Jt);cd=l(Pf,"Create an audio dataset"),Pf.forEach(e),fd=l(ie,`. When creating a custom
audio dataset, consider sharing the final dataset on the Hub so that others in the community can benefit from your
efforts - the audio community is inclusive and wide-ranging, and others will appreciate your work as you do theirs.`),ie.forEach(e),lr=d(s),oe=o(s,"P",{});var Mf=r(oe);ud=l(Mf,`Alright! Now that we\u2019ve gone through all the criterion for selecting an ASR dataset, let\u2019s pick one for the purpose of this tutorial.
We know that Whisper already does a pretty good job at transcribing data in high-resource languages (such as English and Spanish), so
we\u2019ll focus ourselves on low-resource multilingual transcription. We want to retain Whisper\u2019s ability to predict punctuation and casing,
so it seems from the second table that Common Voice 13 is a great candidate dataset!`),Mf.forEach(e),ir=d(s),lt=o(s,"H2",{class:!0});var $r=r(lt);wt=o($r,"A",{id:!0,class:!0,href:!0});var If=r(wt);Mo=o(If,"SPAN",{});var Uf=r(Mo);O(Gt.$$.fragment,Uf),Uf.forEach(e),If.forEach(e),pd=d($r),Io=o($r,"SPAN",{});var Wf=r(Io);md=l(Wf,"Common Voice 13"),Wf.forEach(e),$r.forEach(e),dr=d(s),re=o(s,"P",{});var Yf=r(re);vd=l(Yf,`Common Voice 13 is a crowd-sourced dataset where speakers record text from Wikipedia in various languages. It forms part of
the Common Voice series, a collection of Common Voice datasets released by Mozilla Foundation. At the time of writing,
Common Voice 13 is the latest edition of the dataset, with the most languages and hours per language out of any release to date.`),Yf.forEach(e),hr=d(s),_t=o(s,"P",{});var Sr=r(_t);gd=l(Sr,`We can get the full list of languages for the Common Voice 13 dataset by checking-out the dataset page on the Hub:
`),qt=o(Sr,"A",{href:!0,rel:!0});var Vf=r(qt);wd=l(Vf,"mozilla-foundation/common_voice_13_0"),Vf.forEach(e),_d=l(Sr,`.
The first time you view this page, you\u2019ll be asked to accept the terms of use. After that, you\u2019ll be given full access to the dataset.`),Sr.forEach(e),cr=d(s),J=o(s,"P",{});var de=r(J);Ed=l(de,`Once we\u2019ve provided authentication to use the dataset, we\u2019ll be presented with the dataset preview. The dataset preview
shows us the first 100 samples of the dataset for each language. What\u2019s more, it\u2019s loaded up with audio samples ready for us
to listen to in real time. For this Unit, we\u2019ll select `),Ot=o(de,"A",{href:!0,rel:!0});var Ff=r(Ot);Uo=o(Ff,"EM",{});var Bf=r(Uo);yd=l(Bf,"Dhivehi"),Bf.forEach(e),Ff.forEach(e),bd=l(de,`
(or `),Wo=o(de,"EM",{});var jf=r(Wo);Td=l(jf,"Maldivian"),jf.forEach(e),kd=l(de,`), an Indo-Aryan language spoken in the South Asian island country of the Maldives. While we\u2019re selecting
Dhivehi for this tutorial, the steps covered here apply to any one of the 108 languages in the Common Voice 13 dataset, and
more generally to any one of the 180+ audio datasets on the Hugging Face Hub, so there\u2019s no restriction on language or dialect.`),de.forEach(e),fr=d(s),G=o(s,"P",{});var he=r(G);Dd=l(he,"We can select the Dhivehi subset of Common Voice 13 by setting the subset to "),Yo=o(he,"CODE",{});var Rf=r(Yo);Cd=l(Rf,"dv"),Rf.forEach(e),Ad=l(he," using the dropdown menu ("),Vo=o(he,"CODE",{});var Lf=r(Vo);$d=l(Lf,"dv"),Lf.forEach(e),Sd=l(he,` being the language
identifier code for Dhivehi):`),he.forEach(e),ur=d(s),Zt=o(s,"DIV",{class:!0});var Jf=r(Zt);se=o(Jf,"IMG",{src:!0,alt:!0}),Jf.forEach(e),pr=d(s),ne=o(s,"P",{});var Gf=r(ne);xd=l(Gf,`If we hit the play button on the first sample, we can listen to the audio and see the corresponding text. Have a scroll
through the samples for the train and test sets to get a better feel for the audio and text data that we\u2019re dealing with.
You can tell from the intonation and style that the recordings are taken from narrated speech. You\u2019ll also likely notice
the large variation in speakers and recording quality, a common trait of crowd-sourced data.`),Gf.forEach(e),mr=d(s),le=o(s,"P",{});var qf=r(le);Hd=l(qf,`The Dataset Preview is a brilliant way of experiencing audio datasets before committing to using them. You can pick any
dataset on the Hub, scroll through the samples and listen to the audio for the different subsets and splits, gauging whether
it\u2019s the right dataset for your needs. Once you\u2019ve selected a dataset, it\u2019s trivial to load the data so that you can
start using it.`),qf.forEach(e),vr=d(s),Et=o(s,"P",{});var xr=r(Et);Nd=l(xr,`Now, I personally don\u2019t speak Dhivehi, and expect the vast majority of readers not to either! To know if our fine-tuned model
is any good, we\u2019ll need a rigorous way of `),Fo=o(xr,"EM",{});var Of=r(Fo);Pd=l(Of,"evaluating"),Of.forEach(e),Md=l(xr,` it on unseen data and measuring its transcription accuracy.
We\u2019ll cover exactly this in the next section!`),xr.forEach(e),this.h()},h(){h(K,"name","hf:doc:metadata"),h(K,"content",JSON.stringify(ru)),h(it,"id","choosing-a-dataset"),h(it,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(it,"href","#choosing-a-dataset"),h(tt,"class","relative group"),h(dt,"id","features-of-speech-datasets"),h(dt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(dt,"href","#features-of-speech-datasets"),h(et,"class","relative group"),h(ht,"id","1-number-of-hours"),h(ht,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ht,"href","#1-number-of-hours"),h(at,"class","relative group"),h(ct,"id","2-domain"),h(ct,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ct,"href","#2-domain"),h(ot,"class","relative group"),h(ft,"id","3-speaking-style"),h(ft,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(ft,"href","#3-speaking-style"),h(rt,"class","relative group"),h(pt,"id","4-transcription-style"),h(pt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(pt,"href","#4-transcription-style"),h(st,"class","relative group"),h(mt,"id","a-summary-of-datasets-on-the-hub"),h(mt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(mt,"href","#a-summary-of-datasets-on-the-hub"),h(nt,"class","relative group"),h(Nt,"href","https://huggingface.co/datasets/librispeech_asr"),h(Nt,"rel","nofollow"),h(Pt,"href","https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0"),h(Pt,"rel","nofollow"),h(Mt,"href","https://huggingface.co/datasets/facebook/voxpopuli"),h(Mt,"rel","nofollow"),h(It,"href","https://huggingface.co/datasets/LIUM/tedlium"),h(It,"rel","nofollow"),h(Ut,"href","https://huggingface.co/datasets/speechcolab/gigaspeech"),h(Ut,"rel","nofollow"),h(Wt,"href","https://huggingface.co/datasets/kensho/spgispeech"),h(Wt,"rel","nofollow"),h(Yt,"href","https://huggingface.co/datasets/revdotcom/earnings22"),h(Yt,"rel","nofollow"),h(Vt,"href","https://huggingface.co/datasets/edinburghcstr/ami"),h(Vt,"rel","nofollow"),h(Ft,"href","https://huggingface.co/datasets/facebook/multilingual_librispeech"),h(Ft,"rel","nofollow"),h(Bt,"href","https://huggingface.co/datasets/mozilla-foundation/common_voice_13_0"),h(Bt,"rel","nofollow"),h(jt,"href","https://huggingface.co/datasets/facebook/voxpopuli"),h(jt,"rel","nofollow"),h(Rt,"href","https://huggingface.co/datasets/google/fleurs"),h(Rt,"rel","nofollow"),h(Lt,"href","https://huggingface.co/blog/audio-datasets#a-tour-of-audio-datasets-on-the-hub"),h(Lt,"rel","nofollow"),h(Jt,"href","https://huggingface.co/docs/datasets/audio_dataset"),h(Jt,"rel","nofollow"),h(wt,"id","common-voice-13"),h(wt,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(wt,"href","#common-voice-13"),h(lt,"class","relative group"),h(qt,"href","https://huggingface.co/datasets/mozilla-foundation/common_voice_13_0"),h(qt,"rel","nofollow"),h(Ot,"href","https://en.wikipedia.org/wiki/Maldivian_language"),h(Ot,"rel","nofollow"),tu(se.src,Ud="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/cv_13_dv_selection.png")||h(se,"src",Ud),h(se,"alt","Selecting the Dhivehi split from the Dataset's Preview"),h(Zt,"class","flex justify-center")},m(s,c){t(document.head,K),f(s,Bo,c),f(s,tt,c),t(tt,it),t(it,ce),Z(Tt,ce,null),t(tt,Hr),t(tt,fe),t(fe,Nr),f(s,jo,c),f(s,zt,c),t(zt,Pr),f(s,Ro,c),f(s,Xt,c),t(Xt,Mr),f(s,Lo,c),f(s,et,c),t(et,dt),t(dt,ue),Z(kt,ue,null),t(et,Ir),t(et,pe),t(pe,Ur),f(s,Jo,c),f(s,at,c),t(at,ht),t(ht,me),Z(Dt,me,null),t(at,Wr),t(at,ve),t(ve,Yr),f(s,Go,c),f(s,ot,c),t(ot,ct),t(ct,ge),Z(Ct,ge,null),t(ot,Vr),t(ot,we),t(we,Fr),f(s,qo,c),f(s,Qt,c),t(Qt,Br),f(s,Oo,c),f(s,rt,c),t(rt,ft),t(ft,_e),Z(At,_e,null),t(rt,jr),t(rt,Ee),t(Ee,Rr),f(s,Zo,c),f(s,ut,c),t(ut,ye),t(ye,Lr),t(ut,Jr),t(ut,be),t(be,Gr),f(s,zo,c),f(s,Kt,c),t(Kt,qr),f(s,Xo,c),Z($t,s,c),f(s,Qo,c),f(s,te,c),t(te,Or),f(s,Ko,c),Z(St,s,c),f(s,tr,c),f(s,st,c),t(st,pt),t(pt,Te),Z(xt,Te,null),t(st,Zr),t(st,ke),t(ke,zr),f(s,er,c),f(s,nt,c),t(nt,mt),t(mt,De),Z(Ht,De,null),t(nt,Xr),t(nt,Ce),t(Ce,Qr),f(s,ar,c),f(s,ee,c),t(ee,Kr),f(s,or,c),f(s,vt,c),t(vt,Ae),t(Ae,u),t(u,$e),t($e,ts),t(u,es),t(u,Se),t(Se,as),t(u,os),t(u,xe),t(xe,rs),t(u,ss),t(u,He),t(He,ns),t(u,ls),t(u,Ne),t(Ne,is),t(u,ds),t(u,Pe),t(Pe,hs),t(u,cs),t(u,Me),t(Me,fs),t(u,us),t(u,Ie),t(Ie,ps),t(vt,ms),t(vt,p),t(p,m),t(m,Ue),t(Ue,Nt),t(Nt,vs),t(m,gs),t(m,We),t(We,ws),t(m,_s),t(m,Ye),t(Ye,Es),t(m,ys),t(m,Ve),t(Ve,bs),t(m,Ts),t(m,Fe),t(Fe,ks),t(m,Ds),t(m,Be),t(Be,Cs),t(m,As),t(m,je),t(je,$s),t(m,Ss),t(m,Re),t(Re,xs),t(p,Hs),t(p,v),t(v,Le),t(Le,Pt),t(Pt,Ns),t(v,Ps),t(v,Je),t(Je,Ms),t(v,Is),t(v,Ge),t(Ge,Us),t(v,Ws),t(v,qe),t(qe,Ys),t(v,Vs),t(v,Oe),t(Oe,Fs),t(v,Bs),t(v,Ze),t(Ze,js),t(v,Rs),t(v,ze),t(ze,Ls),t(v,Js),t(v,Xe),t(Xe,Gs),t(p,qs),t(p,g),t(g,Qe),t(Qe,Mt),t(Mt,Os),t(g,Zs),t(g,Ke),t(Ke,zs),t(g,Xs),t(g,ta),t(ta,Qs),t(g,Ks),t(g,ea),t(ea,tn),t(g,en),t(g,aa),t(aa,an),t(g,on),t(g,oa),t(oa,rn),t(g,sn),t(g,ra),t(ra,nn),t(g,ln),t(g,sa),t(sa,dn),t(p,hn),t(p,w),t(w,na),t(na,It),t(It,cn),t(w,fn),t(w,la),t(la,un),t(w,pn),t(w,ia),t(ia,mn),t(w,vn),t(w,da),t(da,gn),t(w,wn),t(w,ha),t(ha,_n),t(w,En),t(w,ca),t(ca,yn),t(w,bn),t(w,fa),t(fa,Tn),t(w,kn),t(w,ua),t(ua,Dn),t(p,Cn),t(p,_),t(_,pa),t(pa,Ut),t(Ut,An),t(_,$n),t(_,ma),t(ma,Sn),t(_,xn),t(_,va),t(va,Hn),t(_,Nn),t(_,ga),t(ga,Pn),t(_,Mn),t(_,wa),t(wa,In),t(_,Un),t(_,_a),t(_a,Wn),t(_,Yn),t(_,Ea),t(Ea,Vn),t(_,Fn),t(_,ya),t(ya,Bn),t(p,jn),t(p,E),t(E,ba),t(ba,Wt),t(Wt,Rn),t(E,Ln),t(E,Ta),t(Ta,Jn),t(E,Gn),t(E,ka),t(ka,qn),t(E,On),t(E,Da),t(Da,Zn),t(E,zn),t(E,Ca),t(Ca,Xn),t(E,Qn),t(E,Aa),t(Aa,Kn),t(E,tl),t(E,$a),t($a,el),t(E,al),t(E,Sa),t(Sa,ol),t(p,rl),t(p,y),t(y,xa),t(xa,Yt),t(Yt,sl),t(y,nl),t(y,Ha),t(Ha,ll),t(y,il),t(y,Na),t(Na,dl),t(y,hl),t(y,Pa),t(Pa,cl),t(y,fl),t(y,Ma),t(Ma,ul),t(y,pl),t(y,Ia),t(Ia,ml),t(y,vl),t(y,Ua),t(Ua,gl),t(y,wl),t(y,Wa),t(Wa,_l),t(p,El),t(p,b),t(b,Ya),t(Ya,Vt),t(Vt,yl),t(b,bl),t(b,Va),t(Va,Tl),t(b,kl),t(b,Fa),t(Fa,Dl),t(b,Cl),t(b,Ba),t(Ba,Al),t(b,$l),t(b,ja),t(ja,Sl),t(b,xl),t(b,Ra),t(Ra,Hl),t(b,Nl),t(b,La),t(La,Pl),t(b,Ml),t(b,Ja),t(Ja,Il),f(s,rr,c),f(s,ae,c),t(ae,Ul),f(s,sr,c),f(s,gt,c),t(gt,Ga),t(Ga,T),t(T,qa),t(qa,Wl),t(T,Yl),t(T,Oa),t(Oa,Vl),t(T,Fl),t(T,Za),t(Za,Bl),t(T,jl),t(T,za),t(za,Rl),t(T,Ll),t(T,Xa),t(Xa,Jl),t(T,Gl),t(T,Qa),t(Qa,ql),t(T,Ol),t(T,Ka),t(Ka,Zl),t(T,zl),t(T,to),t(to,Xl),t(gt,Ql),t(gt,R),t(R,k),t(k,eo),t(eo,Ft),t(Ft,Kl),t(k,ti),t(k,ao),t(ao,ei),t(k,ai),t(k,oo),t(oo,oi),t(k,ri),t(k,ro),t(ro,si),t(k,ni),t(k,so),t(so,li),t(k,ii),t(k,no),t(no,di),t(k,hi),t(k,lo),t(lo,ci),t(k,fi),t(k,io),t(io,ui),t(R,pi),t(R,D),t(D,ho),t(ho,Bt),t(Bt,mi),t(D,vi),t(D,co),t(co,gi),t(D,wi),t(D,fo),t(fo,_i),t(D,Ei),t(D,uo),t(uo,yi),t(D,bi),t(D,po),t(po,Ti),t(D,ki),t(D,mo),t(mo,Di),t(D,Ci),t(D,vo),t(vo,Ai),t(D,$i),t(D,go),t(go,Si),t(R,xi),t(R,C),t(C,wo),t(wo,jt),t(jt,Hi),t(C,Ni),t(C,_o),t(_o,Pi),t(C,Mi),t(C,Eo),t(Eo,Ii),t(C,Ui),t(C,yo),t(yo,Wi),t(C,Yi),t(C,bo),t(bo,Vi),t(C,Fi),t(C,To),t(To,Bi),t(C,ji),t(C,ko),t(ko,Ri),t(C,Li),t(C,Do),t(Do,Ji),t(R,Gi),t(R,A),t(A,Co),t(Co,Rt),t(Rt,qi),t(A,Oi),t(A,Ao),t(Ao,Zi),t(A,zi),t(A,$o),t($o,Xi),t(A,Qi),t(A,So),t(So,Ki),t(A,td),t(A,xo),t(xo,ed),t(A,ad),t(A,Ho),t(Ho,od),t(A,rd),t(A,No),t(No,sd),t(A,nd),t(A,Po),t(Po,ld),f(s,nr,c),f(s,L,c),t(L,id),t(L,Lt),t(Lt,dd),t(L,hd),t(L,Jt),t(Jt,cd),t(L,fd),f(s,lr,c),f(s,oe,c),t(oe,ud),f(s,ir,c),f(s,lt,c),t(lt,wt),t(wt,Mo),Z(Gt,Mo,null),t(lt,pd),t(lt,Io),t(Io,md),f(s,dr,c),f(s,re,c),t(re,vd),f(s,hr,c),f(s,_t,c),t(_t,gd),t(_t,qt),t(qt,wd),t(_t,_d),f(s,cr,c),f(s,J,c),t(J,Ed),t(J,Ot),t(Ot,Uo),t(Uo,yd),t(J,bd),t(J,Wo),t(Wo,Td),t(J,kd),f(s,fr,c),f(s,G,c),t(G,Dd),t(G,Yo),t(Yo,Cd),t(G,Ad),t(G,Vo),t(Vo,$d),t(G,Sd),f(s,ur,c),f(s,Zt,c),t(Zt,se),f(s,pr,c),f(s,ne,c),t(ne,xd),f(s,mr,c),f(s,le,c),t(le,Hd),f(s,vr,c),f(s,Et,c),t(Et,Nd),t(Et,Fo),t(Fo,Pd),t(Et,Md),gr=!0},p:eu,i(s){gr||(z(Tt.$$.fragment,s),z(kt.$$.fragment,s),z(Dt.$$.fragment,s),z(Ct.$$.fragment,s),z(At.$$.fragment,s),z($t.$$.fragment,s),z(St.$$.fragment,s),z(xt.$$.fragment,s),z(Ht.$$.fragment,s),z(Gt.$$.fragment,s),gr=!0)},o(s){X(Tt.$$.fragment,s),X(kt.$$.fragment,s),X(Dt.$$.fragment,s),X(Ct.$$.fragment,s),X(At.$$.fragment,s),X($t.$$.fragment,s),X(St.$$.fragment,s),X(xt.$$.fragment,s),X(Ht.$$.fragment,s),X(Gt.$$.fragment,s),gr=!1},d(s){e(K),s&&e(Bo),s&&e(tt),Q(Tt),s&&e(jo),s&&e(zt),s&&e(Ro),s&&e(Xt),s&&e(Lo),s&&e(et),Q(kt),s&&e(Jo),s&&e(at),Q(Dt),s&&e(Go),s&&e(ot),Q(Ct),s&&e(qo),s&&e(Qt),s&&e(Oo),s&&e(rt),Q(At),s&&e(Zo),s&&e(ut),s&&e(zo),s&&e(Kt),s&&e(Xo),Q($t,s),s&&e(Qo),s&&e(te),s&&e(Ko),Q(St,s),s&&e(tr),s&&e(st),Q(xt),s&&e(er),s&&e(nt),Q(Ht),s&&e(ar),s&&e(ee),s&&e(or),s&&e(vt),s&&e(rr),s&&e(ae),s&&e(sr),s&&e(gt),s&&e(nr),s&&e(L),s&&e(lr),s&&e(oe),s&&e(ir),s&&e(lt),Q(Gt),s&&e(dr),s&&e(re),s&&e(hr),s&&e(_t),s&&e(cr),s&&e(J),s&&e(fr),s&&e(G),s&&e(ur),s&&e(Zt),s&&e(pr),s&&e(ne),s&&e(mr),s&&e(le),s&&e(vr),s&&e(Et)}}}const ru={local:"choosing-a-dataset",sections:[{local:"features-of-speech-datasets",sections:[{local:"1-number-of-hours",title:"1. Number of hours"},{local:"2-domain",title:"2. Domain"},{local:"3-speaking-style",title:"3. Speaking style"},{local:"4-transcription-style",title:"4. Transcription style"}],title:"Features of speech datasets"},{local:"a-summary-of-datasets-on-the-hub",title:"A summary of datasets on the Hub"},{local:"common-voice-13",title:"Common Voice 13"}],title:"Choosing a dataset"};function su(Id){return au(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class du extends zf{constructor(K){super();Xf(this,K,su,ou,Qf,{})}}export{du as default,ru as metadata};
