import{S as Nr,i as Ur,s as Dr,e as r,k as u,w as d,t as o,M as Wr,c as l,d as t,m as h,a as p,x as g,h as n,b as m,N as ra,G as a,g as i,y as v,q as w,o as y,B as b,v as Xr}from"../../chunks/vendor-hf-doc-builder.js";import{T as Zr}from"../../chunks/Tip-hf-doc-builder.js";import{I as xe}from"../../chunks/IconCopyLink-hf-doc-builder.js";import{C as ft}from"../../chunks/CodeBlock-hf-doc-builder.js";function Yr(Me){let c;return{c(){c=o(`\u{1F4A1} In practice, people use the term FFT interchangeably with DFT, as the FFT or Fast Fourier Transform is the only efficient
way to calculate the DFT on a computer.`)},l(f){c=n(f,`\u{1F4A1} In practice, people use the term FFT interchangeably with DFT, as the FFT or Fast Fourier Transform is the only efficient
way to calculate the DFT on a computer.`)},m(f,T){i(f,c,T)},d(f){f&&t(c)}}}function Rr(Me){let c;return{c(){c=o(`\u{1F4A1} Not all mel spectrograms are the same! There are two different mel scales in common use ("htk" and "slaney"),
and instead of the power spectrogram the amplitude spectrogram may be used. The conversion to a log-mel spectrogram doesn't
always compute true decibels but may simply take the \`log\`. Therefore, if a machine learning model expects a mel spectrogram
as input, double check to make sure you're computing it the same way.`)},l(f){c=n(f,`\u{1F4A1} Not all mel spectrograms are the same! There are two different mel scales in common use ("htk" and "slaney"),
and instead of the power spectrogram the amplitude spectrogram may be used. The conversion to a log-mel spectrogram doesn't
always compute true decibels but may simply take the \`log\`. Therefore, if a machine learning model expects a mel spectrogram
as input, double check to make sure you're computing it the same way.`)},m(f,T){i(f,c,T)},d(f){f&&t(c)}}}function Vr(Me){let c,f,T,C,dt,re,Cs,gt,Ps,la,Je,As,pa,k,Hs,vt,Zs,Ns,wt,Us,Ds,yt,Ws,Xs,ua,ze,Ys,ha,S,P,bt,le,Rs,Tt,Vs,ma,A,Os,kt,Qs,Ls,ca,pe,Se,on,fa,Ie,Ge,Ks,ue,ei,da,H,ti,_t,ai,si,ga,Be,ii,va,$,oi,Et,ni,ri,Fe,li,pi,wa,I,Z,qt,he,ui,$t,hi,ya,je,mi,ba,N,ci,xt,fi,di,Ta,U,gi,Mt,vi,wi,ka,Ce,yi,_a,D,bi,Pe,Ti,ki,Ea,Ae,_i,qa,G,W,Jt,me,Ei,zt,qi,$a,x,$i,St,xi,Mi,It,Ji,zi,xa,He,Si,Ma,X,Ii,Gt,Gi,Bi,Ja,ce,za,Ze,Fi,Sa,fe,Ia,_,ji,Bt,Ci,Pi,Ft,Ai,Hi,jt,Zi,Ni,Ga,de,Ba,ge,Ne,nn,Fa,Ue,Ui,ja,De,Di,Ca,B,Y,Ct,ve,Wi,Pt,Xi,Pa,M,Yi,At,Ri,Vi,Ht,Oi,Qi,Aa,R,Li,Zt,Ki,eo,Ha,we,Za,ye,We,rn,Na,Xe,to,Ua,Ye,ao,Da,V,so,Nt,io,oo,Wa,J,no,Ut,ro,lo,Dt,po,uo,Xa,O,Ya,Re,ho,Ra,F,Q,Wt,be,mo,Xt,co,Va,L,fo,Yt,go,vo,Oa,Ve,wo,Qa,Oe,yo,La,z,bo,Rt,To,ko,Vt,_o,Eo,Ka,Te,es,ke,Qe,ln,ts,Le,qo,as,K,$o,Ot,xo,Mo,ss,Ke,Jo,is,et,zo,os,tt,So,ns,j,ee,Qt,_e,Io,Lt,Go,rs,at,Bo,ls,st,Fo,ps,it,jo,us,te,Co,Kt,Po,Ao,hs,Ee,ms,qe,ot,pn,cs,E,Ho,ea,Zo,No,ta,Uo,Do,aa,Wo,Xo,fs,q,Yo,sa,Ro,Vo,ia,Oo,Qo,oa,Lo,Ko,ds,ae,gs,nt,en,vs,rt,tn,ws,lt,an,ys;return re=new xe({}),le=new xe({}),he=new xe({}),me=new xe({}),ce=new ft({props:{code:"cGlwJTIwaW5zdGFsbCUyMGxpYnJvc2E=",highlighted:"pip install librosa"}}),fe=new ft({props:{code:"aW1wb3J0JTIwbGlicm9zYSUwQSUwQWFycmF5JTJDJTIwc2FtcGxpbmdfcmF0ZSUyMCUzRCUyMGxpYnJvc2EubG9hZChsaWJyb3NhLmV4KCUyMnRydW1wZXQlMjIpKQ==",highlighted:`<span class="hljs-keyword">import</span> librosa

array, sampling_rate = librosa.load(librosa.ex(<span class="hljs-string">&quot;trumpet&quot;</span>))`}}),de=new ft({props:{code:"aW1wb3J0JTIwbWF0cGxvdGxpYi5weXBsb3QlMjBhcyUyMHBsdCUwQWltcG9ydCUyMGxpYnJvc2EuZGlzcGxheSUwQSUwQXBsdC5maWd1cmUoKS5zZXRfZmlnd2lkdGgoMTIpJTBBbGlicm9zYS5kaXNwbGF5LndhdmVzaG93KGFycmF5JTJDJTIwc3IlM0RzYW1wbGluZ19yYXRlKQ==",highlighted:`<span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt
<span class="hljs-keyword">import</span> librosa.display

plt.figure().set_figwidth(<span class="hljs-number">12</span>)
librosa.display.waveshow(array, sr=sampling_rate)`}}),ve=new xe({}),we=new ft({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBZGZ0X2lucHV0JTIwJTNEJTIwYXJyYXklNUIlM0E0MDk2JTVEJTBBJTBBJTIzJTIwY2FsY3VsYXRlJTIwdGhlJTIwREZUJTBBd2luZG93JTIwJTNEJTIwbnAuaGFubmluZyhsZW4oZGZ0X2lucHV0KSklMEF3aW5kb3dlZF9pbnB1dCUyMCUzRCUyMGRmdF9pbnB1dCUyMColMjB3aW5kb3clMEFkZnQlMjAlM0QlMjBucC5mZnQucmZmdCh3aW5kb3dlZF9pbnB1dCklMEElMEElMjMlMjBnZXQlMjB0aGUlMjBhbXBsaXR1ZGUlMjBzcGVjdHJ1bSUyMGluJTIwZGVjaWJlbHMlMEFhbXBsaXR1ZGUlMjAlM0QlMjBucC5hYnMoZGZ0KSUwQWFtcGxpdHVkZV9kYiUyMCUzRCUyMGxpYnJvc2EuYW1wbGl0dWRlX3RvX2RiKGFtcGxpdHVkZSUyQyUyMHJlZiUzRG5wLm1heCklMEElMEElMjMlMjBnZXQlMjB0aGUlMjBmcmVxdWVuY3klMjBiaW5zJTBBZnJlcXVlbmN5JTIwJTNEJTIwbGlicm9zYS5mZnRfZnJlcXVlbmNpZXMoc3IlM0RzYW1wbGluZ19yYXRlJTJDJTIwbl9mZnQlM0RsZW4oZGZ0X2lucHV0KSklMEElMEFwbHQuZmlndXJlKCkuc2V0X2ZpZ3dpZHRoKDEyKSUwQXBsdC5wbG90KGZyZXF1ZW5jeSUyQyUyMGFtcGxpdHVkZV9kYiklMEFwbHQueGxhYmVsKCUyMkZyZXF1ZW5jeSUyMChIeiklMjIpJTBBcGx0LnlsYWJlbCglMjJBbXBsaXR1ZGUlMjAoZEIpJTIyKSUwQXBsdC54c2NhbGUoJTIybG9nJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

dft_input = array[:<span class="hljs-number">4096</span>]

<span class="hljs-comment"># calculate the DFT</span>
window = np.hanning(<span class="hljs-built_in">len</span>(dft_input))
windowed_input = dft_input * window
dft = np.fft.rfft(windowed_input)

<span class="hljs-comment"># get the amplitude spectrum in decibels</span>
amplitude = np.<span class="hljs-built_in">abs</span>(dft)
amplitude_db = librosa.amplitude_to_db(amplitude, ref=np.<span class="hljs-built_in">max</span>)

<span class="hljs-comment"># get the frequency bins</span>
frequency = librosa.fft_frequencies(sr=sampling_rate, n_fft=<span class="hljs-built_in">len</span>(dft_input))

plt.figure().set_figwidth(<span class="hljs-number">12</span>)
plt.plot(frequency, amplitude_db)
plt.xlabel(<span class="hljs-string">&quot;Frequency (Hz)&quot;</span>)
plt.ylabel(<span class="hljs-string">&quot;Amplitude (dB)&quot;</span>)
plt.xscale(<span class="hljs-string">&quot;log&quot;</span>)`}}),O=new Zr({props:{$$slots:{default:[Yr]},$$scope:{ctx:Me}}}),be=new xe({}),Te=new ft({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBRCUyMCUzRCUyMGxpYnJvc2Euc3RmdChhcnJheSklMEFTX2RiJTIwJTNEJTIwbGlicm9zYS5hbXBsaXR1ZGVfdG9fZGIobnAuYWJzKEQpJTJDJTIwcmVmJTNEbnAubWF4KSUwQSUwQXBsdC5maWd1cmUoKS5zZXRfZmlnd2lkdGgoMTIpJTBBbGlicm9zYS5kaXNwbGF5LnNwZWNzaG93KFNfZGIlMkMlMjB4X2F4aXMlM0QlMjJ0aW1lJTIyJTJDJTIweV9heGlzJTNEJTIyaHolMjIpJTBBcGx0LmNvbG9yYmFyKCk=",highlighted:`<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

D = librosa.stft(array)
S_db = librosa.amplitude_to_db(np.<span class="hljs-built_in">abs</span>(D), ref=np.<span class="hljs-built_in">max</span>)

plt.figure().set_figwidth(<span class="hljs-number">12</span>)
librosa.display.specshow(S_db, x_axis=<span class="hljs-string">&quot;time&quot;</span>, y_axis=<span class="hljs-string">&quot;hz&quot;</span>)
plt.colorbar()`}}),_e=new xe({}),Ee=new ft({props:{code:"UyUyMCUzRCUyMGxpYnJvc2EuZmVhdHVyZS5tZWxzcGVjdHJvZ3JhbSh5JTNEYXJyYXklMkMlMjBzciUzRHNhbXBsaW5nX3JhdGUlMkMlMjBuX21lbHMlM0QxMjglMkMlMjBmbWF4JTNEODAwMCklMEFTX2RCJTIwJTNEJTIwbGlicm9zYS5wb3dlcl90b19kYihTJTJDJTIwcmVmJTNEbnAubWF4KSUwQSUwQXBsdC5maWd1cmUoKS5zZXRfZmlnd2lkdGgoMTIpJTBBbGlicm9zYS5kaXNwbGF5LnNwZWNzaG93KFNfZEIlMkMlMjB4X2F4aXMlM0QlMjJ0aW1lJTIyJTJDJTIweV9heGlzJTNEJTIybWVsJTIyJTJDJTIwc3IlM0RzYW1wbGluZ19yYXRlJTJDJTIwZm1heCUzRDgwMDApJTBBcGx0LmNvbG9yYmFyKCk=",highlighted:`S = librosa.feature.melspectrogram(y=array, sr=sampling_rate, n_mels=<span class="hljs-number">128</span>, fmax=<span class="hljs-number">8000</span>)
S_dB = librosa.power_to_db(S, ref=np.<span class="hljs-built_in">max</span>)

plt.figure().set_figwidth(<span class="hljs-number">12</span>)
librosa.display.specshow(S_dB, x_axis=<span class="hljs-string">&quot;time&quot;</span>, y_axis=<span class="hljs-string">&quot;mel&quot;</span>, sr=sampling_rate, fmax=<span class="hljs-number">8000</span>)
plt.colorbar()`}}),ae=new Zr({props:{$$slots:{default:[Rr]},$$scope:{ctx:Me}}}),{c(){c=r("meta"),f=u(),T=r("h1"),C=r("a"),dt=r("span"),d(re.$$.fragment),Cs=u(),gt=r("span"),Ps=o("Introduction to audio data"),la=u(),Je=r("p"),As=o(`By nature, a sound wave is a continuous signal, meaning it contains an infinite number of signal values in a given time.
This poses problems for digital devices which expect finite arrays. To be processed, stored, and transmitted by digital
devices, the continuous sound wave needs to be converted into a series of discrete values, known as a digital representation.`),pa=u(),k=r("p"),Hs=o(`If you look at any audio dataset, you\u2019ll find digital files with sound excerpts, such as text narration or music.
You may encounter different file formats such as `),vt=r("code"),Zs=o(".wav"),Ns=o(" (Waveform Audio File), "),wt=r("code"),Us=o(".flac"),Ds=o(` (Free Lossless Audio Codec)
and `),yt=r("code"),Ws=o(".mp3"),Xs=o(" (MPEG-1 Audio Layer 3). These formats mainly differ in how they compress the digital representation of the audio signal."),ua=u(),ze=r("p"),Ys=o(`Let\u2019s take a look at how we arrive from a continuous signal to this representation. The analog signal is first captured by
a microphone, which converts the sound waves into an electrical signal. The electrical signal is then digitized by an
Analog-to-Digital Converter to get the digital representation through sampling.`),ha=u(),S=r("h2"),P=r("a"),bt=r("span"),d(le.$$.fragment),Rs=u(),Tt=r("span"),Vs=o("Sampling and sampling rate"),ma=u(),A=r("p"),Os=o("Sampling is the process of measuring the value of a continuous signal at fixed time steps. The sampled waveform is "),kt=r("em"),Qs=o("discrete"),Ls=o(`,
since it contains a finite number of signal values at uniform intervals.`),ca=u(),pe=r("div"),Se=r("img"),fa=u(),Ie=r("p"),Ge=r("em"),Ks=o("Illustration from Wikipedia article: "),ue=r("a"),ei=o("Sampling (signal processing)"),da=u(),H=r("p"),ti=o("The "),_t=r("strong"),ai=o("sampling rate"),si=o(` (also called sampling frequency) is the number of samples taken in one second and is measured in
hertz (Hz). To give you a point of reference, CD-quality audio has a sampling rate of 44,100 Hz, meaning samples are taken
44,100 times per second. For comparison, high-resolution audio has a sampling rate of 192,000 Hz or 192 kHz. A common
sampling rate used in training speech models is 16,000 Hz or 16 kHz.`),ga=u(),Be=r("p"),ii=o(`The choice of sampling rate primarily determines the highest frequency that can be captured from the signal. This is also
known as the Nyquist limit and is exactly half the sampling rate. The audible frequencies in human speech are below 8 kHz
and therefore sampling speech at 16 kHz is sufficient. Using a higher sampling rate will not capture more information and
merely leads to an increase in the computational cost of processing such files. On the other hand, sampling audio at too
low a sampling rate will result in information loss. Speech sampled at 8 kHz will sound muffled, as the higher frequencies
cannot be captured at this rate.`),va=u(),$=r("p"),oi=o(`It\u2019s important to ensure that all audio examples in your dataset have the same sampling rate when working on any audio task.
If you plan to use custom audio data to fine-tune a pre-trained model, the sampling rate of your data should match the
sampling rate of the data the model was pre-trained on. The sampling rate determines the time interval between successive
audio samples, which impacts the temporal resolution of the audio data. Consider an example: a 5-second sound at a sampling
rate of 16,000 Hz will be represented as a series of 80,000 values, while the same 5-second sound at a sampling rate of
8,000 Hz will be represented as a series of 40,000 values. Transformer models that solve audio tasks treat examples as
sequences and rely on attention mechanisms to learn audio or multimodal representation. Since sequences are different for
audio examples at different sampling rates, it will be challenging for models to generalize between sampling rates.
`),Et=r("strong"),ni=o("Resampling"),ri=o(" is the process of making the sampling rates match, and is part of "),Fe=r("a"),li=o("preprocessing"),pi=o(" the audio data."),wa=u(),I=r("h2"),Z=r("a"),qt=r("span"),d(he.$$.fragment),ui=u(),$t=r("span"),hi=o("Amplitude and bit depth"),ya=u(),je=r("p"),mi=o("While the sampling rate tells you how often the samples are taken, what exactly are the values in each sample?"),ba=u(),N=r("p"),ci=o("Sound is made by changes in air pressure at frequencies that are audible to humans. The "),xt=r("strong"),fi=o("amplitude"),di=o(` of a sound describes
the sound pressure level at any given instant and is measured in decibels (dB). We perceive the amplitude as loudness.
To give you an example, a normal speaking voice is under 60 dB, and a rock concert can be at around 125 dB, pushing the
limits of human hearing.`),Ta=u(),U=r("p"),gi=o("In digital audio, each audio sample records the amplitude of the audio wave at a point in time. The "),Mt=r("strong"),vi=o("bit depth"),wi=o(` of the
sample determines with how much precision this amplitude value can be described. The higher the bit depth, the more
faithfully the digital representation approximates the original continuous sound wave.`),ka=u(),Ce=r("p"),yi=o(`The most common audio bit depths are 16-bit and 24-bit. Each is a binary term, representing the number of possible steps
to which the amplitude value can be quantized when it\u2019s converted from continuous to discrete: 65,536 steps for 16-bit audio,
a whopping 16,777,216 steps for 24-bit audio. Because quantizing involves rounding off the continuous value to a discrete
value, the sampling process introduces noise. The higher the bit depth, the smaller this quantization noise. In practice,
the quantization noise of 16-bit audio is already small enough to be inaudible, and using higher bit depths is generally
not necessary.`),_a=u(),D=r("p"),bi=o(`You may also come across 32-bit audio. This stores the samples as floating-point values, whereas 16-bit and 24-bit audio
use integer samples. The precision of a 32-bit floating-point value is 24 bits, giving it the same bit depth as 24-bit audio.
Floating-point audio samples are expected to lie within the [-1.0, 1.0] range. Since machine learning models naturally
work on floating-point data, the audio must first be converted into floating-point format before it can be used to train
the model. We\u2019ll see how to do this in the next section on `),Pe=r("a"),Ti=o("Preprocessing"),ki=o("."),Ea=u(),Ae=r("p"),_i=o(`Just as with continuous audio signals, the amplitude of digital audio is typically expressed in decibels (dB). Since
human hearing is logarithmic in nature \u2014 our ears are more sensitive to small fluctuations in quiet sounds than in loud
sounds \u2014 the loudness of a sound is easier to interpret if the amplitudes are in decibels, which are also logarithmic.
The decibel scale for real-world audio starts at 0 dB, which represents the quietest possible sound humans can hear, and
louder sounds have larger values. However, for digital audio signals, 0 dB is the loudest possible amplitude, while all
other amplitudes are negative. As a quick rule of thumb: every -6 dB is a halving of the amplitude, and anything below -60 dB
is generally inaudible unless you really crank up the volume.`),qa=u(),G=r("h2"),W=r("a"),Jt=r("span"),d(me.$$.fragment),Ei=u(),zt=r("span"),qi=o("Audio as a waveform"),$a=u(),x=r("p"),$i=o("You may have seen sounds visualized as a "),St=r("strong"),xi=o("waveform"),Mi=o(`, which plots the sample values over time and illustrates the changes
in the sound\u2019s amplitude. This is also known as the `),It=r("em"),Ji=o("time domain"),zi=o(" representation of sound."),xa=u(),He=r("p"),Si=o(`This type of visualization is useful for identifying specific features of the audio signal such as the timing of individual
sound events, the overall loudness of the signal, and any irregularities or noise present in the audio.`),Ma=u(),X=r("p"),Ii=o("To plot the waveform for an audio signal, we can use a Python library called "),Gt=r("code"),Gi=o("librosa"),Bi=o(":"),Ja=u(),d(ce.$$.fragment),za=u(),Ze=r("p"),Fi=o("Let\u2019s take an example sound called \u201Ctrumpet\u201D that comes with the library:"),Sa=u(),d(fe.$$.fragment),Ia=u(),_=r("p"),ji=o("The example is loaded as a tuple of audio time series (here we call it "),Bt=r("code"),Ci=o("array"),Pi=o("), and sampling rate ("),Ft=r("code"),Ai=o("sampling_rate"),Hi=o(`).
Let\u2019s take a look at this sound\u2019s waveform by using librosa\u2019s `),jt=r("code"),Zi=o("waveshow()"),Ni=o(" function:"),Ga=u(),d(de.$$.fragment),Ba=u(),ge=r("div"),Ne=r("img"),Fa=u(),Ue=r("p"),Ui=o(`This plots the amplitude of the signal on the y-axis and time along the x-axis. In other words, each point corresponds
to a single sample value that was taken when this sound was sampled. Also note that librosa returns the audio as
floating-point values already, and that the amplitude values are indeed within the [-1.0, 1.0] range.`),ja=u(),De=r("p"),Di=o(`Visualizing the audio along with listening to it can be a useful tool for understanding the data you are working with.
You can see the shape of the signal, observe patterns, learn to spot noise or distortion. If you preprocess data in some
ways, such as normalization, resampling, or filtering, you can visually confirm that preprocessing steps have been applied as expected.
After training a model, you can also visualize samples where errors occur (e.g. in audio classification task) to debug
the issue.`),Ca=u(),B=r("h2"),Y=r("a"),Ct=r("span"),d(ve.$$.fragment),Wi=u(),Pt=r("span"),Xi=o("The frequency spectrum"),Pa=u(),M=r("p"),Yi=o("Another way to visualize audio data is to plot the "),At=r("strong"),Ri=o("frequency spectrum"),Vi=o(" of an audio signal, also known as the "),Ht=r("em"),Oi=o("frequency domain"),Qi=o(`
representation. The spectrum is computed using the discrete Fourier transform or DFT. It describes the individual frequencies
that make up the signal and how strong they are.`),Aa=u(),R=r("p"),Li=o("Let\u2019s plot the frequency spectrum for the same trumpet sound by taking the DFT using numpy\u2019s "),Zt=r("code"),Ki=o("rfft()"),eo=o(` function. While it
is possible to plot the spectrum of the entire sound, it\u2019s more useful to look at a small region instead. Here we\u2019ll take
the DFT over the first 4096 samples, which is roughly the length of the first note being played:`),Ha=u(),d(we.$$.fragment),Za=u(),ye=r("div"),We=r("img"),Na=u(),Xe=r("p"),to=o(`This plots the strength of the various frequency components that are present in this audio segment. The frequency values are on
the x-axis, usually plotted on a logarithmic scale, while their amplitudes are on the y-axis.`),Ua=u(),Ye=r("p"),ao=o(`The frequency spectrum that we plotted shows several peaks. These peaks correspond to the harmonics of the note that\u2019s
being played, with the higher harmonics being quieter. Since the first peak is at around 620 Hz, this is the frequency spectrum of an E\u266D note.`),Da=u(),V=r("p"),so=o(`The output of the DFT is an array of complex numbers, made up of real and imaginary components. Taking
the magnitude with `),Nt=r("code"),io=o("np.abs(dft)"),oo=o(` extracts the amplitude information from the spectrogram. The angle between the real and
imaginary components provides the so-called phase spectrum, but this is often discarded in machine learning applications.`),Wa=u(),J=r("p"),no=o("You used "),Ut=r("code"),ro=o("librosa.amplitude_to_db()"),lo=o(` to convert the amplitude values to the decibel scale, making it easier to see
the finer details in the spectrum. Sometimes people use the `),Dt=r("strong"),po=o("power spectrum"),uo=o(`, which measures energy rather than amplitude;
this is simply a spectrum with the amplitude values squared.`),Xa=u(),d(O.$$.fragment),Ya=u(),Re=r("p"),ho=o(`The frequency spectrum of an audio signal contains the exact same information as its waveform \u2014 they are simply two different
ways of looking at the same data (here, the first 4096 samples from the trumpet sound). Where the waveform plots the amplitude
of the audio signal over time, the spectrum visualizes the amplitudes of the individual frequencies at a fixed point in time.`),Ra=u(),F=r("h2"),Q=r("a"),Wt=r("span"),d(be.$$.fragment),mo=u(),Xt=r("span"),co=o("Spectrogram"),Va=u(),L=r("p"),fo=o(`What if we want to see how the frequencies in an audio signal change? The trumpet plays several notes and they all have
different frequencies. The problem is that the spectrum only shows a frozen snapshot of the frequencies at a given instant.
The solution is to take multiple DFTs, each covering only a small slice of time, and stack the resulting spectra together
into a `),Yt=r("strong"),go=o("spectrogram"),vo=o("."),Oa=u(),Ve=r("p"),wo=o(`A spectrogram plots the frequency content of an audio signal as it changes over time. It allows you to see time, frequency,
and amplitude all on one graph. The algorithm that performs this computation is the STFT or Short Time Fourier Transform.`),Qa=u(),Oe=r("p"),yo=o(`The spectrogram is one of the most informative audio tools available to you. For example, when working with a music recording,
you can see the various instruments and vocal tracks and how they contribute to the overall sound. In speech, you can
identify different vowel sounds as each vowel is characterized by particular frequencies.`),La=u(),z=r("p"),bo=o("Let\u2019s plot a spectrogram for the same trumpet sound, using librosa\u2019s "),Rt=r("code"),To=o("stft()"),ko=o(" and "),Vt=r("code"),_o=o("specshow()"),Eo=o(" functions:"),Ka=u(),d(Te.$$.fragment),es=u(),ke=r("div"),Qe=r("img"),ts=u(),Le=r("p"),qo=o(`In this plot, the x-axis represents time as in the waveform visualization but now the y-axis represents frequency in Hz.
The intensity of the color gives the amplitude or power of the frequency component at each point in time, measured in decibels (dB).`),as=u(),K=r("p"),$o=o(`The spectrogram is created by taking short segments of the audio signal, typically lasting a few milliseconds, and calculating
the discrete Fourier transform of each segment to obtain its frequency spectrum. The resulting spectra are then stacked
together on the time axis to create the spectrogram. Each vertical slice in this image corresponds to a single frequency
spectrum, seen from the top. By default, `),Ot=r("code"),xo=o("librosa.stft()"),Mo=o(` splits the audio signal into segments of 2048 samples, which
gives a good trade-off between frequency resolution and time resolution.`),ss=u(),Ke=r("p"),Jo=o(`Since the spectrogram and the waveform are different views of the same data, it\u2019s possible to turn the spectrogram back
into the original waveform using the inverse STFT. However, this requires the phase information in addition to the amplitude
information. If the spectrogram was generated by a machine learning model, it typically only outputs the amplitudes. In
that case, we can use a phase reconstruction algorithm such as the classic Griffin-Lim algorithm, or using a neural network
called a vocoder, to reconstruct a waveform from the spectrogram.`),is=u(),et=r("p"),zo=o(`Spectrograms aren\u2019t just used for visualization. Many machine learning models will take spectrograms as input \u2014 as opposed
to waveforms \u2014 and produce spectrograms as output.`),os=u(),tt=r("p"),So=o("Now that we know what a spectrogram is and how it\u2019s made, let\u2019s take a look at a variant of it widely used for speech processing: the mel spectrogram."),ns=u(),j=r("h2"),ee=r("a"),Qt=r("span"),d(_e.$$.fragment),Io=u(),Lt=r("span"),Go=o("Mel spectrogram"),rs=u(),at=r("p"),Bo=o(`A mel spectrogram is a variation of the spectrogram that is commonly used in speech processing and machine learning tasks.
It is similar to a spectrogram in that it shows the frequency content of an audio signal over time, but on a different frequency axis.`),ls=u(),st=r("p"),Fo=o(`In a standard spectrogram, the frequency axis is linear and is measured in hertz (Hz). However, the human auditory system
is more sensitive to changes in lower frequencies than higher frequencies, and this sensitivity decreases logarithmically
as frequency increases. The mel scale is a perceptual scale that approximates the non-linear frequency response of the human ear.`),ps=u(),it=r("p"),jo=o(`To create a mel spectrogram, the STFT is used just like before, splitting the audio into short segments to obtain a sequence
of frequency spectra. Additionally, each spectrum is sent through a set of filters, the so-called mel filterbank, to
transform the frequencies to the mel scale.`),us=u(),te=r("p"),Co=o("Let\u2019s see how we can plot a mel spectrogram using librosa\u2019s "),Kt=r("code"),Po=o("melspectrogram()"),Ao=o(" function, which performs all of those steps for us:"),hs=u(),d(Ee.$$.fragment),ms=u(),qe=r("div"),ot=r("img"),cs=u(),E=r("p"),Ho=o("In the example above, "),ea=r("code"),Zo=o("n_mels"),No=o(` stands for the number of mel bands to generate. The mel bands define a set of frequency
ranges that divide the spectrum into perceptually meaningful components, using a set of filters whose shape and spacing
are chosen to mimic the way the human ear responds to different frequencies. Common values for `),ta=r("code"),Uo=o("n_mels"),Do=o(" are 40 or 80. "),aa=r("code"),Wo=o("fmax"),Xo=o(`
indicates the highest frequency (in Hz) we care about.`),fs=u(),q=r("p"),Yo=o(`Just as with a regular spectrogram, it\u2019s common practice to express the strength of the mel frequency components in
decibels. This is commonly referred to as a `),sa=r("strong"),Ro=o("log-mel spectrogram"),Vo=o(`, because the conversion to decibels involves a
logarithmic operation. The above example used `),ia=r("code"),Oo=o("librosa.power_to_db()"),Qo=o(" as "),oa=r("code"),Lo=o("librosa.feature.melspectrogram()"),Ko=o(" creates a power spectrogram."),ds=u(),d(ae.$$.fragment),gs=u(),nt=r("p"),en=o(`Creating a mel spectrogram is a lossy operation as it involves filtering the signal. Converting a mel spectrogram back
into a waveform is more difficult than doing this for a regular spectrogram, as it requires estimating the frequencies
that were thrown away. This is why machine learning models such as HiFiGAN vocoder are needed to produce a waveform from a mel
spectrogram.`),vs=u(),rt=r("p"),tn=o(`Compared to a standard spectrogram, a mel spectrogram can capture more meaningful features of the audio signal for
human perception, making it a popular choice in tasks such as speech recognition, speaker identification, and music genre classification.`),ws=u(),lt=r("p"),an=o("Now that you know how to visualize audio data examples, go ahead and try to see what your favorite sounds look like. :)"),this.h()},l(e){const s=Wr('[data-svelte="svelte-1phssyn"]',document.head);c=l(s,"META",{name:!0,content:!0}),s.forEach(t),f=h(e),T=l(e,"H1",{class:!0});var $e=p(T);C=l($e,"A",{id:!0,class:!0,href:!0});var na=p(C);dt=l(na,"SPAN",{});var un=p(dt);g(re.$$.fragment,un),un.forEach(t),na.forEach(t),Cs=h($e),gt=l($e,"SPAN",{});var hn=p(gt);Ps=n(hn,"Introduction to audio data"),hn.forEach(t),$e.forEach(t),la=h(e),Je=l(e,"P",{});var mn=p(Je);As=n(mn,`By nature, a sound wave is a continuous signal, meaning it contains an infinite number of signal values in a given time.
This poses problems for digital devices which expect finite arrays. To be processed, stored, and transmitted by digital
devices, the continuous sound wave needs to be converted into a series of discrete values, known as a digital representation.`),mn.forEach(t),pa=h(e),k=l(e,"P",{});var se=p(k);Hs=n(se,`If you look at any audio dataset, you\u2019ll find digital files with sound excerpts, such as text narration or music.
You may encounter different file formats such as `),vt=l(se,"CODE",{});var cn=p(vt);Zs=n(cn,".wav"),cn.forEach(t),Ns=n(se," (Waveform Audio File), "),wt=l(se,"CODE",{});var fn=p(wt);Us=n(fn,".flac"),fn.forEach(t),Ds=n(se,` (Free Lossless Audio Codec)
and `),yt=l(se,"CODE",{});var dn=p(yt);Ws=n(dn,".mp3"),dn.forEach(t),Xs=n(se," (MPEG-1 Audio Layer 3). These formats mainly differ in how they compress the digital representation of the audio signal."),se.forEach(t),ua=h(e),ze=l(e,"P",{});var gn=p(ze);Ys=n(gn,`Let\u2019s take a look at how we arrive from a continuous signal to this representation. The analog signal is first captured by
a microphone, which converts the sound waves into an electrical signal. The electrical signal is then digitized by an
Analog-to-Digital Converter to get the digital representation through sampling.`),gn.forEach(t),ha=h(e),S=l(e,"H2",{class:!0});var bs=p(S);P=l(bs,"A",{id:!0,class:!0,href:!0});var vn=p(P);bt=l(vn,"SPAN",{});var wn=p(bt);g(le.$$.fragment,wn),wn.forEach(t),vn.forEach(t),Rs=h(bs),Tt=l(bs,"SPAN",{});var yn=p(Tt);Vs=n(yn,"Sampling and sampling rate"),yn.forEach(t),bs.forEach(t),ma=h(e),A=l(e,"P",{});var Ts=p(A);Os=n(Ts,"Sampling is the process of measuring the value of a continuous signal at fixed time steps. The sampled waveform is "),kt=l(Ts,"EM",{});var bn=p(kt);Qs=n(bn,"discrete"),bn.forEach(t),Ls=n(Ts,`,
since it contains a finite number of signal values at uniform intervals.`),Ts.forEach(t),ca=h(e),pe=l(e,"DIV",{class:!0});var Tn=p(pe);Se=l(Tn,"IMG",{src:!0,alt:!0}),Tn.forEach(t),fa=h(e),Ie=l(e,"P",{});var kn=p(Ie);Ge=l(kn,"EM",{});var sn=p(Ge);Ks=n(sn,"Illustration from Wikipedia article: "),ue=l(sn,"A",{href:!0,rel:!0});var _n=p(ue);ei=n(_n,"Sampling (signal processing)"),_n.forEach(t),sn.forEach(t),kn.forEach(t),da=h(e),H=l(e,"P",{});var ks=p(H);ti=n(ks,"The "),_t=l(ks,"STRONG",{});var En=p(_t);ai=n(En,"sampling rate"),En.forEach(t),si=n(ks,` (also called sampling frequency) is the number of samples taken in one second and is measured in
hertz (Hz). To give you a point of reference, CD-quality audio has a sampling rate of 44,100 Hz, meaning samples are taken
44,100 times per second. For comparison, high-resolution audio has a sampling rate of 192,000 Hz or 192 kHz. A common
sampling rate used in training speech models is 16,000 Hz or 16 kHz.`),ks.forEach(t),ga=h(e),Be=l(e,"P",{});var qn=p(Be);ii=n(qn,`The choice of sampling rate primarily determines the highest frequency that can be captured from the signal. This is also
known as the Nyquist limit and is exactly half the sampling rate. The audible frequencies in human speech are below 8 kHz
and therefore sampling speech at 16 kHz is sufficient. Using a higher sampling rate will not capture more information and
merely leads to an increase in the computational cost of processing such files. On the other hand, sampling audio at too
low a sampling rate will result in information loss. Speech sampled at 8 kHz will sound muffled, as the higher frequencies
cannot be captured at this rate.`),qn.forEach(t),va=h(e),$=l(e,"P",{});var pt=p($);oi=n(pt,`It\u2019s important to ensure that all audio examples in your dataset have the same sampling rate when working on any audio task.
If you plan to use custom audio data to fine-tune a pre-trained model, the sampling rate of your data should match the
sampling rate of the data the model was pre-trained on. The sampling rate determines the time interval between successive
audio samples, which impacts the temporal resolution of the audio data. Consider an example: a 5-second sound at a sampling
rate of 16,000 Hz will be represented as a series of 80,000 values, while the same 5-second sound at a sampling rate of
8,000 Hz will be represented as a series of 40,000 values. Transformer models that solve audio tasks treat examples as
sequences and rely on attention mechanisms to learn audio or multimodal representation. Since sequences are different for
audio examples at different sampling rates, it will be challenging for models to generalize between sampling rates.
`),Et=l(pt,"STRONG",{});var $n=p(Et);ni=n($n,"Resampling"),$n.forEach(t),ri=n(pt," is the process of making the sampling rates match, and is part of "),Fe=l(pt,"A",{href:!0});var xn=p(Fe);li=n(xn,"preprocessing"),xn.forEach(t),pi=n(pt," the audio data."),pt.forEach(t),wa=h(e),I=l(e,"H2",{class:!0});var _s=p(I);Z=l(_s,"A",{id:!0,class:!0,href:!0});var Mn=p(Z);qt=l(Mn,"SPAN",{});var Jn=p(qt);g(he.$$.fragment,Jn),Jn.forEach(t),Mn.forEach(t),ui=h(_s),$t=l(_s,"SPAN",{});var zn=p($t);hi=n(zn,"Amplitude and bit depth"),zn.forEach(t),_s.forEach(t),ya=h(e),je=l(e,"P",{});var Sn=p(je);mi=n(Sn,"While the sampling rate tells you how often the samples are taken, what exactly are the values in each sample?"),Sn.forEach(t),ba=h(e),N=l(e,"P",{});var Es=p(N);ci=n(Es,"Sound is made by changes in air pressure at frequencies that are audible to humans. The "),xt=l(Es,"STRONG",{});var In=p(xt);fi=n(In,"amplitude"),In.forEach(t),di=n(Es,` of a sound describes
the sound pressure level at any given instant and is measured in decibels (dB). We perceive the amplitude as loudness.
To give you an example, a normal speaking voice is under 60 dB, and a rock concert can be at around 125 dB, pushing the
limits of human hearing.`),Es.forEach(t),Ta=h(e),U=l(e,"P",{});var qs=p(U);gi=n(qs,"In digital audio, each audio sample records the amplitude of the audio wave at a point in time. The "),Mt=l(qs,"STRONG",{});var Gn=p(Mt);vi=n(Gn,"bit depth"),Gn.forEach(t),wi=n(qs,` of the
sample determines with how much precision this amplitude value can be described. The higher the bit depth, the more
faithfully the digital representation approximates the original continuous sound wave.`),qs.forEach(t),ka=h(e),Ce=l(e,"P",{});var Bn=p(Ce);yi=n(Bn,`The most common audio bit depths are 16-bit and 24-bit. Each is a binary term, representing the number of possible steps
to which the amplitude value can be quantized when it\u2019s converted from continuous to discrete: 65,536 steps for 16-bit audio,
a whopping 16,777,216 steps for 24-bit audio. Because quantizing involves rounding off the continuous value to a discrete
value, the sampling process introduces noise. The higher the bit depth, the smaller this quantization noise. In practice,
the quantization noise of 16-bit audio is already small enough to be inaudible, and using higher bit depths is generally
not necessary.`),Bn.forEach(t),_a=h(e),D=l(e,"P",{});var $s=p(D);bi=n($s,`You may also come across 32-bit audio. This stores the samples as floating-point values, whereas 16-bit and 24-bit audio
use integer samples. The precision of a 32-bit floating-point value is 24 bits, giving it the same bit depth as 24-bit audio.
Floating-point audio samples are expected to lie within the [-1.0, 1.0] range. Since machine learning models naturally
work on floating-point data, the audio must first be converted into floating-point format before it can be used to train
the model. We\u2019ll see how to do this in the next section on `),Pe=l($s,"A",{href:!0});var Fn=p(Pe);Ti=n(Fn,"Preprocessing"),Fn.forEach(t),ki=n($s,"."),$s.forEach(t),Ea=h(e),Ae=l(e,"P",{});var jn=p(Ae);_i=n(jn,`Just as with continuous audio signals, the amplitude of digital audio is typically expressed in decibels (dB). Since
human hearing is logarithmic in nature \u2014 our ears are more sensitive to small fluctuations in quiet sounds than in loud
sounds \u2014 the loudness of a sound is easier to interpret if the amplitudes are in decibels, which are also logarithmic.
The decibel scale for real-world audio starts at 0 dB, which represents the quietest possible sound humans can hear, and
louder sounds have larger values. However, for digital audio signals, 0 dB is the loudest possible amplitude, while all
other amplitudes are negative. As a quick rule of thumb: every -6 dB is a halving of the amplitude, and anything below -60 dB
is generally inaudible unless you really crank up the volume.`),jn.forEach(t),qa=h(e),G=l(e,"H2",{class:!0});var xs=p(G);W=l(xs,"A",{id:!0,class:!0,href:!0});var Cn=p(W);Jt=l(Cn,"SPAN",{});var Pn=p(Jt);g(me.$$.fragment,Pn),Pn.forEach(t),Cn.forEach(t),Ei=h(xs),zt=l(xs,"SPAN",{});var An=p(zt);qi=n(An,"Audio as a waveform"),An.forEach(t),xs.forEach(t),$a=h(e),x=l(e,"P",{});var ut=p(x);$i=n(ut,"You may have seen sounds visualized as a "),St=l(ut,"STRONG",{});var Hn=p(St);xi=n(Hn,"waveform"),Hn.forEach(t),Mi=n(ut,`, which plots the sample values over time and illustrates the changes
in the sound\u2019s amplitude. This is also known as the `),It=l(ut,"EM",{});var Zn=p(It);Ji=n(Zn,"time domain"),Zn.forEach(t),zi=n(ut," representation of sound."),ut.forEach(t),xa=h(e),He=l(e,"P",{});var Nn=p(He);Si=n(Nn,`This type of visualization is useful for identifying specific features of the audio signal such as the timing of individual
sound events, the overall loudness of the signal, and any irregularities or noise present in the audio.`),Nn.forEach(t),Ma=h(e),X=l(e,"P",{});var Ms=p(X);Ii=n(Ms,"To plot the waveform for an audio signal, we can use a Python library called "),Gt=l(Ms,"CODE",{});var Un=p(Gt);Gi=n(Un,"librosa"),Un.forEach(t),Bi=n(Ms,":"),Ms.forEach(t),Ja=h(e),g(ce.$$.fragment,e),za=h(e),Ze=l(e,"P",{});var Dn=p(Ze);Fi=n(Dn,"Let\u2019s take an example sound called \u201Ctrumpet\u201D that comes with the library:"),Dn.forEach(t),Sa=h(e),g(fe.$$.fragment,e),Ia=h(e),_=l(e,"P",{});var ie=p(_);ji=n(ie,"The example is loaded as a tuple of audio time series (here we call it "),Bt=l(ie,"CODE",{});var Wn=p(Bt);Ci=n(Wn,"array"),Wn.forEach(t),Pi=n(ie,"), and sampling rate ("),Ft=l(ie,"CODE",{});var Xn=p(Ft);Ai=n(Xn,"sampling_rate"),Xn.forEach(t),Hi=n(ie,`).
Let\u2019s take a look at this sound\u2019s waveform by using librosa\u2019s `),jt=l(ie,"CODE",{});var Yn=p(jt);Zi=n(Yn,"waveshow()"),Yn.forEach(t),Ni=n(ie," function:"),ie.forEach(t),Ga=h(e),g(de.$$.fragment,e),Ba=h(e),ge=l(e,"DIV",{class:!0});var Rn=p(ge);Ne=l(Rn,"IMG",{src:!0,alt:!0}),Rn.forEach(t),Fa=h(e),Ue=l(e,"P",{});var Vn=p(Ue);Ui=n(Vn,`This plots the amplitude of the signal on the y-axis and time along the x-axis. In other words, each point corresponds
to a single sample value that was taken when this sound was sampled. Also note that librosa returns the audio as
floating-point values already, and that the amplitude values are indeed within the [-1.0, 1.0] range.`),Vn.forEach(t),ja=h(e),De=l(e,"P",{});var On=p(De);Di=n(On,`Visualizing the audio along with listening to it can be a useful tool for understanding the data you are working with.
You can see the shape of the signal, observe patterns, learn to spot noise or distortion. If you preprocess data in some
ways, such as normalization, resampling, or filtering, you can visually confirm that preprocessing steps have been applied as expected.
After training a model, you can also visualize samples where errors occur (e.g. in audio classification task) to debug
the issue.`),On.forEach(t),Ca=h(e),B=l(e,"H2",{class:!0});var Js=p(B);Y=l(Js,"A",{id:!0,class:!0,href:!0});var Qn=p(Y);Ct=l(Qn,"SPAN",{});var Ln=p(Ct);g(ve.$$.fragment,Ln),Ln.forEach(t),Qn.forEach(t),Wi=h(Js),Pt=l(Js,"SPAN",{});var Kn=p(Pt);Xi=n(Kn,"The frequency spectrum"),Kn.forEach(t),Js.forEach(t),Pa=h(e),M=l(e,"P",{});var ht=p(M);Yi=n(ht,"Another way to visualize audio data is to plot the "),At=l(ht,"STRONG",{});var er=p(At);Ri=n(er,"frequency spectrum"),er.forEach(t),Vi=n(ht," of an audio signal, also known as the "),Ht=l(ht,"EM",{});var tr=p(Ht);Oi=n(tr,"frequency domain"),tr.forEach(t),Qi=n(ht,`
representation. The spectrum is computed using the discrete Fourier transform or DFT. It describes the individual frequencies
that make up the signal and how strong they are.`),ht.forEach(t),Aa=h(e),R=l(e,"P",{});var zs=p(R);Li=n(zs,"Let\u2019s plot the frequency spectrum for the same trumpet sound by taking the DFT using numpy\u2019s "),Zt=l(zs,"CODE",{});var ar=p(Zt);Ki=n(ar,"rfft()"),ar.forEach(t),eo=n(zs,` function. While it
is possible to plot the spectrum of the entire sound, it\u2019s more useful to look at a small region instead. Here we\u2019ll take
the DFT over the first 4096 samples, which is roughly the length of the first note being played:`),zs.forEach(t),Ha=h(e),g(we.$$.fragment,e),Za=h(e),ye=l(e,"DIV",{class:!0});var sr=p(ye);We=l(sr,"IMG",{src:!0,alt:!0}),sr.forEach(t),Na=h(e),Xe=l(e,"P",{});var ir=p(Xe);to=n(ir,`This plots the strength of the various frequency components that are present in this audio segment. The frequency values are on
the x-axis, usually plotted on a logarithmic scale, while their amplitudes are on the y-axis.`),ir.forEach(t),Ua=h(e),Ye=l(e,"P",{});var or=p(Ye);ao=n(or,`The frequency spectrum that we plotted shows several peaks. These peaks correspond to the harmonics of the note that\u2019s
being played, with the higher harmonics being quieter. Since the first peak is at around 620 Hz, this is the frequency spectrum of an E\u266D note.`),or.forEach(t),Da=h(e),V=l(e,"P",{});var Ss=p(V);so=n(Ss,`The output of the DFT is an array of complex numbers, made up of real and imaginary components. Taking
the magnitude with `),Nt=l(Ss,"CODE",{});var nr=p(Nt);io=n(nr,"np.abs(dft)"),nr.forEach(t),oo=n(Ss,` extracts the amplitude information from the spectrogram. The angle between the real and
imaginary components provides the so-called phase spectrum, but this is often discarded in machine learning applications.`),Ss.forEach(t),Wa=h(e),J=l(e,"P",{});var mt=p(J);no=n(mt,"You used "),Ut=l(mt,"CODE",{});var rr=p(Ut);ro=n(rr,"librosa.amplitude_to_db()"),rr.forEach(t),lo=n(mt,` to convert the amplitude values to the decibel scale, making it easier to see
the finer details in the spectrum. Sometimes people use the `),Dt=l(mt,"STRONG",{});var lr=p(Dt);po=n(lr,"power spectrum"),lr.forEach(t),uo=n(mt,`, which measures energy rather than amplitude;
this is simply a spectrum with the amplitude values squared.`),mt.forEach(t),Xa=h(e),g(O.$$.fragment,e),Ya=h(e),Re=l(e,"P",{});var pr=p(Re);ho=n(pr,`The frequency spectrum of an audio signal contains the exact same information as its waveform \u2014 they are simply two different
ways of looking at the same data (here, the first 4096 samples from the trumpet sound). Where the waveform plots the amplitude
of the audio signal over time, the spectrum visualizes the amplitudes of the individual frequencies at a fixed point in time.`),pr.forEach(t),Ra=h(e),F=l(e,"H2",{class:!0});var Is=p(F);Q=l(Is,"A",{id:!0,class:!0,href:!0});var ur=p(Q);Wt=l(ur,"SPAN",{});var hr=p(Wt);g(be.$$.fragment,hr),hr.forEach(t),ur.forEach(t),mo=h(Is),Xt=l(Is,"SPAN",{});var mr=p(Xt);co=n(mr,"Spectrogram"),mr.forEach(t),Is.forEach(t),Va=h(e),L=l(e,"P",{});var Gs=p(L);fo=n(Gs,`What if we want to see how the frequencies in an audio signal change? The trumpet plays several notes and they all have
different frequencies. The problem is that the spectrum only shows a frozen snapshot of the frequencies at a given instant.
The solution is to take multiple DFTs, each covering only a small slice of time, and stack the resulting spectra together
into a `),Yt=l(Gs,"STRONG",{});var cr=p(Yt);go=n(cr,"spectrogram"),cr.forEach(t),vo=n(Gs,"."),Gs.forEach(t),Oa=h(e),Ve=l(e,"P",{});var fr=p(Ve);wo=n(fr,`A spectrogram plots the frequency content of an audio signal as it changes over time. It allows you to see time, frequency,
and amplitude all on one graph. The algorithm that performs this computation is the STFT or Short Time Fourier Transform.`),fr.forEach(t),Qa=h(e),Oe=l(e,"P",{});var dr=p(Oe);yo=n(dr,`The spectrogram is one of the most informative audio tools available to you. For example, when working with a music recording,
you can see the various instruments and vocal tracks and how they contribute to the overall sound. In speech, you can
identify different vowel sounds as each vowel is characterized by particular frequencies.`),dr.forEach(t),La=h(e),z=l(e,"P",{});var ct=p(z);bo=n(ct,"Let\u2019s plot a spectrogram for the same trumpet sound, using librosa\u2019s "),Rt=l(ct,"CODE",{});var gr=p(Rt);To=n(gr,"stft()"),gr.forEach(t),ko=n(ct," and "),Vt=l(ct,"CODE",{});var vr=p(Vt);_o=n(vr,"specshow()"),vr.forEach(t),Eo=n(ct," functions:"),ct.forEach(t),Ka=h(e),g(Te.$$.fragment,e),es=h(e),ke=l(e,"DIV",{class:!0});var wr=p(ke);Qe=l(wr,"IMG",{src:!0,alt:!0}),wr.forEach(t),ts=h(e),Le=l(e,"P",{});var yr=p(Le);qo=n(yr,`In this plot, the x-axis represents time as in the waveform visualization but now the y-axis represents frequency in Hz.
The intensity of the color gives the amplitude or power of the frequency component at each point in time, measured in decibels (dB).`),yr.forEach(t),as=h(e),K=l(e,"P",{});var Bs=p(K);$o=n(Bs,`The spectrogram is created by taking short segments of the audio signal, typically lasting a few milliseconds, and calculating
the discrete Fourier transform of each segment to obtain its frequency spectrum. The resulting spectra are then stacked
together on the time axis to create the spectrogram. Each vertical slice in this image corresponds to a single frequency
spectrum, seen from the top. By default, `),Ot=l(Bs,"CODE",{});var br=p(Ot);xo=n(br,"librosa.stft()"),br.forEach(t),Mo=n(Bs,` splits the audio signal into segments of 2048 samples, which
gives a good trade-off between frequency resolution and time resolution.`),Bs.forEach(t),ss=h(e),Ke=l(e,"P",{});var Tr=p(Ke);Jo=n(Tr,`Since the spectrogram and the waveform are different views of the same data, it\u2019s possible to turn the spectrogram back
into the original waveform using the inverse STFT. However, this requires the phase information in addition to the amplitude
information. If the spectrogram was generated by a machine learning model, it typically only outputs the amplitudes. In
that case, we can use a phase reconstruction algorithm such as the classic Griffin-Lim algorithm, or using a neural network
called a vocoder, to reconstruct a waveform from the spectrogram.`),Tr.forEach(t),is=h(e),et=l(e,"P",{});var kr=p(et);zo=n(kr,`Spectrograms aren\u2019t just used for visualization. Many machine learning models will take spectrograms as input \u2014 as opposed
to waveforms \u2014 and produce spectrograms as output.`),kr.forEach(t),os=h(e),tt=l(e,"P",{});var _r=p(tt);So=n(_r,"Now that we know what a spectrogram is and how it\u2019s made, let\u2019s take a look at a variant of it widely used for speech processing: the mel spectrogram."),_r.forEach(t),ns=h(e),j=l(e,"H2",{class:!0});var Fs=p(j);ee=l(Fs,"A",{id:!0,class:!0,href:!0});var Er=p(ee);Qt=l(Er,"SPAN",{});var qr=p(Qt);g(_e.$$.fragment,qr),qr.forEach(t),Er.forEach(t),Io=h(Fs),Lt=l(Fs,"SPAN",{});var $r=p(Lt);Go=n($r,"Mel spectrogram"),$r.forEach(t),Fs.forEach(t),rs=h(e),at=l(e,"P",{});var xr=p(at);Bo=n(xr,`A mel spectrogram is a variation of the spectrogram that is commonly used in speech processing and machine learning tasks.
It is similar to a spectrogram in that it shows the frequency content of an audio signal over time, but on a different frequency axis.`),xr.forEach(t),ls=h(e),st=l(e,"P",{});var Mr=p(st);Fo=n(Mr,`In a standard spectrogram, the frequency axis is linear and is measured in hertz (Hz). However, the human auditory system
is more sensitive to changes in lower frequencies than higher frequencies, and this sensitivity decreases logarithmically
as frequency increases. The mel scale is a perceptual scale that approximates the non-linear frequency response of the human ear.`),Mr.forEach(t),ps=h(e),it=l(e,"P",{});var Jr=p(it);jo=n(Jr,`To create a mel spectrogram, the STFT is used just like before, splitting the audio into short segments to obtain a sequence
of frequency spectra. Additionally, each spectrum is sent through a set of filters, the so-called mel filterbank, to
transform the frequencies to the mel scale.`),Jr.forEach(t),us=h(e),te=l(e,"P",{});var js=p(te);Co=n(js,"Let\u2019s see how we can plot a mel spectrogram using librosa\u2019s "),Kt=l(js,"CODE",{});var zr=p(Kt);Po=n(zr,"melspectrogram()"),zr.forEach(t),Ao=n(js," function, which performs all of those steps for us:"),js.forEach(t),hs=h(e),g(Ee.$$.fragment,e),ms=h(e),qe=l(e,"DIV",{class:!0});var Sr=p(qe);ot=l(Sr,"IMG",{src:!0,alt:!0}),Sr.forEach(t),cs=h(e),E=l(e,"P",{});var oe=p(E);Ho=n(oe,"In the example above, "),ea=l(oe,"CODE",{});var Ir=p(ea);Zo=n(Ir,"n_mels"),Ir.forEach(t),No=n(oe,` stands for the number of mel bands to generate. The mel bands define a set of frequency
ranges that divide the spectrum into perceptually meaningful components, using a set of filters whose shape and spacing
are chosen to mimic the way the human ear responds to different frequencies. Common values for `),ta=l(oe,"CODE",{});var Gr=p(ta);Uo=n(Gr,"n_mels"),Gr.forEach(t),Do=n(oe," are 40 or 80. "),aa=l(oe,"CODE",{});var Br=p(aa);Wo=n(Br,"fmax"),Br.forEach(t),Xo=n(oe,`
indicates the highest frequency (in Hz) we care about.`),oe.forEach(t),fs=h(e),q=l(e,"P",{});var ne=p(q);Yo=n(ne,`Just as with a regular spectrogram, it\u2019s common practice to express the strength of the mel frequency components in
decibels. This is commonly referred to as a `),sa=l(ne,"STRONG",{});var Fr=p(sa);Ro=n(Fr,"log-mel spectrogram"),Fr.forEach(t),Vo=n(ne,`, because the conversion to decibels involves a
logarithmic operation. The above example used `),ia=l(ne,"CODE",{});var jr=p(ia);Oo=n(jr,"librosa.power_to_db()"),jr.forEach(t),Qo=n(ne," as "),oa=l(ne,"CODE",{});var Cr=p(oa);Lo=n(Cr,"librosa.feature.melspectrogram()"),Cr.forEach(t),Ko=n(ne," creates a power spectrogram."),ne.forEach(t),ds=h(e),g(ae.$$.fragment,e),gs=h(e),nt=l(e,"P",{});var Pr=p(nt);en=n(Pr,`Creating a mel spectrogram is a lossy operation as it involves filtering the signal. Converting a mel spectrogram back
into a waveform is more difficult than doing this for a regular spectrogram, as it requires estimating the frequencies
that were thrown away. This is why machine learning models such as HiFiGAN vocoder are needed to produce a waveform from a mel
spectrogram.`),Pr.forEach(t),vs=h(e),rt=l(e,"P",{});var Ar=p(rt);tn=n(Ar,`Compared to a standard spectrogram, a mel spectrogram can capture more meaningful features of the audio signal for
human perception, making it a popular choice in tasks such as speech recognition, speaker identification, and music genre classification.`),Ar.forEach(t),ws=h(e),lt=l(e,"P",{});var Hr=p(lt);an=n(Hr,"Now that you know how to visualize audio data examples, go ahead and try to see what your favorite sounds look like. :)"),Hr.forEach(t),this.h()},h(){m(c,"name","hf:doc:metadata"),m(c,"content",JSON.stringify(Or)),m(C,"id","introduction-to-audio-data"),m(C,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(C,"href","#introduction-to-audio-data"),m(T,"class","relative group"),m(P,"id","sampling-and-sampling-rate"),m(P,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(P,"href","#sampling-and-sampling-rate"),m(S,"class","relative group"),ra(Se.src,on="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/Signal_Sampling.png")||m(Se,"src",on),m(Se,"alt","Signal sampling illustration"),m(pe,"class","flex justify-center"),m(ue,"href","https://en.wikipedia.org/wiki/Sampling_(signal_processing)"),m(ue,"rel","nofollow"),m(Fe,"href","preprocessing#resampling-the-audio-data"),m(Z,"id","amplitude-and-bit-depth"),m(Z,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Z,"href","#amplitude-and-bit-depth"),m(I,"class","relative group"),m(Pe,"href","preprocessing"),m(W,"id","audio-as-a-waveform"),m(W,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(W,"href","#audio-as-a-waveform"),m(G,"class","relative group"),ra(Ne.src,nn="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/waveform_plot.png")||m(Ne,"src",nn),m(Ne,"alt","Waveform plot"),m(ge,"class","flex justify-center"),m(Y,"id","the-frequency-spectrum"),m(Y,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Y,"href","#the-frequency-spectrum"),m(B,"class","relative group"),ra(We.src,rn="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/spectrum_plot.png")||m(We,"src",rn),m(We,"alt","Spectrum plot"),m(ye,"class","flex justify-center"),m(Q,"id","spectrogram"),m(Q,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(Q,"href","#spectrogram"),m(F,"class","relative group"),ra(Qe.src,ln="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/spectrogram_plot.png")||m(Qe,"src",ln),m(Qe,"alt","Spectrogram plot"),m(ke,"class","flex justify-center"),m(ee,"id","mel-spectrogram"),m(ee,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ee,"href","#mel-spectrogram"),m(j,"class","relative group"),ra(ot.src,pn="https://huggingface.co/datasets/huggingface-course/audio-course-images/resolve/main/mel-spectrogram.png")||m(ot,"src",pn),m(ot,"alt","Mel spectrogram plot"),m(qe,"class","flex justify-center")},m(e,s){a(document.head,c),i(e,f,s),i(e,T,s),a(T,C),a(C,dt),v(re,dt,null),a(T,Cs),a(T,gt),a(gt,Ps),i(e,la,s),i(e,Je,s),a(Je,As),i(e,pa,s),i(e,k,s),a(k,Hs),a(k,vt),a(vt,Zs),a(k,Ns),a(k,wt),a(wt,Us),a(k,Ds),a(k,yt),a(yt,Ws),a(k,Xs),i(e,ua,s),i(e,ze,s),a(ze,Ys),i(e,ha,s),i(e,S,s),a(S,P),a(P,bt),v(le,bt,null),a(S,Rs),a(S,Tt),a(Tt,Vs),i(e,ma,s),i(e,A,s),a(A,Os),a(A,kt),a(kt,Qs),a(A,Ls),i(e,ca,s),i(e,pe,s),a(pe,Se),i(e,fa,s),i(e,Ie,s),a(Ie,Ge),a(Ge,Ks),a(Ge,ue),a(ue,ei),i(e,da,s),i(e,H,s),a(H,ti),a(H,_t),a(_t,ai),a(H,si),i(e,ga,s),i(e,Be,s),a(Be,ii),i(e,va,s),i(e,$,s),a($,oi),a($,Et),a(Et,ni),a($,ri),a($,Fe),a(Fe,li),a($,pi),i(e,wa,s),i(e,I,s),a(I,Z),a(Z,qt),v(he,qt,null),a(I,ui),a(I,$t),a($t,hi),i(e,ya,s),i(e,je,s),a(je,mi),i(e,ba,s),i(e,N,s),a(N,ci),a(N,xt),a(xt,fi),a(N,di),i(e,Ta,s),i(e,U,s),a(U,gi),a(U,Mt),a(Mt,vi),a(U,wi),i(e,ka,s),i(e,Ce,s),a(Ce,yi),i(e,_a,s),i(e,D,s),a(D,bi),a(D,Pe),a(Pe,Ti),a(D,ki),i(e,Ea,s),i(e,Ae,s),a(Ae,_i),i(e,qa,s),i(e,G,s),a(G,W),a(W,Jt),v(me,Jt,null),a(G,Ei),a(G,zt),a(zt,qi),i(e,$a,s),i(e,x,s),a(x,$i),a(x,St),a(St,xi),a(x,Mi),a(x,It),a(It,Ji),a(x,zi),i(e,xa,s),i(e,He,s),a(He,Si),i(e,Ma,s),i(e,X,s),a(X,Ii),a(X,Gt),a(Gt,Gi),a(X,Bi),i(e,Ja,s),v(ce,e,s),i(e,za,s),i(e,Ze,s),a(Ze,Fi),i(e,Sa,s),v(fe,e,s),i(e,Ia,s),i(e,_,s),a(_,ji),a(_,Bt),a(Bt,Ci),a(_,Pi),a(_,Ft),a(Ft,Ai),a(_,Hi),a(_,jt),a(jt,Zi),a(_,Ni),i(e,Ga,s),v(de,e,s),i(e,Ba,s),i(e,ge,s),a(ge,Ne),i(e,Fa,s),i(e,Ue,s),a(Ue,Ui),i(e,ja,s),i(e,De,s),a(De,Di),i(e,Ca,s),i(e,B,s),a(B,Y),a(Y,Ct),v(ve,Ct,null),a(B,Wi),a(B,Pt),a(Pt,Xi),i(e,Pa,s),i(e,M,s),a(M,Yi),a(M,At),a(At,Ri),a(M,Vi),a(M,Ht),a(Ht,Oi),a(M,Qi),i(e,Aa,s),i(e,R,s),a(R,Li),a(R,Zt),a(Zt,Ki),a(R,eo),i(e,Ha,s),v(we,e,s),i(e,Za,s),i(e,ye,s),a(ye,We),i(e,Na,s),i(e,Xe,s),a(Xe,to),i(e,Ua,s),i(e,Ye,s),a(Ye,ao),i(e,Da,s),i(e,V,s),a(V,so),a(V,Nt),a(Nt,io),a(V,oo),i(e,Wa,s),i(e,J,s),a(J,no),a(J,Ut),a(Ut,ro),a(J,lo),a(J,Dt),a(Dt,po),a(J,uo),i(e,Xa,s),v(O,e,s),i(e,Ya,s),i(e,Re,s),a(Re,ho),i(e,Ra,s),i(e,F,s),a(F,Q),a(Q,Wt),v(be,Wt,null),a(F,mo),a(F,Xt),a(Xt,co),i(e,Va,s),i(e,L,s),a(L,fo),a(L,Yt),a(Yt,go),a(L,vo),i(e,Oa,s),i(e,Ve,s),a(Ve,wo),i(e,Qa,s),i(e,Oe,s),a(Oe,yo),i(e,La,s),i(e,z,s),a(z,bo),a(z,Rt),a(Rt,To),a(z,ko),a(z,Vt),a(Vt,_o),a(z,Eo),i(e,Ka,s),v(Te,e,s),i(e,es,s),i(e,ke,s),a(ke,Qe),i(e,ts,s),i(e,Le,s),a(Le,qo),i(e,as,s),i(e,K,s),a(K,$o),a(K,Ot),a(Ot,xo),a(K,Mo),i(e,ss,s),i(e,Ke,s),a(Ke,Jo),i(e,is,s),i(e,et,s),a(et,zo),i(e,os,s),i(e,tt,s),a(tt,So),i(e,ns,s),i(e,j,s),a(j,ee),a(ee,Qt),v(_e,Qt,null),a(j,Io),a(j,Lt),a(Lt,Go),i(e,rs,s),i(e,at,s),a(at,Bo),i(e,ls,s),i(e,st,s),a(st,Fo),i(e,ps,s),i(e,it,s),a(it,jo),i(e,us,s),i(e,te,s),a(te,Co),a(te,Kt),a(Kt,Po),a(te,Ao),i(e,hs,s),v(Ee,e,s),i(e,ms,s),i(e,qe,s),a(qe,ot),i(e,cs,s),i(e,E,s),a(E,Ho),a(E,ea),a(ea,Zo),a(E,No),a(E,ta),a(ta,Uo),a(E,Do),a(E,aa),a(aa,Wo),a(E,Xo),i(e,fs,s),i(e,q,s),a(q,Yo),a(q,sa),a(sa,Ro),a(q,Vo),a(q,ia),a(ia,Oo),a(q,Qo),a(q,oa),a(oa,Lo),a(q,Ko),i(e,ds,s),v(ae,e,s),i(e,gs,s),i(e,nt,s),a(nt,en),i(e,vs,s),i(e,rt,s),a(rt,tn),i(e,ws,s),i(e,lt,s),a(lt,an),ys=!0},p(e,[s]){const $e={};s&2&&($e.$$scope={dirty:s,ctx:e}),O.$set($e);const na={};s&2&&(na.$$scope={dirty:s,ctx:e}),ae.$set(na)},i(e){ys||(w(re.$$.fragment,e),w(le.$$.fragment,e),w(he.$$.fragment,e),w(me.$$.fragment,e),w(ce.$$.fragment,e),w(fe.$$.fragment,e),w(de.$$.fragment,e),w(ve.$$.fragment,e),w(we.$$.fragment,e),w(O.$$.fragment,e),w(be.$$.fragment,e),w(Te.$$.fragment,e),w(_e.$$.fragment,e),w(Ee.$$.fragment,e),w(ae.$$.fragment,e),ys=!0)},o(e){y(re.$$.fragment,e),y(le.$$.fragment,e),y(he.$$.fragment,e),y(me.$$.fragment,e),y(ce.$$.fragment,e),y(fe.$$.fragment,e),y(de.$$.fragment,e),y(ve.$$.fragment,e),y(we.$$.fragment,e),y(O.$$.fragment,e),y(be.$$.fragment,e),y(Te.$$.fragment,e),y(_e.$$.fragment,e),y(Ee.$$.fragment,e),y(ae.$$.fragment,e),ys=!1},d(e){t(c),e&&t(f),e&&t(T),b(re),e&&t(la),e&&t(Je),e&&t(pa),e&&t(k),e&&t(ua),e&&t(ze),e&&t(ha),e&&t(S),b(le),e&&t(ma),e&&t(A),e&&t(ca),e&&t(pe),e&&t(fa),e&&t(Ie),e&&t(da),e&&t(H),e&&t(ga),e&&t(Be),e&&t(va),e&&t($),e&&t(wa),e&&t(I),b(he),e&&t(ya),e&&t(je),e&&t(ba),e&&t(N),e&&t(Ta),e&&t(U),e&&t(ka),e&&t(Ce),e&&t(_a),e&&t(D),e&&t(Ea),e&&t(Ae),e&&t(qa),e&&t(G),b(me),e&&t($a),e&&t(x),e&&t(xa),e&&t(He),e&&t(Ma),e&&t(X),e&&t(Ja),b(ce,e),e&&t(za),e&&t(Ze),e&&t(Sa),b(fe,e),e&&t(Ia),e&&t(_),e&&t(Ga),b(de,e),e&&t(Ba),e&&t(ge),e&&t(Fa),e&&t(Ue),e&&t(ja),e&&t(De),e&&t(Ca),e&&t(B),b(ve),e&&t(Pa),e&&t(M),e&&t(Aa),e&&t(R),e&&t(Ha),b(we,e),e&&t(Za),e&&t(ye),e&&t(Na),e&&t(Xe),e&&t(Ua),e&&t(Ye),e&&t(Da),e&&t(V),e&&t(Wa),e&&t(J),e&&t(Xa),b(O,e),e&&t(Ya),e&&t(Re),e&&t(Ra),e&&t(F),b(be),e&&t(Va),e&&t(L),e&&t(Oa),e&&t(Ve),e&&t(Qa),e&&t(Oe),e&&t(La),e&&t(z),e&&t(Ka),b(Te,e),e&&t(es),e&&t(ke),e&&t(ts),e&&t(Le),e&&t(as),e&&t(K),e&&t(ss),e&&t(Ke),e&&t(is),e&&t(et),e&&t(os),e&&t(tt),e&&t(ns),e&&t(j),b(_e),e&&t(rs),e&&t(at),e&&t(ls),e&&t(st),e&&t(ps),e&&t(it),e&&t(us),e&&t(te),e&&t(hs),b(Ee,e),e&&t(ms),e&&t(qe),e&&t(cs),e&&t(E),e&&t(fs),e&&t(q),e&&t(ds),b(ae,e),e&&t(gs),e&&t(nt),e&&t(vs),e&&t(rt),e&&t(ws),e&&t(lt)}}}const Or={local:"introduction-to-audio-data",sections:[{local:"sampling-and-sampling-rate",title:"Sampling and sampling rate"},{local:"amplitude-and-bit-depth",title:"Amplitude and bit depth"},{local:"audio-as-a-waveform",title:"Audio as a waveform"},{local:"the-frequency-spectrum",title:"The frequency spectrum"},{local:"spectrogram",title:"Spectrogram"},{local:"mel-spectrogram",title:"Mel spectrogram"}],title:"Introduction to audio data"};function Qr(Me){return Xr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class al extends Nr{constructor(c){super();Ur(this,c,Qr,Vr,Dr,{})}}export{al as default,Or as metadata};
