import{S as mi,i as ii,s as bi,e as p,k as u,w as j,t as n,M as ji,c as r,d as e,m as h,a as o,x as f,h as t,b as m,N as ci,G as a,g as c,y as d,q as g,o as _,B as $,v as fi,L as ui}from"../chunks/vendor-hf-doc-builder.js";import{T as Np}from"../chunks/Tip-hf-doc-builder.js";import{Y as di}from"../chunks/Youtube-hf-doc-builder.js";import{I as Ss}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as y}from"../chunks/CodeBlock-hf-doc-builder.js";import{D as gi}from"../chunks/DocNotebookDropdown-hf-doc-builder.js";import{F as _i,M as hi}from"../chunks/Markdown-hf-doc-builder.js";function $i(A){let i,v,b,w,k,E,x;return{c(){i=p("p"),v=p("code"),b=n("AutoProcessor"),w=u(),k=p("strong"),E=n("always"),x=n(" works and automatically chooses the correct class for the model you\u2019re using, whether you\u2019re using a tokenizer, image processor, feature extractor or processor.")},l(q){i=r(q,"P",{});var I=o(i);v=r(I,"CODE",{});var Ns=o(v);b=t(Ns,"AutoProcessor"),Ns.forEach(e),w=h(I),k=r(I,"STRONG",{});var L=o(k);E=t(L,"always"),L.forEach(e),x=t(I," works and automatically chooses the correct class for the model you\u2019re using, whether you\u2019re using a tokenizer, image processor, feature extractor or processor."),I.forEach(e)},m(q,I){c(q,i,I),a(i,v),a(v,b),a(i,w),a(i,k),a(k,E),a(i,x)},d(q){q&&e(i)}}}function vi(A){let i,v,b,w,k;return{c(){i=p("p"),v=n("If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),b=p("em"),w=n("vocab"),k=n(") during pretraining.")},l(E){i=r(E,"P",{});var x=o(i);v=t(x,"If you plan on using a pretrained model, it\u2019s important to use the associated pretrained tokenizer. This ensures the text is split the same way as the pretraining corpus, and uses the same corresponding tokens-to-index (usually referrred to as the "),b=r(x,"EM",{});var q=o(b);w=t(q,"vocab"),q.forEach(e),k=t(x,") during pretraining."),x.forEach(e)},m(E,x){c(E,i,x),a(i,v),a(i,b),a(b,w),a(i,k)},d(E){E&&e(i)}}}function wi(A){let i,v,b,w,k;return{c(){i=p("p"),v=n("Check out the "),b=p("a"),w=n("Padding and truncation"),k=n(" concept guide to learn more different padding and truncation arguments."),this.h()},l(E){i=r(E,"P",{});var x=o(i);v=t(x,"Check out the "),b=r(x,"A",{href:!0});var q=o(b);w=t(q,"Padding and truncation"),q.forEach(e),k=t(x," concept guide to learn more different padding and truncation arguments."),x.forEach(e),this.h()},h(){m(b,"href","./pad_truncation")},m(E,x){c(E,i,x),a(i,v),a(i,b),a(b,w),a(i,k)},d(E){E&&e(i)}}}function yi(A){let i,v;return i=new y({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="pt")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: tensor([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
                      [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: tensor([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]), 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: tensor([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
                           [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]])}`}}),{c(){j(i.$$.fragment)},l(b){f(i.$$.fragment,b)},m(b,w){d(i,b,w),v=!0},p:ui,i(b){v||(g(i.$$.fragment,b),v=!0)},o(b){_(i.$$.fragment,b),v=!1},d(b){$(i,b)}}}function ki(A){let i,v;return i=new hi({props:{$$slots:{default:[yi]},$$scope:{ctx:A}}}),{c(){j(i.$$.fragment)},l(b){f(i.$$.fragment,b)},m(b,w){d(i,b,w),v=!0},p(b,w){const k={};w&2&&(k.$$scope={dirty:w,ctx:b}),i.$set(k)},i(b){v||(g(i.$$.fragment,b),v=!0)},o(b){_(i.$$.fragment,b),v=!1},d(b){$(i,b)}}}function Ei(A){let i,v;return i=new y({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True, return_tensors="tf")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>],
       [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]],
      dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;, 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">9</span>), dtype=int32, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>],
       [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], dtype=int32)&gt;}`}}),{c(){j(i.$$.fragment)},l(b){f(i.$$.fragment,b)},m(b,w){d(i,b,w),v=!0},p:ui,i(b){v||(g(i.$$.fragment,b),v=!0)},o(b){_(i.$$.fragment,b),v=!1},d(b){$(i,b)}}}function xi(A){let i,v;return i=new hi({props:{$$slots:{default:[Ei]},$$scope:{ctx:A}}}),{c(){j(i.$$.fragment)},l(b){f(i.$$.fragment,b)},m(b,w){d(i,b,w),v=!0},p(b,w){const k={};w&2&&(k.$$scope={dirty:w,ctx:b}),i.$set(k)},i(b){v||(g(i.$$.fragment,b),v=!0)},o(b){_(i.$$.fragment,b),v=!1},d(b){$(i,b)}}}function qi(A){let i,v,b,w,k;return{c(){i=p("p"),v=n("Use \u{1F917} Datasets "),b=p("code"),w=n("split"),k=n(" parameter to only load a small sample from the training split since the dataset is quite large!")},l(E){i=r(E,"P",{});var x=o(i);v=t(x,"Use \u{1F917} Datasets "),b=r(x,"CODE",{});var q=o(b);w=t(q,"split"),q.forEach(e),k=t(x," parameter to only load a small sample from the training split since the dataset is quite large!"),x.forEach(e)},m(E,x){c(E,i,x),a(i,v),a(i,b),a(b,w),a(i,k)},d(E){E&&e(i)}}}function Ai(A){let i,v,b,w,k,E,x,q,I,Ns,L,dt,te,Fp,gt,T,Fs,Rp,le,Bp,Mp,Wp,Rs,Jp,pe,Hp,Up,Yp,Bs,Gp,re,Vp,Kp,Qp,Ms,Xp,oe,Zp,sr,_t,rs,$t,ce,ar,vt,Ws,wt,Z,os,tn,Js,er,ln,nr,yt,Hs,kt,S,tr,ue,lr,pr,pn,rr,or,Et,cs,xt,N,cr,he,ur,hr,rn,mr,ir,qt,Us,At,me,br,Pt,Ys,Tt,ie,jr,zt,F,be,je,fr,dr,gr,fe,de,_r,$r,vr,ge,_e,wr,yr,Ct,us,kr,on,Er,xr,Dt,Gs,It,R,qr,cn,Ar,Pr,un,Tr,zr,Ot,$e,Cr,Lt,Vs,St,ss,hs,hn,Ks,Dr,mn,Ir,Nt,ms,Or,bn,Lr,Sr,Ft,B,Nr,jn,Fr,Rr,fn,Br,Mr,Rt,Qs,Bt,is,Wr,dn,Jr,Hr,Mt,as,bs,gn,Xs,Ur,_n,Yr,Wt,ve,Gr,Jt,M,Vr,$n,Kr,Qr,vn,Xr,Zr,Ht,Zs,Ut,js,Yt,es,fs,wn,sa,so,yn,ao,Gt,we,eo,Vt,z,no,kn,to,lo,En,po,ro,xn,oo,co,Kt,ds,Qt,ns,gs,qn,aa,uo,An,ho,Xt,_s,mo,ye,io,bo,Zt,W,jo,ea,fo,go,na,_o,$o,sl,ta,al,J,vo,Pn,wo,yo,Tn,ko,Eo,el,la,nl,ke,xo,tl,H,Ee,zn,qo,Ao,Po,xe,Cn,To,zo,Co,qe,Dn,Do,Io,ll,$s,Oo,pa,Lo,So,pl,Ae,ra,No,oa,Fo,Ro,rl,ca,ol,ua,ha,Bo,In,Mo,Wo,cl,ma,ul,C,Jo,On,Ho,Uo,Ln,Yo,Go,Sn,Vo,Ko,hl,vs,Qo,Pe,Xo,Zo,ml,ia,il,U,sc,Nn,ac,ec,Fn,nc,tc,bl,ba,jl,Te,lc,fl,ja,dl,ze,pc,gl,fa,_l,ws,rc,Rn,oc,cc,$l,da,vl,Ce,uc,wl,ga,yl,ts,ys,Bn,_a,hc,Mn,mc,kl,ks,ic,De,bc,jc,El,Y,fc,$a,dc,gc,va,_c,$c,xl,Es,ql,wa,Al,xs,vc,ya,Wn,wc,yc,Pl,ka,Tl,Ea,Jn,Gu,zl,qs,kc,Ie,Ec,xc,Cl,xa,Dl,D,qc,qa,Hn,Ac,Pc,Aa,Tc,zc,Pa,Cc,Dc,Il,Oe,O,Ic,Ta,Un,Oc,Lc,za,Yn,Sc,Nc,Ca,Gn,Fc,Rc,Ol,Da,Ll,Ia,ls,Bc,Le,Vn,Mc,Wc,Kn,Jc,Hc,Sl,Oa,Nl,La,Sa,Uc,Na,Qn,Yc,Gc,Fl,Fa,Rl,Ra,Ba,Vc,Xn,Kc,Qc,Bl,Ma,Ml,Se,Xc,Wl,Wa,Jl,Ja,Zn,Vu,Hl,ps,As,st,Ha,Zc,at,su,Ul,Ps,au,Ne,eu,nu,Yl,G,tu,Ua,lu,pu,Ya,ru,ou,Gl,Ga,Vl,V,cu,et,uu,hu,nt,mu,iu,Kl,Va,Ql,K,bu,tt,ju,fu,lt,du,gu,Xl,Ka,Zl,Ts,_u,Fe,$u,vu,sp,Qa,ap,zs,wu,Re,yu,ku,ep,Xa,np,Be,P,Eu,pt,xu,qu,rt,Au,Pu,ot,Tu,zu,ct,Cu,Du,tp,Za,lp,se,ae,Iu,ut,Ou,Lu,pp,ee,rp,Q,Su,ht,Nu,Fu,mt,Ru,Bu,op;return E=new Ss({}),L=new gi({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/preprocessing.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/preprocessing.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/preprocessing.ipynb"}]}}),rs=new Np({props:{$$slots:{default:[$i]},$$scope:{ctx:A}}}),Ws=new y({props:{code:"pip install datasets",highlighted:"pip install datasets"}}),Js=new Ss({}),Hs=new di({props:{id:"Yffk5aydLzg"}}),cs=new Np({props:{$$slots:{default:[vi]},$$scope:{ctx:A}}}),Us=new y({props:{code:`from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-cased&quot;</span>)`}}),Ys=new y({props:{code:`encoded_input = tokenizer("Do not meddle in the affairs of wizards, for they are subtle and quick to anger.")
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(<span class="hljs-string">&quot;Do not meddle in the affairs of wizards, for they are subtle and quick to anger.&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">2079</span>, <span class="hljs-number">2025</span>, <span class="hljs-number">19960</span>, <span class="hljs-number">10362</span>, <span class="hljs-number">1999</span>, <span class="hljs-number">1996</span>, <span class="hljs-number">3821</span>, <span class="hljs-number">1997</span>, <span class="hljs-number">16657</span>, <span class="hljs-number">1010</span>, <span class="hljs-number">2005</span>, <span class="hljs-number">2027</span>, <span class="hljs-number">2024</span>, <span class="hljs-number">11259</span>, <span class="hljs-number">1998</span>, <span class="hljs-number">4248</span>, <span class="hljs-number">2000</span>, <span class="hljs-number">4963</span>, <span class="hljs-number">1012</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),Gs=new y({props:{code:'tokenizer.decode(encoded_input["input_ids"])',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.decode(encoded_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-string">&#x27;[CLS] Do not meddle in the affairs of wizards, for they are subtle and quick to anger. [SEP]&#x27;</span>`}}),Vs=new y({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_inputs = tokenizer(batch_sentences)
print(encoded_inputs)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_inputs = tokenizer(batch_sentences)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_inputs)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]]}`}}),Ks=new Ss({}),Qs=new y({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),Xs=new Ss({}),Zs=new y({props:{code:`batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True, truncation=True)
print(encoded_input)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>batch_sentences = [
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;But what about second breakfast?&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;Don&#x27;t think he knows about second breakfast, Pip.&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;What about elevensies?&quot;</span>,
<span class="hljs-meta">... </span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>encoded_input = tokenizer(batch_sentences, padding=<span class="hljs-literal">True</span>, truncation=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(encoded_input)
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [[<span class="hljs-number">101</span>, <span class="hljs-number">1252</span>, <span class="hljs-number">1184</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1790</span>, <span class="hljs-number">112</span>, <span class="hljs-number">189</span>, <span class="hljs-number">1341</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">3520</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">1248</span>, <span class="hljs-number">6462</span>, <span class="hljs-number">117</span>, <span class="hljs-number">21902</span>, <span class="hljs-number">1643</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
               [<span class="hljs-number">101</span>, <span class="hljs-number">1327</span>, <span class="hljs-number">1164</span>, <span class="hljs-number">5450</span>, <span class="hljs-number">23434</span>, <span class="hljs-number">136</span>, <span class="hljs-number">102</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [[<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>], 
                    [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>]]}`}}),js=new Np({props:{$$slots:{default:[wi]},$$scope:{ctx:A}}}),sa=new Ss({}),ds=new _i({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[xi],pytorch:[ki]},$$scope:{ctx:A}}}),aa=new Ss({}),ta=new y({props:{code:`from datasets import load_dataset, Audio

dataset = load_dataset("PolyAI/minds14", name="en-US", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, name=<span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),la=new y({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),ca=new y({props:{code:'dataset = dataset.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),ma=new y({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),ia=new y({props:{code:`from transformers import AutoFeatureExtractor

feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`}}),ba=new y({props:{code:`audio_input = [dataset[0]["audio"]["array"]]
feature_extractor(audio_input, sampling_rate=16000)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>audio_input = [dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor(audio_input, sampling_rate=<span class="hljs-number">16000</span>)
{<span class="hljs-string">&#x27;input_values&#x27;</span>: [array([ <span class="hljs-number">3.8106556e-04</span>,  <span class="hljs-number">2.7506407e-03</span>,  <span class="hljs-number">2.8015103e-03</span>, ...,
        <span class="hljs-number">5.6335266e-04</span>,  <span class="hljs-number">4.6588284e-06</span>, -<span class="hljs-number">1.7142107e-04</span>], dtype=float32)]}`}}),ja=new y({props:{code:`dataset[0]["audio"]["array"].shape

dataset[1]["audio"]["array"].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">173398</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">1</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>].shape
(<span class="hljs-number">106496</span>,)`}}),fa=new y({props:{code:`def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]
    inputs = feature_extractor(
        audio_arrays,
        sampling_rate=16000,
        padding=True,
        max_length=100000,
        truncation=True,
    )
    return inputs`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">100000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs`}}),da=new y({props:{code:"processed_dataset = preprocess_function(dataset[:5])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset = preprocess_function(dataset[:<span class="hljs-number">5</span>])'}}),ga=new y({props:{code:`processed_dataset["input_values"][0].shape

processed_dataset["input_values"][1].shape`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">0</span>].shape
(<span class="hljs-number">100000</span>,)

<span class="hljs-meta">&gt;&gt;&gt; </span>processed_dataset[<span class="hljs-string">&quot;input_values&quot;</span>][<span class="hljs-number">1</span>].shape
(<span class="hljs-number">100000</span>,)`}}),_a=new Ss({}),Es=new Np({props:{$$slots:{default:[qi]},$$scope:{ctx:A}}}),wa=new y({props:{code:`from datasets import load_dataset

dataset = load_dataset("food101", split="train[:100]")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;food101&quot;</span>, split=<span class="hljs-string">&quot;train[:100]&quot;</span>)`}}),ka=new y({props:{code:'dataset[0]["image"]',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]'}}),xa=new y({props:{code:`from transformers import AutoImageProcessor

image_processor = AutoImageProcessor.from_pretrained("google/vit-base-patch16-224")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoImageProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>image_processor = AutoImageProcessor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224&quot;</span>)`}}),Da=new y({props:{code:`from torchvision.transforms import Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
size = (
    feature_extractor.size["shortest_edge"]
    if "shortest_edge" in feature_extractor.size
    else (feature_extractor.size["height"], feature_extractor.size["width"])
)
_transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=0.5, hue=0.5), ToTensor(), normalize])`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> Compose, Normalize, RandomResizedCrop, ColorJitter, ToTensor

<span class="hljs-meta">&gt;&gt;&gt; </span>normalize = Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
<span class="hljs-meta">&gt;&gt;&gt; </span>size = (
<span class="hljs-meta">... </span>    feature_extractor.size[<span class="hljs-string">&quot;shortest_edge&quot;</span>]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">if</span> <span class="hljs-string">&quot;shortest_edge&quot;</span> <span class="hljs-keyword">in</span> feature_extractor.size
<span class="hljs-meta">... </span>    <span class="hljs-keyword">else</span> (feature_extractor.size[<span class="hljs-string">&quot;height&quot;</span>], feature_extractor.size[<span class="hljs-string">&quot;width&quot;</span>])
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>_transforms = Compose([RandomResizedCrop(size), ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>), ToTensor(), normalize])`}}),Oa=new y({props:{code:`def transforms(examples):
    examples["pixel_values"] = [_transforms(image.convert("RGB")) for image in examples["image"]]
    return examples`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = [_transforms(image.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),Fa=new y({props:{code:"dataset.set_transform(transforms)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)'}}),Ma=new y({props:{code:'dataset[0]["image"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at <span class="hljs-number">0x7F1A7B0630D0</span>&gt;,
 <span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">6</span>,
 <span class="hljs-string">&#x27;pixel_values&#x27;</span>: tensor([[[ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.1216</span>,  ..., -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>, -<span class="hljs-number">0.9922</span>],
          [-<span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.1294</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9843</span>, -<span class="hljs-number">0.9922</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.1137</span>,  ..., -<span class="hljs-number">0.9765</span>, -<span class="hljs-number">0.9686</span>, -<span class="hljs-number">0.8667</span>],
          ...,
          [ <span class="hljs-number">0.0275</span>,  <span class="hljs-number">0.0745</span>,  <span class="hljs-number">0.0510</span>,  ..., -<span class="hljs-number">0.1137</span>, -<span class="hljs-number">0.1216</span>, -<span class="hljs-number">0.0824</span>],
          [ <span class="hljs-number">0.0667</span>,  <span class="hljs-number">0.0824</span>,  <span class="hljs-number">0.0667</span>,  ..., -<span class="hljs-number">0.0588</span>, -<span class="hljs-number">0.0745</span>, -<span class="hljs-number">0.0980</span>],
          [ <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0431</span>,  ..., -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0588</span>]],
 
         [[ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.2863</span>,  ..., -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>],
          [ <span class="hljs-number">0.1608</span>,  <span class="hljs-number">0.2471</span>,  <span class="hljs-number">0.3098</span>,  ..., -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.9451</span>, -<span class="hljs-number">0.9373</span>],
          [ <span class="hljs-number">0.2078</span>,  <span class="hljs-number">0.2706</span>,  <span class="hljs-number">0.3020</span>,  ..., -<span class="hljs-number">0.9608</span>, -<span class="hljs-number">0.9373</span>, -<span class="hljs-number">0.8275</span>],
          ...,
          [-<span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.2392</span>, -<span class="hljs-number">0.2471</span>, -<span class="hljs-number">0.2078</span>],
          [ <span class="hljs-number">0.0196</span>,  <span class="hljs-number">0.0353</span>,  <span class="hljs-number">0.0196</span>,  ..., -<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.2000</span>, -<span class="hljs-number">0.2235</span>],
          [-<span class="hljs-number">0.0118</span>, -<span class="hljs-number">0.0039</span>, -<span class="hljs-number">0.0039</span>,  ..., -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.0980</span>, -<span class="hljs-number">0.1529</span>]],
 
         [[ <span class="hljs-number">0.3961</span>,  <span class="hljs-number">0.4431</span>,  <span class="hljs-number">0.4980</span>,  ..., -<span class="hljs-number">0.9216</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9216</span>],
          [ <span class="hljs-number">0.3569</span>,  <span class="hljs-number">0.4510</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9059</span>, -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.9137</span>],
          [ <span class="hljs-number">0.4118</span>,  <span class="hljs-number">0.4745</span>,  <span class="hljs-number">0.5216</span>,  ..., -<span class="hljs-number">0.9137</span>, -<span class="hljs-number">0.8902</span>, -<span class="hljs-number">0.7804</span>],
          ...,
          [-<span class="hljs-number">0.2314</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.2078</span>,  ..., -<span class="hljs-number">0.4196</span>, -<span class="hljs-number">0.4275</span>, -<span class="hljs-number">0.3882</span>],
          [-<span class="hljs-number">0.1843</span>, -<span class="hljs-number">0.1686</span>, -<span class="hljs-number">0.2000</span>,  ..., -<span class="hljs-number">0.3647</span>, -<span class="hljs-number">0.3804</span>, -<span class="hljs-number">0.4039</span>],
          [-<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>, -<span class="hljs-number">0.1922</span>,  ..., -<span class="hljs-number">0.2941</span>, -<span class="hljs-number">0.2863</span>, -<span class="hljs-number">0.3412</span>]]])}`}}),Wa=new y({props:{code:`import numpy as np
import matplotlib.pyplot as plt

img = dataset[0]["pixel_values"]
plt.imshow(img.permute(1, 2, 0))`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt

<span class="hljs-meta">&gt;&gt;&gt; </span>img = dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>plt.imshow(img.permute(<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">0</span>))`}}),Ha=new Ss({}),Ga=new y({props:{code:`from datasets import load_dataset

lj_speech = load_dataset("lj_speech", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = load_dataset(<span class="hljs-string">&quot;lj_speech&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),Va=new y({props:{code:'lj_speech = lj_speech.map(remove_columns=["file", "id", "normalized_text"])',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.<span class="hljs-built_in">map</span>(remove_columns=[<span class="hljs-string">&quot;file&quot;</span>, <span class="hljs-string">&quot;id&quot;</span>, <span class="hljs-string">&quot;normalized_text&quot;</span>])'}}),Ka=new y({props:{code:`lj_speech[0]["audio"]

lj_speech[0]["text"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([-<span class="hljs-number">7.3242188e-04</span>, -<span class="hljs-number">7.6293945e-04</span>, -<span class="hljs-number">6.4086914e-04</span>, ...,
         <span class="hljs-number">7.3242188e-04</span>,  <span class="hljs-number">2.1362305e-04</span>,  <span class="hljs-number">6.1035156e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/917ece08c95cf0c4115e45294e3cd0dee724a1165b7fc11798369308a465bd26/LJSpeech-1.1/wavs/LJ001-0001.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">22050</span>}

<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>]
<span class="hljs-string">&#x27;Printing, in the only sense with which we are at present concerned, differs from most if not from all the arts and crafts represented in the Exhibition&#x27;</span>`}}),Qa=new y({props:{code:'lj_speech = lj_speech.cast_column("audio", Audio(sampling_rate=16_000))',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>lj_speech = lj_speech.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))'}}),Xa=new y({props:{code:`from transformers import AutoProcessor

processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor

<span class="hljs-meta">&gt;&gt;&gt; </span>processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)`}}),Za=new y({props:{code:`def prepare_dataset(example):
    audio = example["audio"]

    example.update(processor(audio=audio["array"], text=example["text"], sampling_rate=16000))

    return example`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">prepare_dataset</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    audio = example[<span class="hljs-string">&quot;audio&quot;</span>]

<span class="hljs-meta">... </span>    example.update(processor(audio=audio[<span class="hljs-string">&quot;array&quot;</span>], text=example[<span class="hljs-string">&quot;text&quot;</span>], sampling_rate=<span class="hljs-number">16000</span>))

<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> example`}}),ee=new y({props:{code:"prepare_dataset(lj_speech[0])",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>prepare_dataset(lj_speech[<span class="hljs-number">0</span>])'}}),{c(){i=p("meta"),v=u(),b=p("h1"),w=p("a"),k=p("span"),j(E.$$.fragment),x=u(),q=p("span"),I=n("Preprocess"),Ns=u(),j(L.$$.fragment),dt=u(),te=p("p"),Fp=n("Before you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \u{1F917} Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you\u2019ll learn that for:"),gt=u(),T=p("ul"),Fs=p("li"),Rp=n("Text, use a "),le=p("a"),Bp=n("Tokenizer"),Mp=n(" to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors."),Wp=u(),Rs=p("li"),Jp=n("Image inputs use a "),pe=p("a"),Hp=n("ImageProcessor"),Up=n(" to convert images into tensors."),Yp=u(),Bs=p("li"),Gp=n("Speech and audio, use a "),re=p("a"),Vp=n("Feature extractor"),Kp=n(" to extract sequential features from audio waveforms and convert them into tensors."),Qp=u(),Ms=p("li"),Xp=n("Multimodal inputs, use a "),oe=p("a"),Zp=n("Processor"),sr=n(" to combine a tokenizer and a feature extractor or image processor."),_t=u(),j(rs.$$.fragment),$t=u(),ce=p("p"),ar=n("Before you begin, install \u{1F917} Datasets so you can load some datasets to experiment with:"),vt=u(),j(Ws.$$.fragment),wt=u(),Z=p("h2"),os=p("a"),tn=p("span"),j(Js.$$.fragment),er=u(),ln=p("span"),nr=n("Natural Language Processing"),yt=u(),j(Hs.$$.fragment),kt=u(),S=p("p"),tr=n("The main tool for preprocessing textual data is a "),ue=p("a"),lr=n("tokenizer"),pr=n(". A tokenizer splits text into "),pn=p("em"),rr=n("tokens"),or=n(" according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer."),Et=u(),j(cs.$$.fragment),xt=u(),N=p("p"),cr=n("Get started by loading a pretrained tokenizer with the "),he=p("a"),ur=n("AutoTokenizer.from_pretrained()"),hr=n(" method. This downloads the "),rn=p("em"),mr=n("vocab"),ir=n(" a model was pretrained with:"),qt=u(),j(Us.$$.fragment),At=u(),me=p("p"),br=n("Then pass your text to the tokenizer:"),Pt=u(),j(Ys.$$.fragment),Tt=u(),ie=p("p"),jr=n("The tokenizer returns a dictionary with three important items:"),zt=u(),F=p("ul"),be=p("li"),je=p("a"),fr=n("input_ids"),dr=n(" are the indices corresponding to each token in the sentence."),gr=u(),fe=p("li"),de=p("a"),_r=n("attention_mask"),$r=n(" indicates whether a token should be attended to or not."),vr=u(),ge=p("li"),_e=p("a"),wr=n("token_type_ids"),yr=n(" identifies which sequence a token belongs to when there is more than one sequence."),Ct=u(),us=p("p"),kr=n("Return your input by decoding the "),on=p("code"),Er=n("input_ids"),xr=n(":"),Dt=u(),j(Gs.$$.fragment),It=u(),R=p("p"),qr=n("As you can see, the tokenizer added two special tokens - "),cn=p("code"),Ar=n("CLS"),Pr=n(" and "),un=p("code"),Tr=n("SEP"),zr=n(` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer automatically adds them for you.`),Ot=u(),$e=p("p"),Cr=n("If there are several sentences you want to preprocess, pass them as a list to the tokenizer:"),Lt=u(),j(Vs.$$.fragment),St=u(),ss=p("h3"),hs=p("a"),hn=p("span"),j(Ks.$$.fragment),Dr=u(),mn=p("span"),Ir=n("Pad"),Nt=u(),ms=p("p"),Or=n("Sentences aren\u2019t always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),bn=p("em"),Lr=n("padding token"),Sr=n(" to shorter sentences."),Ft=u(),B=p("p"),Nr=n("Set the "),jn=p("code"),Fr=n("padding"),Rr=n(" parameter to "),fn=p("code"),Br=n("True"),Mr=n(" to pad the shorter sequences in the batch to match the longest sequence:"),Rt=u(),j(Qs.$$.fragment),Bt=u(),is=p("p"),Wr=n("The first and third sentences are now padded with "),dn=p("code"),Jr=n("0"),Hr=n("\u2019s because they are shorter."),Mt=u(),as=p("h3"),bs=p("a"),gn=p("span"),j(Xs.$$.fragment),Ur=u(),_n=p("span"),Yr=n("Truncation"),Wt=u(),ve=p("p"),Gr=n("On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you\u2019ll need to truncate the sequence to a shorter length."),Jt=u(),M=p("p"),Vr=n("Set the "),$n=p("code"),Kr=n("truncation"),Qr=n(" parameter to "),vn=p("code"),Xr=n("True"),Zr=n(" to truncate a sequence to the maximum length accepted by the model:"),Ht=u(),j(Zs.$$.fragment),Ut=u(),j(js.$$.fragment),Yt=u(),es=p("h3"),fs=p("a"),wn=p("span"),j(sa.$$.fragment),so=u(),yn=p("span"),ao=n("Build tensors"),Gt=u(),we=p("p"),eo=n("Finally, you want the tokenizer to return the actual tensors that get fed to the model."),Vt=u(),z=p("p"),no=n("Set the "),kn=p("code"),to=n("return_tensors"),lo=n(" parameter to either "),En=p("code"),po=n("pt"),ro=n(" for PyTorch, or "),xn=p("code"),oo=n("tf"),co=n(" for TensorFlow:"),Kt=u(),j(ds.$$.fragment),Qt=u(),ns=p("h2"),gs=p("a"),qn=p("span"),j(aa.$$.fragment),uo=u(),An=p("span"),ho=n("Audio"),Xt=u(),_s=p("p"),mo=n("For audio tasks, you\u2019ll need a "),ye=p("a"),io=n("feature extractor"),bo=n(" to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."),Zt=u(),W=p("p"),jo=n("Load the "),ea=p("a"),fo=n("MInDS-14"),go=n(" dataset (see the \u{1F917} "),na=p("a"),_o=n("Datasets tutorial"),$o=n(" for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:"),sl=u(),j(ta.$$.fragment),al=u(),J=p("p"),vo=n("Access the first element of the "),Pn=p("code"),wo=n("audio"),yo=n(" column to take a look at the input. Calling the "),Tn=p("code"),ko=n("audio"),Eo=n(" column automatically loads and resamples the audio file:"),el=u(),j(la.$$.fragment),nl=u(),ke=p("p"),xo=n("This returns three items:"),tl=u(),H=p("ul"),Ee=p("li"),zn=p("code"),qo=n("array"),Ao=n(" is the speech signal loaded - and potentially resampled - as a 1D array."),Po=u(),xe=p("li"),Cn=p("code"),To=n("path"),zo=n(" points to the location of the audio file."),Co=u(),qe=p("li"),Dn=p("code"),Do=n("sampling_rate"),Io=n(" refers to how many data points in the speech signal are measured per second."),ll=u(),$s=p("p"),Oo=n("For this tutorial, you\u2019ll use the "),pa=p("a"),Lo=n("Wav2Vec2"),So=n(" model. Take a look at the model card, and you\u2019ll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your data."),pl=u(),Ae=p("ol"),ra=p("li"),No=n("Use \u{1F917} Datasets\u2019 "),oa=p("a"),Fo=n("cast_column"),Ro=n(" method to upsample the sampling rate to 16kHz:"),rl=u(),j(ca.$$.fragment),ol=u(),ua=p("ol"),ha=p("li"),Bo=n("Call the "),In=p("code"),Mo=n("audio"),Wo=n(" column again to resample the audio file:"),cl=u(),j(ma.$$.fragment),ul=u(),C=p("p"),Jo=n("Next, load a feature extractor to normalize and pad the input. When padding textual data, a "),On=p("code"),Ho=n("0"),Uo=n(" is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a "),Ln=p("code"),Yo=n("0"),Go=n(" - interpreted as silence - to "),Sn=p("code"),Vo=n("array"),Ko=n("."),hl=u(),vs=p("p"),Qo=n("Load the feature extractor with "),Pe=p("a"),Xo=n("AutoFeatureExtractor.from_pretrained()"),Zo=n(":"),ml=u(),j(ia.$$.fragment),il=u(),U=p("p"),sc=n("Pass the audio "),Nn=p("code"),ac=n("array"),ec=n(" to the feature extractor. We also recommend adding the "),Fn=p("code"),nc=n("sampling_rate"),tc=n(" argument in the feature extractor in order to better debug any silent errors that may occur."),bl=u(),j(ba.$$.fragment),jl=u(),Te=p("p"),lc=n("Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),fl=u(),j(ja.$$.fragment),dl=u(),ze=p("p"),pc=n("Create a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),gl=u(),j(fa.$$.fragment),_l=u(),ws=p("p"),rc=n("Apply the "),Rn=p("code"),oc=n("preprocess_function"),cc=n(" to the the first few examples in the dataset:"),$l=u(),j(da.$$.fragment),vl=u(),Ce=p("p"),uc=n("The sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"),wl=u(),j(ga.$$.fragment),yl=u(),ts=p("h2"),ys=p("a"),Bn=p("span"),j(_a.$$.fragment),hc=u(),Mn=p("span"),mc=n("Computer vision"),kl=u(),ks=p("p"),ic=n("For computer vision tasks, you\u2019ll need an "),De=p("a"),bc=n("image processor"),jc=n(" to prepare your dataset for the model. The image processor is designed to preprocess images, and convert them into tensors."),El=u(),Y=p("p"),fc=n("Load the "),$a=p("a"),dc=n("food101"),gc=n(" dataset (see the \u{1F917} "),va=p("a"),_c=n("Datasets tutorial"),$c=n(" for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:"),xl=u(),j(Es.$$.fragment),ql=u(),j(wa.$$.fragment),Al=u(),xs=p("p"),vc=n("Next, take a look at the image with \u{1F917} Datasets "),ya=p("a"),Wn=p("code"),wc=n("Image"),yc=n(" feature:"),Pl=u(),j(ka.$$.fragment),Tl=u(),Ea=p("div"),Jn=p("img"),zl=u(),qs=p("p"),kc=n("Load the image processor with "),Ie=p("a"),Ec=n("AutoImageProcessor.from_pretrained()"),xc=n(":"),Cl=u(),j(xa.$$.fragment),Dl=u(),D=p("p"),qc=n("For computer vision tasks, it is common to add some type of data augmentation to the images as a part of preprocessing. You can add augmentations with any library you\u2019d like, but in this tutorial, you\u2019ll use torchvision\u2019s "),qa=p("a"),Hn=p("code"),Ac=n("transforms"),Pc=n(" module. If you\u2019re interested in using another data augmentation library, learn how in the "),Aa=p("a"),Tc=n("Albumentations"),zc=n(" or "),Pa=p("a"),Cc=n("Kornia notebooks"),Dc=n("."),Il=u(),Oe=p("ol"),O=p("li"),Ic=n("Normalize the image with the image processor and use "),Ta=p("a"),Un=p("code"),Oc=n("Compose"),Lc=n(" to chain some transforms - "),za=p("a"),Yn=p("code"),Sc=n("RandomResizedCrop"),Nc=n(" and "),Ca=p("a"),Gn=p("code"),Fc=n("ColorJitter"),Rc=n(" - together:"),Ol=u(),j(Da.$$.fragment),Ll=u(),Ia=p("ol"),ls=p("li"),Bc=n("The model accepts "),Le=p("a"),Vn=p("code"),Mc=n("pixel_values"),Wc=n(" as its input, which is generated by the image processor. Create a function that generates "),Kn=p("code"),Jc=n("pixel_values"),Hc=n(" from the transforms:"),Sl=u(),j(Oa.$$.fragment),Nl=u(),La=p("ol"),Sa=p("li"),Uc=n("Then use \u{1F917} Datasets "),Na=p("a"),Qn=p("code"),Yc=n("set_transform"),Gc=n(" to apply the transforms on the fly:"),Fl=u(),j(Fa.$$.fragment),Rl=u(),Ra=p("ol"),Ba=p("li"),Vc=n("Now when you access the image, you\u2019ll notice the image processor has added "),Xn=p("code"),Kc=n("pixel_values"),Qc=n(". You can pass your processed dataset to the model now!"),Bl=u(),j(Ma.$$.fragment),Ml=u(),Se=p("p"),Xc=n("Here is what the image looks like after the transforms are applied. The image has been randomly cropped and it\u2019s color properties are different."),Wl=u(),j(Wa.$$.fragment),Jl=u(),Ja=p("div"),Zn=p("img"),Hl=u(),ps=p("h2"),As=p("a"),st=p("span"),j(Ha.$$.fragment),Zc=u(),at=p("span"),su=n("Multimodal"),Ul=u(),Ps=p("p"),au=n("For tasks involving multimodal inputs, you\u2019ll need a "),Ne=p("a"),eu=n("processor"),nu=n(" to prepare your dataset for the model. A processor couples together two processing objects such as as tokenizer and feature extractor."),Yl=u(),G=p("p"),tu=n("Load the "),Ua=p("a"),lu=n("LJ Speech"),pu=n(" dataset (see the \u{1F917} "),Ya=p("a"),ru=n("Datasets tutorial"),ou=n(" for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):"),Gl=u(),j(Ga.$$.fragment),Vl=u(),V=p("p"),cu=n("For ASR, you\u2019re mainly focused on "),et=p("code"),uu=n("audio"),hu=n(" and "),nt=p("code"),mu=n("text"),iu=n(" so you can remove the other columns:"),Kl=u(),j(Va.$$.fragment),Ql=u(),K=p("p"),bu=n("Now take a look at the "),tt=p("code"),ju=n("audio"),fu=n(" and "),lt=p("code"),du=n("text"),gu=n(" columns:"),Xl=u(),j(Ka.$$.fragment),Zl=u(),Ts=p("p"),_u=n("Remember you should always "),Fe=p("a"),$u=n("resample"),vu=n(" your audio dataset\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model!"),sp=u(),j(Qa.$$.fragment),ap=u(),zs=p("p"),wu=n("Load a processor with "),Re=p("a"),yu=n("AutoProcessor.from_pretrained()"),ku=n(":"),ep=u(),j(Xa.$$.fragment),np=u(),Be=p("ol"),P=p("li"),Eu=n("Create a function to process the audio data contained in "),pt=p("code"),xu=n("array"),qu=n(" to "),rt=p("code"),Au=n("input_values"),Pu=n(", and tokenize "),ot=p("code"),Tu=n("text"),zu=n(" to "),ct=p("code"),Cu=n("labels"),Du=n(". These are the inputs to the model:"),tp=u(),j(Za.$$.fragment),lp=u(),se=p("ol"),ae=p("li"),Iu=n("Apply the "),ut=p("code"),Ou=n("prepare_dataset"),Lu=n(" function to a sample:"),pp=u(),j(ee.$$.fragment),rp=u(),Q=p("p"),Su=n("The processor has now added "),ht=p("code"),Nu=n("input_values"),Fu=n(" and "),mt=p("code"),Ru=n("labels"),Bu=n(", and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"),this.h()},l(s){const l=ji('[data-svelte="svelte-1phssyn"]',document.head);i=r(l,"META",{name:!0,content:!0}),l.forEach(e),v=h(s),b=r(s,"H1",{class:!0});var ne=o(b);w=r(ne,"A",{id:!0,class:!0,href:!0});var it=o(w);k=r(it,"SPAN",{});var bt=o(k);f(E.$$.fragment,bt),bt.forEach(e),it.forEach(e),x=h(ne),q=r(ne,"SPAN",{});var jt=o(q);I=t(jt,"Preprocess"),jt.forEach(e),ne.forEach(e),Ns=h(s),f(L.$$.fragment,s),dt=h(s),te=r(s,"P",{});var ft=o(te);Fp=t(ft,"Before you can train a model on a dataset, it needs to be preprocessed into the expected model input format. Whether your data is text, images, or audio, they need to be converted and assembled into batches of tensors. \u{1F917} Transformers provides a set of preprocessing classes to help prepare your data for the model. In this tutorial, you\u2019ll learn that for:"),ft.forEach(e),gt=h(s),T=r(s,"UL",{});var Cs=o(T);Fs=r(Cs,"LI",{});var cp=o(Fs);Rp=t(cp,"Text, use a "),le=r(cp,"A",{href:!0});var Ku=o(le);Bp=t(Ku,"Tokenizer"),Ku.forEach(e),Mp=t(cp," to convert text into a sequence of tokens, create a numerical representation of the tokens, and assemble them into tensors."),cp.forEach(e),Wp=h(Cs),Rs=r(Cs,"LI",{});var up=o(Rs);Jp=t(up,"Image inputs use a "),pe=r(up,"A",{href:!0});var Qu=o(pe);Hp=t(Qu,"ImageProcessor"),Qu.forEach(e),Up=t(up," to convert images into tensors."),up.forEach(e),Yp=h(Cs),Bs=r(Cs,"LI",{});var hp=o(Bs);Gp=t(hp,"Speech and audio, use a "),re=r(hp,"A",{href:!0});var Xu=o(re);Vp=t(Xu,"Feature extractor"),Xu.forEach(e),Kp=t(hp," to extract sequential features from audio waveforms and convert them into tensors."),hp.forEach(e),Qp=h(Cs),Ms=r(Cs,"LI",{});var mp=o(Ms);Xp=t(mp,"Multimodal inputs, use a "),oe=r(mp,"A",{href:!0});var Zu=o(oe);Zp=t(Zu,"Processor"),Zu.forEach(e),sr=t(mp," to combine a tokenizer and a feature extractor or image processor."),mp.forEach(e),Cs.forEach(e),_t=h(s),f(rs.$$.fragment,s),$t=h(s),ce=r(s,"P",{});var sh=o(ce);ar=t(sh,"Before you begin, install \u{1F917} Datasets so you can load some datasets to experiment with:"),sh.forEach(e),vt=h(s),f(Ws.$$.fragment,s),wt=h(s),Z=r(s,"H2",{class:!0});var ip=o(Z);os=r(ip,"A",{id:!0,class:!0,href:!0});var ah=o(os);tn=r(ah,"SPAN",{});var eh=o(tn);f(Js.$$.fragment,eh),eh.forEach(e),ah.forEach(e),er=h(ip),ln=r(ip,"SPAN",{});var nh=o(ln);nr=t(nh,"Natural Language Processing"),nh.forEach(e),ip.forEach(e),yt=h(s),f(Hs.$$.fragment,s),kt=h(s),S=r(s,"P",{});var Me=o(S);tr=t(Me,"The main tool for preprocessing textual data is a "),ue=r(Me,"A",{href:!0});var th=o(ue);lr=t(th,"tokenizer"),th.forEach(e),pr=t(Me,". A tokenizer splits text into "),pn=r(Me,"EM",{});var lh=o(pn);rr=t(lh,"tokens"),lh.forEach(e),or=t(Me," according to a set of rules. The tokens are converted into numbers and then tensors, which become the model inputs. Any additional inputs required by the model are added by the tokenizer."),Me.forEach(e),Et=h(s),f(cs.$$.fragment,s),xt=h(s),N=r(s,"P",{});var We=o(N);cr=t(We,"Get started by loading a pretrained tokenizer with the "),he=r(We,"A",{href:!0});var ph=o(he);ur=t(ph,"AutoTokenizer.from_pretrained()"),ph.forEach(e),hr=t(We," method. This downloads the "),rn=r(We,"EM",{});var rh=o(rn);mr=t(rh,"vocab"),rh.forEach(e),ir=t(We," a model was pretrained with:"),We.forEach(e),qt=h(s),f(Us.$$.fragment,s),At=h(s),me=r(s,"P",{});var oh=o(me);br=t(oh,"Then pass your text to the tokenizer:"),oh.forEach(e),Pt=h(s),f(Ys.$$.fragment,s),Tt=h(s),ie=r(s,"P",{});var ch=o(ie);jr=t(ch,"The tokenizer returns a dictionary with three important items:"),ch.forEach(e),zt=h(s),F=r(s,"UL",{});var Je=o(F);be=r(Je,"LI",{});var Mu=o(be);je=r(Mu,"A",{href:!0});var uh=o(je);fr=t(uh,"input_ids"),uh.forEach(e),dr=t(Mu," are the indices corresponding to each token in the sentence."),Mu.forEach(e),gr=h(Je),fe=r(Je,"LI",{});var Wu=o(fe);de=r(Wu,"A",{href:!0});var hh=o(de);_r=t(hh,"attention_mask"),hh.forEach(e),$r=t(Wu," indicates whether a token should be attended to or not."),Wu.forEach(e),vr=h(Je),ge=r(Je,"LI",{});var Ju=o(ge);_e=r(Ju,"A",{href:!0});var mh=o(_e);wr=t(mh,"token_type_ids"),mh.forEach(e),yr=t(Ju," identifies which sequence a token belongs to when there is more than one sequence."),Ju.forEach(e),Je.forEach(e),Ct=h(s),us=r(s,"P",{});var bp=o(us);kr=t(bp,"Return your input by decoding the "),on=r(bp,"CODE",{});var ih=o(on);Er=t(ih,"input_ids"),ih.forEach(e),xr=t(bp,":"),bp.forEach(e),Dt=h(s),f(Gs.$$.fragment,s),It=h(s),R=r(s,"P",{});var He=o(R);qr=t(He,"As you can see, the tokenizer added two special tokens - "),cn=r(He,"CODE",{});var bh=o(cn);Ar=t(bh,"CLS"),bh.forEach(e),Pr=t(He," and "),un=r(He,"CODE",{});var jh=o(un);Tr=t(jh,"SEP"),jh.forEach(e),zr=t(He,` (classifier and separator) - to the sentence. Not all models need
special tokens, but if they do, the tokenizer automatically adds them for you.`),He.forEach(e),Ot=h(s),$e=r(s,"P",{});var fh=o($e);Cr=t(fh,"If there are several sentences you want to preprocess, pass them as a list to the tokenizer:"),fh.forEach(e),Lt=h(s),f(Vs.$$.fragment,s),St=h(s),ss=r(s,"H3",{class:!0});var jp=o(ss);hs=r(jp,"A",{id:!0,class:!0,href:!0});var dh=o(hs);hn=r(dh,"SPAN",{});var gh=o(hn);f(Ks.$$.fragment,gh),gh.forEach(e),dh.forEach(e),Dr=h(jp),mn=r(jp,"SPAN",{});var _h=o(mn);Ir=t(_h,"Pad"),_h.forEach(e),jp.forEach(e),Nt=h(s),ms=r(s,"P",{});var fp=o(ms);Or=t(fp,"Sentences aren\u2019t always the same length which can be an issue because tensors, the model inputs, need to have a uniform shape. Padding is a strategy for ensuring tensors are rectangular by adding a special "),bn=r(fp,"EM",{});var $h=o(bn);Lr=t($h,"padding token"),$h.forEach(e),Sr=t(fp," to shorter sentences."),fp.forEach(e),Ft=h(s),B=r(s,"P",{});var Ue=o(B);Nr=t(Ue,"Set the "),jn=r(Ue,"CODE",{});var vh=o(jn);Fr=t(vh,"padding"),vh.forEach(e),Rr=t(Ue," parameter to "),fn=r(Ue,"CODE",{});var wh=o(fn);Br=t(wh,"True"),wh.forEach(e),Mr=t(Ue," to pad the shorter sequences in the batch to match the longest sequence:"),Ue.forEach(e),Rt=h(s),f(Qs.$$.fragment,s),Bt=h(s),is=r(s,"P",{});var dp=o(is);Wr=t(dp,"The first and third sentences are now padded with "),dn=r(dp,"CODE",{});var yh=o(dn);Jr=t(yh,"0"),yh.forEach(e),Hr=t(dp,"\u2019s because they are shorter."),dp.forEach(e),Mt=h(s),as=r(s,"H3",{class:!0});var gp=o(as);bs=r(gp,"A",{id:!0,class:!0,href:!0});var kh=o(bs);gn=r(kh,"SPAN",{});var Eh=o(gn);f(Xs.$$.fragment,Eh),Eh.forEach(e),kh.forEach(e),Ur=h(gp),_n=r(gp,"SPAN",{});var xh=o(_n);Yr=t(xh,"Truncation"),xh.forEach(e),gp.forEach(e),Wt=h(s),ve=r(s,"P",{});var qh=o(ve);Gr=t(qh,"On the other end of the spectrum, sometimes a sequence may be too long for a model to handle. In this case, you\u2019ll need to truncate the sequence to a shorter length."),qh.forEach(e),Jt=h(s),M=r(s,"P",{});var Ye=o(M);Vr=t(Ye,"Set the "),$n=r(Ye,"CODE",{});var Ah=o($n);Kr=t(Ah,"truncation"),Ah.forEach(e),Qr=t(Ye," parameter to "),vn=r(Ye,"CODE",{});var Ph=o(vn);Xr=t(Ph,"True"),Ph.forEach(e),Zr=t(Ye," to truncate a sequence to the maximum length accepted by the model:"),Ye.forEach(e),Ht=h(s),f(Zs.$$.fragment,s),Ut=h(s),f(js.$$.fragment,s),Yt=h(s),es=r(s,"H3",{class:!0});var _p=o(es);fs=r(_p,"A",{id:!0,class:!0,href:!0});var Th=o(fs);wn=r(Th,"SPAN",{});var zh=o(wn);f(sa.$$.fragment,zh),zh.forEach(e),Th.forEach(e),so=h(_p),yn=r(_p,"SPAN",{});var Ch=o(yn);ao=t(Ch,"Build tensors"),Ch.forEach(e),_p.forEach(e),Gt=h(s),we=r(s,"P",{});var Dh=o(we);eo=t(Dh,"Finally, you want the tokenizer to return the actual tensors that get fed to the model."),Dh.forEach(e),Vt=h(s),z=r(s,"P",{});var Ds=o(z);no=t(Ds,"Set the "),kn=r(Ds,"CODE",{});var Ih=o(kn);to=t(Ih,"return_tensors"),Ih.forEach(e),lo=t(Ds," parameter to either "),En=r(Ds,"CODE",{});var Oh=o(En);po=t(Oh,"pt"),Oh.forEach(e),ro=t(Ds," for PyTorch, or "),xn=r(Ds,"CODE",{});var Lh=o(xn);oo=t(Lh,"tf"),Lh.forEach(e),co=t(Ds," for TensorFlow:"),Ds.forEach(e),Kt=h(s),f(ds.$$.fragment,s),Qt=h(s),ns=r(s,"H2",{class:!0});var $p=o(ns);gs=r($p,"A",{id:!0,class:!0,href:!0});var Sh=o(gs);qn=r(Sh,"SPAN",{});var Nh=o(qn);f(aa.$$.fragment,Nh),Nh.forEach(e),Sh.forEach(e),uo=h($p),An=r($p,"SPAN",{});var Fh=o(An);ho=t(Fh,"Audio"),Fh.forEach(e),$p.forEach(e),Xt=h(s),_s=r(s,"P",{});var vp=o(_s);mo=t(vp,"For audio tasks, you\u2019ll need a "),ye=r(vp,"A",{href:!0});var Rh=o(ye);io=t(Rh,"feature extractor"),Rh.forEach(e),bo=t(vp," to prepare your dataset for the model. The feature extractor is designed to extract features from raw audio data, and convert them into tensors."),vp.forEach(e),Zt=h(s),W=r(s,"P",{});var Ge=o(W);jo=t(Ge,"Load the "),ea=r(Ge,"A",{href:!0,rel:!0});var Bh=o(ea);fo=t(Bh,"MInDS-14"),Bh.forEach(e),go=t(Ge," dataset (see the \u{1F917} "),na=r(Ge,"A",{href:!0,rel:!0});var Mh=o(na);_o=t(Mh,"Datasets tutorial"),Mh.forEach(e),$o=t(Ge," for more details on how to load a dataset) to see how you can use a feature extractor with audio datasets:"),Ge.forEach(e),sl=h(s),f(ta.$$.fragment,s),al=h(s),J=r(s,"P",{});var Ve=o(J);vo=t(Ve,"Access the first element of the "),Pn=r(Ve,"CODE",{});var Wh=o(Pn);wo=t(Wh,"audio"),Wh.forEach(e),yo=t(Ve," column to take a look at the input. Calling the "),Tn=r(Ve,"CODE",{});var Jh=o(Tn);ko=t(Jh,"audio"),Jh.forEach(e),Eo=t(Ve," column automatically loads and resamples the audio file:"),Ve.forEach(e),el=h(s),f(la.$$.fragment,s),nl=h(s),ke=r(s,"P",{});var Hh=o(ke);xo=t(Hh,"This returns three items:"),Hh.forEach(e),tl=h(s),H=r(s,"UL",{});var Ke=o(H);Ee=r(Ke,"LI",{});var Hu=o(Ee);zn=r(Hu,"CODE",{});var Uh=o(zn);qo=t(Uh,"array"),Uh.forEach(e),Ao=t(Hu," is the speech signal loaded - and potentially resampled - as a 1D array."),Hu.forEach(e),Po=h(Ke),xe=r(Ke,"LI",{});var Uu=o(xe);Cn=r(Uu,"CODE",{});var Yh=o(Cn);To=t(Yh,"path"),Yh.forEach(e),zo=t(Uu," points to the location of the audio file."),Uu.forEach(e),Co=h(Ke),qe=r(Ke,"LI",{});var Yu=o(qe);Dn=r(Yu,"CODE",{});var Gh=o(Dn);Do=t(Gh,"sampling_rate"),Gh.forEach(e),Io=t(Yu," refers to how many data points in the speech signal are measured per second."),Yu.forEach(e),Ke.forEach(e),ll=h(s),$s=r(s,"P",{});var wp=o($s);Oo=t(wp,"For this tutorial, you\u2019ll use the "),pa=r(wp,"A",{href:!0,rel:!0});var Vh=o(pa);Lo=t(Vh,"Wav2Vec2"),Vh.forEach(e),So=t(wp," model. Take a look at the model card, and you\u2019ll learn Wav2Vec2 is pretrained on 16kHz sampled speech audio. It is important your audio data\u2019s sampling rate matches the sampling rate of the dataset used to pretrain the model. If your data\u2019s sampling rate isn\u2019t the same, then you need to resample your data."),wp.forEach(e),pl=h(s),Ae=r(s,"OL",{});var Kh=o(Ae);ra=r(Kh,"LI",{});var yp=o(ra);No=t(yp,"Use \u{1F917} Datasets\u2019 "),oa=r(yp,"A",{href:!0,rel:!0});var Qh=o(oa);Fo=t(Qh,"cast_column"),Qh.forEach(e),Ro=t(yp," method to upsample the sampling rate to 16kHz:"),yp.forEach(e),Kh.forEach(e),rl=h(s),f(ca.$$.fragment,s),ol=h(s),ua=r(s,"OL",{start:!0});var Xh=o(ua);ha=r(Xh,"LI",{});var kp=o(ha);Bo=t(kp,"Call the "),In=r(kp,"CODE",{});var Zh=o(In);Mo=t(Zh,"audio"),Zh.forEach(e),Wo=t(kp," column again to resample the audio file:"),kp.forEach(e),Xh.forEach(e),cl=h(s),f(ma.$$.fragment,s),ul=h(s),C=r(s,"P",{});var Is=o(C);Jo=t(Is,"Next, load a feature extractor to normalize and pad the input. When padding textual data, a "),On=r(Is,"CODE",{});var sm=o(On);Ho=t(sm,"0"),sm.forEach(e),Uo=t(Is," is added for shorter sequences. The same idea applies to audio data. The feature extractor adds a "),Ln=r(Is,"CODE",{});var am=o(Ln);Yo=t(am,"0"),am.forEach(e),Go=t(Is," - interpreted as silence - to "),Sn=r(Is,"CODE",{});var em=o(Sn);Vo=t(em,"array"),em.forEach(e),Ko=t(Is,"."),Is.forEach(e),hl=h(s),vs=r(s,"P",{});var Ep=o(vs);Qo=t(Ep,"Load the feature extractor with "),Pe=r(Ep,"A",{href:!0});var nm=o(Pe);Xo=t(nm,"AutoFeatureExtractor.from_pretrained()"),nm.forEach(e),Zo=t(Ep,":"),Ep.forEach(e),ml=h(s),f(ia.$$.fragment,s),il=h(s),U=r(s,"P",{});var Qe=o(U);sc=t(Qe,"Pass the audio "),Nn=r(Qe,"CODE",{});var tm=o(Nn);ac=t(tm,"array"),tm.forEach(e),ec=t(Qe," to the feature extractor. We also recommend adding the "),Fn=r(Qe,"CODE",{});var lm=o(Fn);nc=t(lm,"sampling_rate"),lm.forEach(e),tc=t(Qe," argument in the feature extractor in order to better debug any silent errors that may occur."),Qe.forEach(e),bl=h(s),f(ba.$$.fragment,s),jl=h(s),Te=r(s,"P",{});var pm=o(Te);lc=t(pm,"Just like the tokenizer, you can apply padding or truncation to handle variable sequences in a batch. Take a look at the sequence length of these two audio samples:"),pm.forEach(e),fl=h(s),f(ja.$$.fragment,s),dl=h(s),ze=r(s,"P",{});var rm=o(ze);pc=t(rm,"Create a function to preprocess the dataset so the audio samples are the same lengths. Specify a maximum sample length, and the feature extractor will either pad or truncate the sequences to match it:"),rm.forEach(e),gl=h(s),f(fa.$$.fragment,s),_l=h(s),ws=r(s,"P",{});var xp=o(ws);rc=t(xp,"Apply the "),Rn=r(xp,"CODE",{});var om=o(Rn);oc=t(om,"preprocess_function"),om.forEach(e),cc=t(xp," to the the first few examples in the dataset:"),xp.forEach(e),$l=h(s),f(da.$$.fragment,s),vl=h(s),Ce=r(s,"P",{});var cm=o(Ce);uc=t(cm,"The sample lengths are now the same and match the specified maximum length. You can pass your processed dataset to the model now!"),cm.forEach(e),wl=h(s),f(ga.$$.fragment,s),yl=h(s),ts=r(s,"H2",{class:!0});var qp=o(ts);ys=r(qp,"A",{id:!0,class:!0,href:!0});var um=o(ys);Bn=r(um,"SPAN",{});var hm=o(Bn);f(_a.$$.fragment,hm),hm.forEach(e),um.forEach(e),hc=h(qp),Mn=r(qp,"SPAN",{});var mm=o(Mn);mc=t(mm,"Computer vision"),mm.forEach(e),qp.forEach(e),kl=h(s),ks=r(s,"P",{});var Ap=o(ks);ic=t(Ap,"For computer vision tasks, you\u2019ll need an "),De=r(Ap,"A",{href:!0});var im=o(De);bc=t(im,"image processor"),im.forEach(e),jc=t(Ap," to prepare your dataset for the model. The image processor is designed to preprocess images, and convert them into tensors."),Ap.forEach(e),El=h(s),Y=r(s,"P",{});var Xe=o(Y);fc=t(Xe,"Load the "),$a=r(Xe,"A",{href:!0,rel:!0});var bm=o($a);dc=t(bm,"food101"),bm.forEach(e),gc=t(Xe," dataset (see the \u{1F917} "),va=r(Xe,"A",{href:!0,rel:!0});var jm=o(va);_c=t(jm,"Datasets tutorial"),jm.forEach(e),$c=t(Xe," for more details on how to load a dataset) to see how you can use an image processor with computer vision datasets:"),Xe.forEach(e),xl=h(s),f(Es.$$.fragment,s),ql=h(s),f(wa.$$.fragment,s),Al=h(s),xs=r(s,"P",{});var Pp=o(xs);vc=t(Pp,"Next, take a look at the image with \u{1F917} Datasets "),ya=r(Pp,"A",{href:!0,rel:!0});var fm=o(ya);Wn=r(fm,"CODE",{});var dm=o(Wn);wc=t(dm,"Image"),dm.forEach(e),fm.forEach(e),yc=t(Pp," feature:"),Pp.forEach(e),Pl=h(s),f(ka.$$.fragment,s),Tl=h(s),Ea=r(s,"DIV",{class:!0});var gm=o(Ea);Jn=r(gm,"IMG",{src:!0}),gm.forEach(e),zl=h(s),qs=r(s,"P",{});var Tp=o(qs);kc=t(Tp,"Load the image processor with "),Ie=r(Tp,"A",{href:!0});var _m=o(Ie);Ec=t(_m,"AutoImageProcessor.from_pretrained()"),_m.forEach(e),xc=t(Tp,":"),Tp.forEach(e),Cl=h(s),f(xa.$$.fragment,s),Dl=h(s),D=r(s,"P",{});var Os=o(D);qc=t(Os,"For computer vision tasks, it is common to add some type of data augmentation to the images as a part of preprocessing. You can add augmentations with any library you\u2019d like, but in this tutorial, you\u2019ll use torchvision\u2019s "),qa=r(Os,"A",{href:!0,rel:!0});var $m=o(qa);Hn=r($m,"CODE",{});var vm=o(Hn);Ac=t(vm,"transforms"),vm.forEach(e),$m.forEach(e),Pc=t(Os," module. If you\u2019re interested in using another data augmentation library, learn how in the "),Aa=r(Os,"A",{href:!0,rel:!0});var wm=o(Aa);Tc=t(wm,"Albumentations"),wm.forEach(e),zc=t(Os," or "),Pa=r(Os,"A",{href:!0,rel:!0});var ym=o(Pa);Cc=t(ym,"Kornia notebooks"),ym.forEach(e),Dc=t(Os,"."),Os.forEach(e),Il=h(s),Oe=r(s,"OL",{});var km=o(Oe);O=r(km,"LI",{});var Ls=o(O);Ic=t(Ls,"Normalize the image with the image processor and use "),Ta=r(Ls,"A",{href:!0,rel:!0});var Em=o(Ta);Un=r(Em,"CODE",{});var xm=o(Un);Oc=t(xm,"Compose"),xm.forEach(e),Em.forEach(e),Lc=t(Ls," to chain some transforms - "),za=r(Ls,"A",{href:!0,rel:!0});var qm=o(za);Yn=r(qm,"CODE",{});var Am=o(Yn);Sc=t(Am,"RandomResizedCrop"),Am.forEach(e),qm.forEach(e),Nc=t(Ls," and "),Ca=r(Ls,"A",{href:!0,rel:!0});var Pm=o(Ca);Gn=r(Pm,"CODE",{});var Tm=o(Gn);Fc=t(Tm,"ColorJitter"),Tm.forEach(e),Pm.forEach(e),Rc=t(Ls," - together:"),Ls.forEach(e),km.forEach(e),Ol=h(s),f(Da.$$.fragment,s),Ll=h(s),Ia=r(s,"OL",{start:!0});var zm=o(Ia);ls=r(zm,"LI",{});var Ze=o(ls);Bc=t(Ze,"The model accepts "),Le=r(Ze,"A",{href:!0});var Cm=o(Le);Vn=r(Cm,"CODE",{});var Dm=o(Vn);Mc=t(Dm,"pixel_values"),Dm.forEach(e),Cm.forEach(e),Wc=t(Ze," as its input, which is generated by the image processor. Create a function that generates "),Kn=r(Ze,"CODE",{});var Im=o(Kn);Jc=t(Im,"pixel_values"),Im.forEach(e),Hc=t(Ze," from the transforms:"),Ze.forEach(e),zm.forEach(e),Sl=h(s),f(Oa.$$.fragment,s),Nl=h(s),La=r(s,"OL",{start:!0});var Om=o(La);Sa=r(Om,"LI",{});var zp=o(Sa);Uc=t(zp,"Then use \u{1F917} Datasets "),Na=r(zp,"A",{href:!0,rel:!0});var Lm=o(Na);Qn=r(Lm,"CODE",{});var Sm=o(Qn);Yc=t(Sm,"set_transform"),Sm.forEach(e),Lm.forEach(e),Gc=t(zp," to apply the transforms on the fly:"),zp.forEach(e),Om.forEach(e),Fl=h(s),f(Fa.$$.fragment,s),Rl=h(s),Ra=r(s,"OL",{start:!0});var Nm=o(Ra);Ba=r(Nm,"LI",{});var Cp=o(Ba);Vc=t(Cp,"Now when you access the image, you\u2019ll notice the image processor has added "),Xn=r(Cp,"CODE",{});var Fm=o(Xn);Kc=t(Fm,"pixel_values"),Fm.forEach(e),Qc=t(Cp,". You can pass your processed dataset to the model now!"),Cp.forEach(e),Nm.forEach(e),Bl=h(s),f(Ma.$$.fragment,s),Ml=h(s),Se=r(s,"P",{});var Rm=o(Se);Xc=t(Rm,"Here is what the image looks like after the transforms are applied. The image has been randomly cropped and it\u2019s color properties are different."),Rm.forEach(e),Wl=h(s),f(Wa.$$.fragment,s),Jl=h(s),Ja=r(s,"DIV",{class:!0});var Bm=o(Ja);Zn=r(Bm,"IMG",{src:!0}),Bm.forEach(e),Hl=h(s),ps=r(s,"H2",{class:!0});var Dp=o(ps);As=r(Dp,"A",{id:!0,class:!0,href:!0});var Mm=o(As);st=r(Mm,"SPAN",{});var Wm=o(st);f(Ha.$$.fragment,Wm),Wm.forEach(e),Mm.forEach(e),Zc=h(Dp),at=r(Dp,"SPAN",{});var Jm=o(at);su=t(Jm,"Multimodal"),Jm.forEach(e),Dp.forEach(e),Ul=h(s),Ps=r(s,"P",{});var Ip=o(Ps);au=t(Ip,"For tasks involving multimodal inputs, you\u2019ll need a "),Ne=r(Ip,"A",{href:!0});var Hm=o(Ne);eu=t(Hm,"processor"),Hm.forEach(e),nu=t(Ip," to prepare your dataset for the model. A processor couples together two processing objects such as as tokenizer and feature extractor."),Ip.forEach(e),Yl=h(s),G=r(s,"P",{});var sn=o(G);tu=t(sn,"Load the "),Ua=r(sn,"A",{href:!0,rel:!0});var Um=o(Ua);lu=t(Um,"LJ Speech"),Um.forEach(e),pu=t(sn," dataset (see the \u{1F917} "),Ya=r(sn,"A",{href:!0,rel:!0});var Ym=o(Ya);ru=t(Ym,"Datasets tutorial"),Ym.forEach(e),ou=t(sn," for more details on how to load a dataset) to see how you can use a processor for automatic speech recognition (ASR):"),sn.forEach(e),Gl=h(s),f(Ga.$$.fragment,s),Vl=h(s),V=r(s,"P",{});var an=o(V);cu=t(an,"For ASR, you\u2019re mainly focused on "),et=r(an,"CODE",{});var Gm=o(et);uu=t(Gm,"audio"),Gm.forEach(e),hu=t(an," and "),nt=r(an,"CODE",{});var Vm=o(nt);mu=t(Vm,"text"),Vm.forEach(e),iu=t(an," so you can remove the other columns:"),an.forEach(e),Kl=h(s),f(Va.$$.fragment,s),Ql=h(s),K=r(s,"P",{});var en=o(K);bu=t(en,"Now take a look at the "),tt=r(en,"CODE",{});var Km=o(tt);ju=t(Km,"audio"),Km.forEach(e),fu=t(en," and "),lt=r(en,"CODE",{});var Qm=o(lt);du=t(Qm,"text"),Qm.forEach(e),gu=t(en," columns:"),en.forEach(e),Xl=h(s),f(Ka.$$.fragment,s),Zl=h(s),Ts=r(s,"P",{});var Op=o(Ts);_u=t(Op,"Remember you should always "),Fe=r(Op,"A",{href:!0});var Xm=o(Fe);$u=t(Xm,"resample"),Xm.forEach(e),vu=t(Op," your audio dataset\u2019s sampling rate to match the sampling rate of the dataset used to pretrain a model!"),Op.forEach(e),sp=h(s),f(Qa.$$.fragment,s),ap=h(s),zs=r(s,"P",{});var Lp=o(zs);wu=t(Lp,"Load a processor with "),Re=r(Lp,"A",{href:!0});var Zm=o(Re);yu=t(Zm,"AutoProcessor.from_pretrained()"),Zm.forEach(e),ku=t(Lp,":"),Lp.forEach(e),ep=h(s),f(Xa.$$.fragment,s),np=h(s),Be=r(s,"OL",{});var si=o(Be);P=r(si,"LI",{});var X=o(P);Eu=t(X,"Create a function to process the audio data contained in "),pt=r(X,"CODE",{});var ai=o(pt);xu=t(ai,"array"),ai.forEach(e),qu=t(X," to "),rt=r(X,"CODE",{});var ei=o(rt);Au=t(ei,"input_values"),ei.forEach(e),Pu=t(X,", and tokenize "),ot=r(X,"CODE",{});var ni=o(ot);Tu=t(ni,"text"),ni.forEach(e),zu=t(X," to "),ct=r(X,"CODE",{});var ti=o(ct);Cu=t(ti,"labels"),ti.forEach(e),Du=t(X,". These are the inputs to the model:"),X.forEach(e),si.forEach(e),tp=h(s),f(Za.$$.fragment,s),lp=h(s),se=r(s,"OL",{start:!0});var li=o(se);ae=r(li,"LI",{});var Sp=o(ae);Iu=t(Sp,"Apply the "),ut=r(Sp,"CODE",{});var pi=o(ut);Ou=t(pi,"prepare_dataset"),pi.forEach(e),Lu=t(Sp," function to a sample:"),Sp.forEach(e),li.forEach(e),pp=h(s),f(ee.$$.fragment,s),rp=h(s),Q=r(s,"P",{});var nn=o(Q);Su=t(nn,"The processor has now added "),ht=r(nn,"CODE",{});var ri=o(ht);Nu=t(ri,"input_values"),ri.forEach(e),Fu=t(nn," and "),mt=r(nn,"CODE",{});var oi=o(mt);Ru=t(oi,"labels"),oi.forEach(e),Bu=t(nn,", and the sampling rate has also been correctly downsampled to 16kHz. You can pass your processed dataset to the model now!"),nn.forEach(e),this.h()},h(){m(i,"name","hf:doc:metadata"),m(i,"content",JSON.stringify(Pi)),m(w,"id","preprocess"),m(w,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(w,"href","#preprocess"),m(b,"class","relative group"),m(le,"href","./main_classes/tokenizer"),m(pe,"href","./main_classes/image"),m(re,"href","./main_classes/feature_extractor"),m(oe,"href","./main_classes/processors"),m(os,"id","natural-language-processing"),m(os,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(os,"href","#natural-language-processing"),m(Z,"class","relative group"),m(ue,"href","main_classes/tokenizer"),m(he,"href","/docs/transformers/v4.25.1/en/model_doc/auto#transformers.AutoTokenizer.from_pretrained"),m(je,"href","glossary#input-ids"),m(de,"href","glossary#attention-mask"),m(_e,"href","glossary#token-type-ids"),m(hs,"id","pad"),m(hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(hs,"href","#pad"),m(ss,"class","relative group"),m(bs,"id","truncation"),m(bs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(bs,"href","#truncation"),m(as,"class","relative group"),m(fs,"id","build-tensors"),m(fs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(fs,"href","#build-tensors"),m(es,"class","relative group"),m(gs,"id","audio"),m(gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(gs,"href","#audio"),m(ns,"class","relative group"),m(ye,"href","main_classes/feature_extractor"),m(ea,"href","https://huggingface.co/datasets/PolyAI/minds14"),m(ea,"rel","nofollow"),m(na,"href","https://huggingface.co/docs/datasets/load_hub.html"),m(na,"rel","nofollow"),m(pa,"href","https://huggingface.co/facebook/wav2vec2-base"),m(pa,"rel","nofollow"),m(oa,"href","https://huggingface.co/docs/datasets/v2.7.1/en/package_reference/main_classes#datasets.Dataset.cast_column"),m(oa,"rel","nofollow"),m(ua,"start","2"),m(Pe,"href","/docs/transformers/v4.25.1/en/model_doc/auto#transformers.AutoFeatureExtractor.from_pretrained"),m(ys,"id","computer-vision"),m(ys,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(ys,"href","#computer-vision"),m(ts,"class","relative group"),m(De,"href","main_classes/image_processor"),m($a,"href","https://huggingface.co/datasets/food101"),m($a,"rel","nofollow"),m(va,"href","https://huggingface.co/docs/datasets/load_hub.html"),m(va,"rel","nofollow"),m(ya,"href","https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=image#datasets.Image"),m(ya,"rel","nofollow"),ci(Jn.src,Gu="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/vision-preprocess-tutorial.png")||m(Jn,"src",Gu),m(Ea,"class","flex justify-center"),m(Ie,"href","/docs/transformers/v4.25.1/en/model_doc/auto#transformers.AutoImageProcessor.from_pretrained"),m(qa,"href","https://pytorch.org/vision/stable/transforms.html"),m(qa,"rel","nofollow"),m(Aa,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_albumentations.ipynb"),m(Aa,"rel","nofollow"),m(Pa,"href","https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification_kornia.ipynb"),m(Pa,"rel","nofollow"),m(Ta,"href","https://pytorch.org/vision/master/generated/torchvision.transforms.Compose.html"),m(Ta,"rel","nofollow"),m(za,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.RandomResizedCrop.html"),m(za,"rel","nofollow"),m(Ca,"href","https://pytorch.org/vision/main/generated/torchvision.transforms.ColorJitter.html"),m(Ca,"rel","nofollow"),m(Le,"href","model_doc/visionencoderdecoder#transformers.VisionEncoderDecoderModel.forward.pixel_values"),m(Ia,"start","2"),m(Na,"href","https://huggingface.co/docs/datasets/process.html#format-transform"),m(Na,"rel","nofollow"),m(La,"start","3"),m(Ra,"start","4"),ci(Zn.src,Vu="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/preprocessed_image.png")||m(Zn,"src",Vu),m(Ja,"class","flex justify-center"),m(As,"id","multimodal"),m(As,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),m(As,"href","#multimodal"),m(ps,"class","relative group"),m(Ne,"href","main_classes/processors"),m(Ua,"href","https://huggingface.co/datasets/lj_speech"),m(Ua,"rel","nofollow"),m(Ya,"href","https://huggingface.co/docs/datasets/load_hub.html"),m(Ya,"rel","nofollow"),m(Fe,"href","preprocessing#audio"),m(Re,"href","/docs/transformers/v4.25.1/en/model_doc/auto#transformers.AutoProcessor.from_pretrained"),m(se,"start","2")},m(s,l){a(document.head,i),c(s,v,l),c(s,b,l),a(b,w),a(w,k),d(E,k,null),a(b,x),a(b,q),a(q,I),c(s,Ns,l),d(L,s,l),c(s,dt,l),c(s,te,l),a(te,Fp),c(s,gt,l),c(s,T,l),a(T,Fs),a(Fs,Rp),a(Fs,le),a(le,Bp),a(Fs,Mp),a(T,Wp),a(T,Rs),a(Rs,Jp),a(Rs,pe),a(pe,Hp),a(Rs,Up),a(T,Yp),a(T,Bs),a(Bs,Gp),a(Bs,re),a(re,Vp),a(Bs,Kp),a(T,Qp),a(T,Ms),a(Ms,Xp),a(Ms,oe),a(oe,Zp),a(Ms,sr),c(s,_t,l),d(rs,s,l),c(s,$t,l),c(s,ce,l),a(ce,ar),c(s,vt,l),d(Ws,s,l),c(s,wt,l),c(s,Z,l),a(Z,os),a(os,tn),d(Js,tn,null),a(Z,er),a(Z,ln),a(ln,nr),c(s,yt,l),d(Hs,s,l),c(s,kt,l),c(s,S,l),a(S,tr),a(S,ue),a(ue,lr),a(S,pr),a(S,pn),a(pn,rr),a(S,or),c(s,Et,l),d(cs,s,l),c(s,xt,l),c(s,N,l),a(N,cr),a(N,he),a(he,ur),a(N,hr),a(N,rn),a(rn,mr),a(N,ir),c(s,qt,l),d(Us,s,l),c(s,At,l),c(s,me,l),a(me,br),c(s,Pt,l),d(Ys,s,l),c(s,Tt,l),c(s,ie,l),a(ie,jr),c(s,zt,l),c(s,F,l),a(F,be),a(be,je),a(je,fr),a(be,dr),a(F,gr),a(F,fe),a(fe,de),a(de,_r),a(fe,$r),a(F,vr),a(F,ge),a(ge,_e),a(_e,wr),a(ge,yr),c(s,Ct,l),c(s,us,l),a(us,kr),a(us,on),a(on,Er),a(us,xr),c(s,Dt,l),d(Gs,s,l),c(s,It,l),c(s,R,l),a(R,qr),a(R,cn),a(cn,Ar),a(R,Pr),a(R,un),a(un,Tr),a(R,zr),c(s,Ot,l),c(s,$e,l),a($e,Cr),c(s,Lt,l),d(Vs,s,l),c(s,St,l),c(s,ss,l),a(ss,hs),a(hs,hn),d(Ks,hn,null),a(ss,Dr),a(ss,mn),a(mn,Ir),c(s,Nt,l),c(s,ms,l),a(ms,Or),a(ms,bn),a(bn,Lr),a(ms,Sr),c(s,Ft,l),c(s,B,l),a(B,Nr),a(B,jn),a(jn,Fr),a(B,Rr),a(B,fn),a(fn,Br),a(B,Mr),c(s,Rt,l),d(Qs,s,l),c(s,Bt,l),c(s,is,l),a(is,Wr),a(is,dn),a(dn,Jr),a(is,Hr),c(s,Mt,l),c(s,as,l),a(as,bs),a(bs,gn),d(Xs,gn,null),a(as,Ur),a(as,_n),a(_n,Yr),c(s,Wt,l),c(s,ve,l),a(ve,Gr),c(s,Jt,l),c(s,M,l),a(M,Vr),a(M,$n),a($n,Kr),a(M,Qr),a(M,vn),a(vn,Xr),a(M,Zr),c(s,Ht,l),d(Zs,s,l),c(s,Ut,l),d(js,s,l),c(s,Yt,l),c(s,es,l),a(es,fs),a(fs,wn),d(sa,wn,null),a(es,so),a(es,yn),a(yn,ao),c(s,Gt,l),c(s,we,l),a(we,eo),c(s,Vt,l),c(s,z,l),a(z,no),a(z,kn),a(kn,to),a(z,lo),a(z,En),a(En,po),a(z,ro),a(z,xn),a(xn,oo),a(z,co),c(s,Kt,l),d(ds,s,l),c(s,Qt,l),c(s,ns,l),a(ns,gs),a(gs,qn),d(aa,qn,null),a(ns,uo),a(ns,An),a(An,ho),c(s,Xt,l),c(s,_s,l),a(_s,mo),a(_s,ye),a(ye,io),a(_s,bo),c(s,Zt,l),c(s,W,l),a(W,jo),a(W,ea),a(ea,fo),a(W,go),a(W,na),a(na,_o),a(W,$o),c(s,sl,l),d(ta,s,l),c(s,al,l),c(s,J,l),a(J,vo),a(J,Pn),a(Pn,wo),a(J,yo),a(J,Tn),a(Tn,ko),a(J,Eo),c(s,el,l),d(la,s,l),c(s,nl,l),c(s,ke,l),a(ke,xo),c(s,tl,l),c(s,H,l),a(H,Ee),a(Ee,zn),a(zn,qo),a(Ee,Ao),a(H,Po),a(H,xe),a(xe,Cn),a(Cn,To),a(xe,zo),a(H,Co),a(H,qe),a(qe,Dn),a(Dn,Do),a(qe,Io),c(s,ll,l),c(s,$s,l),a($s,Oo),a($s,pa),a(pa,Lo),a($s,So),c(s,pl,l),c(s,Ae,l),a(Ae,ra),a(ra,No),a(ra,oa),a(oa,Fo),a(ra,Ro),c(s,rl,l),d(ca,s,l),c(s,ol,l),c(s,ua,l),a(ua,ha),a(ha,Bo),a(ha,In),a(In,Mo),a(ha,Wo),c(s,cl,l),d(ma,s,l),c(s,ul,l),c(s,C,l),a(C,Jo),a(C,On),a(On,Ho),a(C,Uo),a(C,Ln),a(Ln,Yo),a(C,Go),a(C,Sn),a(Sn,Vo),a(C,Ko),c(s,hl,l),c(s,vs,l),a(vs,Qo),a(vs,Pe),a(Pe,Xo),a(vs,Zo),c(s,ml,l),d(ia,s,l),c(s,il,l),c(s,U,l),a(U,sc),a(U,Nn),a(Nn,ac),a(U,ec),a(U,Fn),a(Fn,nc),a(U,tc),c(s,bl,l),d(ba,s,l),c(s,jl,l),c(s,Te,l),a(Te,lc),c(s,fl,l),d(ja,s,l),c(s,dl,l),c(s,ze,l),a(ze,pc),c(s,gl,l),d(fa,s,l),c(s,_l,l),c(s,ws,l),a(ws,rc),a(ws,Rn),a(Rn,oc),a(ws,cc),c(s,$l,l),d(da,s,l),c(s,vl,l),c(s,Ce,l),a(Ce,uc),c(s,wl,l),d(ga,s,l),c(s,yl,l),c(s,ts,l),a(ts,ys),a(ys,Bn),d(_a,Bn,null),a(ts,hc),a(ts,Mn),a(Mn,mc),c(s,kl,l),c(s,ks,l),a(ks,ic),a(ks,De),a(De,bc),a(ks,jc),c(s,El,l),c(s,Y,l),a(Y,fc),a(Y,$a),a($a,dc),a(Y,gc),a(Y,va),a(va,_c),a(Y,$c),c(s,xl,l),d(Es,s,l),c(s,ql,l),d(wa,s,l),c(s,Al,l),c(s,xs,l),a(xs,vc),a(xs,ya),a(ya,Wn),a(Wn,wc),a(xs,yc),c(s,Pl,l),d(ka,s,l),c(s,Tl,l),c(s,Ea,l),a(Ea,Jn),c(s,zl,l),c(s,qs,l),a(qs,kc),a(qs,Ie),a(Ie,Ec),a(qs,xc),c(s,Cl,l),d(xa,s,l),c(s,Dl,l),c(s,D,l),a(D,qc),a(D,qa),a(qa,Hn),a(Hn,Ac),a(D,Pc),a(D,Aa),a(Aa,Tc),a(D,zc),a(D,Pa),a(Pa,Cc),a(D,Dc),c(s,Il,l),c(s,Oe,l),a(Oe,O),a(O,Ic),a(O,Ta),a(Ta,Un),a(Un,Oc),a(O,Lc),a(O,za),a(za,Yn),a(Yn,Sc),a(O,Nc),a(O,Ca),a(Ca,Gn),a(Gn,Fc),a(O,Rc),c(s,Ol,l),d(Da,s,l),c(s,Ll,l),c(s,Ia,l),a(Ia,ls),a(ls,Bc),a(ls,Le),a(Le,Vn),a(Vn,Mc),a(ls,Wc),a(ls,Kn),a(Kn,Jc),a(ls,Hc),c(s,Sl,l),d(Oa,s,l),c(s,Nl,l),c(s,La,l),a(La,Sa),a(Sa,Uc),a(Sa,Na),a(Na,Qn),a(Qn,Yc),a(Sa,Gc),c(s,Fl,l),d(Fa,s,l),c(s,Rl,l),c(s,Ra,l),a(Ra,Ba),a(Ba,Vc),a(Ba,Xn),a(Xn,Kc),a(Ba,Qc),c(s,Bl,l),d(Ma,s,l),c(s,Ml,l),c(s,Se,l),a(Se,Xc),c(s,Wl,l),d(Wa,s,l),c(s,Jl,l),c(s,Ja,l),a(Ja,Zn),c(s,Hl,l),c(s,ps,l),a(ps,As),a(As,st),d(Ha,st,null),a(ps,Zc),a(ps,at),a(at,su),c(s,Ul,l),c(s,Ps,l),a(Ps,au),a(Ps,Ne),a(Ne,eu),a(Ps,nu),c(s,Yl,l),c(s,G,l),a(G,tu),a(G,Ua),a(Ua,lu),a(G,pu),a(G,Ya),a(Ya,ru),a(G,ou),c(s,Gl,l),d(Ga,s,l),c(s,Vl,l),c(s,V,l),a(V,cu),a(V,et),a(et,uu),a(V,hu),a(V,nt),a(nt,mu),a(V,iu),c(s,Kl,l),d(Va,s,l),c(s,Ql,l),c(s,K,l),a(K,bu),a(K,tt),a(tt,ju),a(K,fu),a(K,lt),a(lt,du),a(K,gu),c(s,Xl,l),d(Ka,s,l),c(s,Zl,l),c(s,Ts,l),a(Ts,_u),a(Ts,Fe),a(Fe,$u),a(Ts,vu),c(s,sp,l),d(Qa,s,l),c(s,ap,l),c(s,zs,l),a(zs,wu),a(zs,Re),a(Re,yu),a(zs,ku),c(s,ep,l),d(Xa,s,l),c(s,np,l),c(s,Be,l),a(Be,P),a(P,Eu),a(P,pt),a(pt,xu),a(P,qu),a(P,rt),a(rt,Au),a(P,Pu),a(P,ot),a(ot,Tu),a(P,zu),a(P,ct),a(ct,Cu),a(P,Du),c(s,tp,l),d(Za,s,l),c(s,lp,l),c(s,se,l),a(se,ae),a(ae,Iu),a(ae,ut),a(ut,Ou),a(ae,Lu),c(s,pp,l),d(ee,s,l),c(s,rp,l),c(s,Q,l),a(Q,Su),a(Q,ht),a(ht,Nu),a(Q,Fu),a(Q,mt),a(mt,Ru),a(Q,Bu),op=!0},p(s,[l]){const ne={};l&2&&(ne.$$scope={dirty:l,ctx:s}),rs.$set(ne);const it={};l&2&&(it.$$scope={dirty:l,ctx:s}),cs.$set(it);const bt={};l&2&&(bt.$$scope={dirty:l,ctx:s}),js.$set(bt);const jt={};l&2&&(jt.$$scope={dirty:l,ctx:s}),ds.$set(jt);const ft={};l&2&&(ft.$$scope={dirty:l,ctx:s}),Es.$set(ft)},i(s){op||(g(E.$$.fragment,s),g(L.$$.fragment,s),g(rs.$$.fragment,s),g(Ws.$$.fragment,s),g(Js.$$.fragment,s),g(Hs.$$.fragment,s),g(cs.$$.fragment,s),g(Us.$$.fragment,s),g(Ys.$$.fragment,s),g(Gs.$$.fragment,s),g(Vs.$$.fragment,s),g(Ks.$$.fragment,s),g(Qs.$$.fragment,s),g(Xs.$$.fragment,s),g(Zs.$$.fragment,s),g(js.$$.fragment,s),g(sa.$$.fragment,s),g(ds.$$.fragment,s),g(aa.$$.fragment,s),g(ta.$$.fragment,s),g(la.$$.fragment,s),g(ca.$$.fragment,s),g(ma.$$.fragment,s),g(ia.$$.fragment,s),g(ba.$$.fragment,s),g(ja.$$.fragment,s),g(fa.$$.fragment,s),g(da.$$.fragment,s),g(ga.$$.fragment,s),g(_a.$$.fragment,s),g(Es.$$.fragment,s),g(wa.$$.fragment,s),g(ka.$$.fragment,s),g(xa.$$.fragment,s),g(Da.$$.fragment,s),g(Oa.$$.fragment,s),g(Fa.$$.fragment,s),g(Ma.$$.fragment,s),g(Wa.$$.fragment,s),g(Ha.$$.fragment,s),g(Ga.$$.fragment,s),g(Va.$$.fragment,s),g(Ka.$$.fragment,s),g(Qa.$$.fragment,s),g(Xa.$$.fragment,s),g(Za.$$.fragment,s),g(ee.$$.fragment,s),op=!0)},o(s){_(E.$$.fragment,s),_(L.$$.fragment,s),_(rs.$$.fragment,s),_(Ws.$$.fragment,s),_(Js.$$.fragment,s),_(Hs.$$.fragment,s),_(cs.$$.fragment,s),_(Us.$$.fragment,s),_(Ys.$$.fragment,s),_(Gs.$$.fragment,s),_(Vs.$$.fragment,s),_(Ks.$$.fragment,s),_(Qs.$$.fragment,s),_(Xs.$$.fragment,s),_(Zs.$$.fragment,s),_(js.$$.fragment,s),_(sa.$$.fragment,s),_(ds.$$.fragment,s),_(aa.$$.fragment,s),_(ta.$$.fragment,s),_(la.$$.fragment,s),_(ca.$$.fragment,s),_(ma.$$.fragment,s),_(ia.$$.fragment,s),_(ba.$$.fragment,s),_(ja.$$.fragment,s),_(fa.$$.fragment,s),_(da.$$.fragment,s),_(ga.$$.fragment,s),_(_a.$$.fragment,s),_(Es.$$.fragment,s),_(wa.$$.fragment,s),_(ka.$$.fragment,s),_(xa.$$.fragment,s),_(Da.$$.fragment,s),_(Oa.$$.fragment,s),_(Fa.$$.fragment,s),_(Ma.$$.fragment,s),_(Wa.$$.fragment,s),_(Ha.$$.fragment,s),_(Ga.$$.fragment,s),_(Va.$$.fragment,s),_(Ka.$$.fragment,s),_(Qa.$$.fragment,s),_(Xa.$$.fragment,s),_(Za.$$.fragment,s),_(ee.$$.fragment,s),op=!1},d(s){e(i),s&&e(v),s&&e(b),$(E),s&&e(Ns),$(L,s),s&&e(dt),s&&e(te),s&&e(gt),s&&e(T),s&&e(_t),$(rs,s),s&&e($t),s&&e(ce),s&&e(vt),$(Ws,s),s&&e(wt),s&&e(Z),$(Js),s&&e(yt),$(Hs,s),s&&e(kt),s&&e(S),s&&e(Et),$(cs,s),s&&e(xt),s&&e(N),s&&e(qt),$(Us,s),s&&e(At),s&&e(me),s&&e(Pt),$(Ys,s),s&&e(Tt),s&&e(ie),s&&e(zt),s&&e(F),s&&e(Ct),s&&e(us),s&&e(Dt),$(Gs,s),s&&e(It),s&&e(R),s&&e(Ot),s&&e($e),s&&e(Lt),$(Vs,s),s&&e(St),s&&e(ss),$(Ks),s&&e(Nt),s&&e(ms),s&&e(Ft),s&&e(B),s&&e(Rt),$(Qs,s),s&&e(Bt),s&&e(is),s&&e(Mt),s&&e(as),$(Xs),s&&e(Wt),s&&e(ve),s&&e(Jt),s&&e(M),s&&e(Ht),$(Zs,s),s&&e(Ut),$(js,s),s&&e(Yt),s&&e(es),$(sa),s&&e(Gt),s&&e(we),s&&e(Vt),s&&e(z),s&&e(Kt),$(ds,s),s&&e(Qt),s&&e(ns),$(aa),s&&e(Xt),s&&e(_s),s&&e(Zt),s&&e(W),s&&e(sl),$(ta,s),s&&e(al),s&&e(J),s&&e(el),$(la,s),s&&e(nl),s&&e(ke),s&&e(tl),s&&e(H),s&&e(ll),s&&e($s),s&&e(pl),s&&e(Ae),s&&e(rl),$(ca,s),s&&e(ol),s&&e(ua),s&&e(cl),$(ma,s),s&&e(ul),s&&e(C),s&&e(hl),s&&e(vs),s&&e(ml),$(ia,s),s&&e(il),s&&e(U),s&&e(bl),$(ba,s),s&&e(jl),s&&e(Te),s&&e(fl),$(ja,s),s&&e(dl),s&&e(ze),s&&e(gl),$(fa,s),s&&e(_l),s&&e(ws),s&&e($l),$(da,s),s&&e(vl),s&&e(Ce),s&&e(wl),$(ga,s),s&&e(yl),s&&e(ts),$(_a),s&&e(kl),s&&e(ks),s&&e(El),s&&e(Y),s&&e(xl),$(Es,s),s&&e(ql),$(wa,s),s&&e(Al),s&&e(xs),s&&e(Pl),$(ka,s),s&&e(Tl),s&&e(Ea),s&&e(zl),s&&e(qs),s&&e(Cl),$(xa,s),s&&e(Dl),s&&e(D),s&&e(Il),s&&e(Oe),s&&e(Ol),$(Da,s),s&&e(Ll),s&&e(Ia),s&&e(Sl),$(Oa,s),s&&e(Nl),s&&e(La),s&&e(Fl),$(Fa,s),s&&e(Rl),s&&e(Ra),s&&e(Bl),$(Ma,s),s&&e(Ml),s&&e(Se),s&&e(Wl),$(Wa,s),s&&e(Jl),s&&e(Ja),s&&e(Hl),s&&e(ps),$(Ha),s&&e(Ul),s&&e(Ps),s&&e(Yl),s&&e(G),s&&e(Gl),$(Ga,s),s&&e(Vl),s&&e(V),s&&e(Kl),$(Va,s),s&&e(Ql),s&&e(K),s&&e(Xl),$(Ka,s),s&&e(Zl),s&&e(Ts),s&&e(sp),$(Qa,s),s&&e(ap),s&&e(zs),s&&e(ep),$(Xa,s),s&&e(np),s&&e(Be),s&&e(tp),$(Za,s),s&&e(lp),s&&e(se),s&&e(pp),$(ee,s),s&&e(rp),s&&e(Q)}}}const Pi={local:"preprocess",sections:[{local:"natural-language-processing",sections:[{local:"pad",title:"Pad"},{local:"truncation",title:"Truncation"},{local:"build-tensors",title:"Build tensors"}],title:"Natural Language Processing"},{local:"audio",title:"Audio"},{local:"computer-vision",title:"Computer vision"},{local:"multimodal",title:"Multimodal"}],title:"Preprocess"};function Ti(A){return fi(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ni extends mi{constructor(i){super();ii(this,i,Ti,Ai,bi,{})}}export{Ni as default,Pi as metadata};
