import{S as Vp,i as Bp,s as Yp,e as r,k as f,w as k,t,M as Jp,c as l,d as s,m as d,a as o,x,h as n,b as h,G as e,g as u,y as E,q,o as A,B as T,v as Qp,L as Qs}from"../chunks/vendor-hf-doc-builder.js";import{T as Kp}from"../chunks/Tip-hf-doc-builder.js";import{I as Js}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as F}from"../chunks/CodeBlock-hf-doc-builder.js";import{C as Zp}from"../chunks/CodeBlockFw-hf-doc-builder.js";import{F as ho,M as Ks}from"../chunks/Markdown-hf-doc-builder.js";import"../chunks/IconTensorflow-hf-doc-builder.js";function Xp(N){let i,g,c,m,$;return{c(){i=r("p"),g=t("Check out "),c=r("a"),m=t("Chapter 5"),$=t(" of the Hugging Face course to learn more about other important topics such as loading remote or local datasets, tools for cleaning up a dataset, and creating your own dataset."),this.h()},l(j){i=l(j,"P",{});var S=o(i);g=n(S,"Check out "),c=l(S,"A",{href:!0,rel:!0});var D=o(c);m=n(D,"Chapter 5"),D.forEach(s),$=n(S," of the Hugging Face course to learn more about other important topics such as loading remote or local datasets, tools for cleaning up a dataset, and creating your own dataset."),S.forEach(s),this.h()},h(){h(c,"href","https://huggingface.co/course/chapter5/1?fw=pt"),h(c,"rel","nofollow")},m(j,S){u(j,i,S),e(i,g),e(i,c),e(c,m),e(i,$)},d(j){j&&s(i)}}}function ai(N){let i,g,c,m,$,j,S,D,C,_,y,b,w,z,O;return z=new F({props:{code:`from torch.utils.data import DataLoader

dataset.set_format(type="torch", columns=["input_values", "labels"])
dataloader = DataLoader(dataset, batch_size=4)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_format(<span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;torch&quot;</span>, columns=[<span class="hljs-string">&quot;input_values&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(dataset, batch_size=<span class="hljs-number">4</span>)`}}),{c(){i=r("p"),g=t("Use the "),c=r("a"),m=t("set_format()"),$=t(" function to set the dataset format to "),j=r("code"),S=t("torch"),D=t(" and specify the columns you want to format. This function applies formatting on-the-fly. After converting to PyTorch tensors, wrap the dataset in "),C=r("a"),_=r("code"),y=t("torch.utils.data.DataLoader"),b=t(":"),w=f(),k(z.$$.fragment),this.h()},l(v){i=l(v,"P",{});var P=o(i);g=n(P,"Use the "),c=l(P,"A",{href:!0});var ea=o(c);m=n(ea,"set_format()"),ea.forEach(s),$=n(P," function to set the dataset format to "),j=l(P,"CODE",{});var M=o(j);S=n(M,"torch"),M.forEach(s),D=n(P," and specify the columns you want to format. This function applies formatting on-the-fly. After converting to PyTorch tensors, wrap the dataset in "),C=l(P,"A",{href:!0,rel:!0});var sa=o(C);_=l(sa,"CODE",{});var ta=o(_);y=n(ta,"torch.utils.data.DataLoader"),ta.forEach(s),sa.forEach(s),b=n(P,":"),P.forEach(s),w=d(v),x(z.$$.fragment,v),this.h()},h(){h(c,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.set_format"),h(C,"href","https://alband.github.io/doc_view/data.html?highlight=torch%20utils%20data%20dataloader#torch.utils.data.DataLoader"),h(C,"rel","nofollow")},m(v,P){u(v,i,P),e(i,g),e(i,c),e(c,m),e(i,$),e(i,j),e(j,S),e(i,D),e(i,C),e(C,_),e(_,y),e(i,b),u(v,w,P),E(z,v,P),O=!0},p:Qs,i(v){O||(q(z.$$.fragment,v),O=!0)},o(v){A(z.$$.fragment,v),O=!1},d(v){v&&s(i),v&&s(w),T(z,v)}}}function ei(N){let i,g;return i=new Ks({props:{$$slots:{default:[ai]},$$scope:{ctx:N}}}),{c(){k(i.$$.fragment)},l(c){x(i.$$.fragment,c)},m(c,m){E(i,c,m),g=!0},p(c,m){const $={};m&2&&($.$$scope={dirty:m,ctx:c}),i.$set($)},i(c){g||(q(i.$$.fragment,c),g=!0)},o(c){A(i.$$.fragment,c),g=!1},d(c){T(i,c)}}}function si(N){let i,g,c,m,$,j,S,D,C,_,y;return _=new F({props:{code:`import tensorflow as tf

tf_dataset = dataset.to_tf_dataset(
    columns=["input_values"],
    label_cols=["labels"],
    batch_size=4,
    shuffle=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_dataset = dataset.to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;input_values&quot;</span>],
<span class="hljs-meta">... </span>    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>)`}}),{c(){i=r("p"),g=t("Use the "),c=r("a"),m=t("to_tf_dataset()"),$=t(" function to set the dataset format to be compatible with TensorFlow. You\u2019ll also need to import a "),j=r("a"),S=t("data collator"),D=t(" from \u{1F917} Transformers to combine the varying sequence lengths into a single batch of equal lengths:"),C=f(),k(_.$$.fragment),this.h()},l(b){i=l(b,"P",{});var w=o(i);g=n(w,"Use the "),c=l(w,"A",{href:!0});var z=o(c);m=n(z,"to_tf_dataset()"),z.forEach(s),$=n(w," function to set the dataset format to be compatible with TensorFlow. You\u2019ll also need to import a "),j=l(w,"A",{href:!0,rel:!0});var O=o(j);S=n(O,"data collator"),O.forEach(s),D=n(w," from \u{1F917} Transformers to combine the varying sequence lengths into a single batch of equal lengths:"),w.forEach(s),C=d(b),x(_.$$.fragment,b),this.h()},h(){h(c,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.to_tf_dataset"),h(j,"href","https://huggingface.co/docs/transformers/main_classes/data_collator#transformers.DataCollatorWithPadding"),h(j,"rel","nofollow")},m(b,w){u(b,i,w),e(i,g),e(i,c),e(c,m),e(i,$),e(i,j),e(j,S),e(i,D),u(b,C,w),E(_,b,w),y=!0},p:Qs,i(b){y||(q(_.$$.fragment,b),y=!0)},o(b){A(_.$$.fragment,b),y=!1},d(b){b&&s(i),b&&s(C),T(_,b)}}}function ti(N){let i,g;return i=new Ks({props:{$$slots:{default:[si]},$$scope:{ctx:N}}}),{c(){k(i.$$.fragment)},l(c){x(i.$$.fragment,c)},m(c,m){E(i,c,m),g=!0},p(c,m){const $={};m&2&&($.$$scope={dirty:m,ctx:c}),i.$set($)},i(c){g||(q(i.$$.fragment,c),g=!0)},o(c){A(i.$$.fragment,c),g=!1},d(c){T(i,c)}}}function ni(N){let i,g,c,m,$,j,S,D,C;return D=new F({props:{code:`from torch.utils.data import DataLoader

def collate_fn(examples):
    images = []
    labels = []
    for example in examples:
        images.append((example["pixel_values"]))
        labels.append(example["labels"])
        
    pixel_values = torch.stack(images)
    labels = torch.tensor(labels)
    return {"pixel_values": pixel_values, "labels": labels}
dataloader = DataLoader(dataset, collate_fn=collate_fn, batch_size=4)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">collate_fn</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    images = []
<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> example <span class="hljs-keyword">in</span> examples:
<span class="hljs-meta">... </span>        images.append((example[<span class="hljs-string">&quot;pixel_values&quot;</span>]))
<span class="hljs-meta">... </span>        labels.append(example[<span class="hljs-string">&quot;labels&quot;</span>])
<span class="hljs-meta">... </span>        
<span class="hljs-meta">... </span>    pixel_values = torch.stack(images)
<span class="hljs-meta">... </span>    labels = torch.tensor(labels)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {<span class="hljs-string">&quot;pixel_values&quot;</span>: pixel_values, <span class="hljs-string">&quot;labels&quot;</span>: labels}
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(dataset, collate_fn=collate_fn, batch_size=<span class="hljs-number">4</span>)`}}),{c(){i=r("p"),g=t("Wrap the dataset in "),c=r("a"),m=r("code"),$=t("torch.utils.data.DataLoader"),j=t(". You\u2019ll also need to create a collate function to collate the samples into batches:"),S=f(),k(D.$$.fragment),this.h()},l(_){i=l(_,"P",{});var y=o(i);g=n(y,"Wrap the dataset in "),c=l(y,"A",{href:!0,rel:!0});var b=o(c);m=l(b,"CODE",{});var w=o(m);$=n(w,"torch.utils.data.DataLoader"),w.forEach(s),b.forEach(s),j=n(y,". You\u2019ll also need to create a collate function to collate the samples into batches:"),y.forEach(s),S=d(_),x(D.$$.fragment,_),this.h()},h(){h(c,"href","https://alband.github.io/doc_view/data.html?highlight=torch%20utils%20data%20dataloader#torch.utils.data.DataLoader"),h(c,"rel","nofollow")},m(_,y){u(_,i,y),e(i,g),e(i,c),e(c,m),e(m,$),e(i,j),u(_,S,y),E(D,_,y),C=!0},p:Qs,i(_){C||(q(D.$$.fragment,_),C=!0)},o(_){A(D.$$.fragment,_),C=!1},d(_){_&&s(i),_&&s(S),T(D,_)}}}function ri(N){let i,g;return i=new Ks({props:{$$slots:{default:[ni]},$$scope:{ctx:N}}}),{c(){k(i.$$.fragment)},l(c){x(i.$$.fragment,c)},m(c,m){E(i,c,m),g=!0},p(c,m){const $={};m&2&&($.$$scope={dirty:m,ctx:c}),i.$set($)},i(c){g||(q(i.$$.fragment,c),g=!0)},o(c){A(i.$$.fragment,c),g=!1},d(c){T(i,c)}}}function li(N){let i,g,c,m,$,j,S,D,C,_,y,b,w,z,O;return z=new F({props:{code:`import torch

dataset.set_format(type="torch", columns=["input_ids", "token_type_ids", "attention_mask", "labels"])
dataloader = torch.utils.data.DataLoader(dataset, batch_size=32)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_format(<span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;torch&quot;</span>, columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>, <span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = torch.utils.data.DataLoader(dataset, batch_size=<span class="hljs-number">32</span>)`}}),{c(){i=r("p"),g=t("Use the "),c=r("a"),m=t("set_format()"),$=t(" function to set the dataset format to "),j=r("code"),S=t("torch"),D=t(" and specify the columns you want to format. This function applies formatting on-the-fly. After converting to PyTorch tensors, wrap the dataset in "),C=r("a"),_=r("code"),y=t("torch.utils.data.DataLoader"),b=t(":"),w=f(),k(z.$$.fragment),this.h()},l(v){i=l(v,"P",{});var P=o(i);g=n(P,"Use the "),c=l(P,"A",{href:!0});var ea=o(c);m=n(ea,"set_format()"),ea.forEach(s),$=n(P," function to set the dataset format to "),j=l(P,"CODE",{});var M=o(j);S=n(M,"torch"),M.forEach(s),D=n(P," and specify the columns you want to format. This function applies formatting on-the-fly. After converting to PyTorch tensors, wrap the dataset in "),C=l(P,"A",{href:!0,rel:!0});var sa=o(C);_=l(sa,"CODE",{});var ta=o(_);y=n(ta,"torch.utils.data.DataLoader"),ta.forEach(s),sa.forEach(s),b=n(P,":"),P.forEach(s),w=d(v),x(z.$$.fragment,v),this.h()},h(){h(c,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.set_format"),h(C,"href","https://alband.github.io/doc_view/data.html?highlight=torch%20utils%20data%20dataloader#torch.utils.data.DataLoader"),h(C,"rel","nofollow")},m(v,P){u(v,i,P),e(i,g),e(i,c),e(c,m),e(i,$),e(i,j),e(j,S),e(i,D),e(i,C),e(C,_),e(_,y),e(i,b),u(v,w,P),E(z,v,P),O=!0},p:Qs,i(v){O||(q(z.$$.fragment,v),O=!0)},o(v){A(z.$$.fragment,v),O=!1},d(v){v&&s(i),v&&s(w),T(z,v)}}}function oi(N){let i,g;return i=new Ks({props:{$$slots:{default:[li]},$$scope:{ctx:N}}}),{c(){k(i.$$.fragment)},l(c){x(i.$$.fragment,c)},m(c,m){E(i,c,m),g=!0},p(c,m){const $={};m&2&&($.$$scope={dirty:m,ctx:c}),i.$set($)},i(c){g||(q(i.$$.fragment,c),g=!0)},o(c){A(i.$$.fragment,c),g=!1},d(c){T(i,c)}}}function pi(N){let i,g,c,m,$,j,S,D,C,_,y;return _=new F({props:{code:`import tensorflow as tf
from transformers import DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors="tf")
tf_dataset = dataset.to_tf_dataset(
    columns=["input_ids", "token_type_ids", "attention_mask"],
    label_cols=["labels"],
    batch_size=2,
    collate_fn=data_collator,
    shuffle=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tf_dataset = dataset.to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>, <span class="hljs-string">&quot;attention_mask&quot;</span>],
<span class="hljs-meta">... </span>    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>)`}}),{c(){i=r("p"),g=t("Use the "),c=r("a"),m=t("to_tf_dataset()"),$=t(" function to set the dataset format to be compatible with TensorFlow. You\u2019ll also need to import a "),j=r("a"),S=t("data collator"),D=t(" from \u{1F917} Transformers to combine the varying sequence lengths into a single batch of equal lengths:"),C=f(),k(_.$$.fragment),this.h()},l(b){i=l(b,"P",{});var w=o(i);g=n(w,"Use the "),c=l(w,"A",{href:!0});var z=o(c);m=n(z,"to_tf_dataset()"),z.forEach(s),$=n(w," function to set the dataset format to be compatible with TensorFlow. You\u2019ll also need to import a "),j=l(w,"A",{href:!0,rel:!0});var O=o(j);S=n(O,"data collator"),O.forEach(s),D=n(w," from \u{1F917} Transformers to combine the varying sequence lengths into a single batch of equal lengths:"),w.forEach(s),C=d(b),x(_.$$.fragment,b),this.h()},h(){h(c,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.to_tf_dataset"),h(j,"href","https://huggingface.co/docs/transformers/main_classes/data_collator#transformers.DataCollatorWithPadding"),h(j,"rel","nofollow")},m(b,w){u(b,i,w),e(i,g),e(i,c),e(c,m),e(i,$),e(i,j),e(j,S),e(i,D),u(b,C,w),E(_,b,w),y=!0},p:Qs,i(b){y||(q(_.$$.fragment,b),y=!0)},o(b){A(_.$$.fragment,b),y=!1},d(b){b&&s(i),b&&s(C),T(_,b)}}}function ii(N){let i,g;return i=new Ks({props:{$$slots:{default:[pi]},$$scope:{ctx:N}}}),{c(){k(i.$$.fragment)},l(c){x(i.$$.fragment,c)},m(c,m){E(i,c,m),g=!0},p(c,m){const $={};m&2&&($.$$scope={dirty:m,ctx:c}),i.$set($)},i(c){g||(q(i.$$.fragment,c),g=!0)},o(c){A(i.$$.fragment,c),g=!1},d(c){T(i,c)}}}function ci(N){let i,g,c,m,$,j,S,D,C,_,y,b,w,z,O,v,P,ea,M,sa,ta,Zs,za,Q,na,qe,fn,dn,Ae,mn,gn,ra,Te,_n,bn,Ce,jn,$n,la,Pe,wn,vn,De,yn,Xs,va,at,Se,kn,et,Na,st,ya,xn,ze,En,qn,tt,Fa,nt,ka,An,Ne,Tn,Cn,rt,Oa,lt,oa,xa,Xe,Ia,Pn,as,Dn,ot,K,Sn,Ra,zn,Nn,La,Fn,On,pt,pa,es,In,Rn,Fe,Ln,Mn,it,Ma,ct,W,ss,Gn,Hn,Ga,Un,Wn,Ha,Vn,Bn,ht,Ua,ut,I,ts,Yn,Jn,Wa,Qn,Kn,ns,Zn,Xn,Oe,ar,er,Ie,sr,tr,ft,Va,dt,G,rs,nr,rr,ls,lr,or,os,pr,ir,ps,cr,hr,mt,Ea,ur,Re,fr,dr,gt,Ba,_t,R,is,mr,gr,Le,_r,br,cs,jr,$r,hs,wr,vr,Ya,yr,kr,bt,Ja,jt,Qa,us,xr,Er,$t,qa,wt,ia,fs,qr,Ar,Ka,Tr,Cr,vt,ca,Aa,ds,Za,Pr,ms,Dr,yt,Z,Sr,Xa,zr,Nr,ae,Fr,Or,kt,ha,gs,Ir,Rr,Me,Lr,Mr,xt,ee,Et,L,_s,Gr,Hr,se,Ur,Wr,te,Vr,Br,ne,Yr,Jr,re,Qr,Kr,qt,le,At,ua,bs,Zr,Xr,js,al,el,Tt,oe,Ct,fa,$s,sl,tl,Ge,nl,rl,Pt,pe,Dt,ie,ws,ll,ol,St,Ta,zt,da,vs,pl,il,ce,cl,hl,Nt,ma,Ca,ys,he,ul,ks,fl,Ft,X,dl,ue,ml,gl,fe,_l,bl,Ot,ga,xs,jl,$l,He,wl,vl,It,de,Rt,V,Es,yl,kl,me,xl,El,ge,ql,Al,Lt,_e,Mt,H,qs,Tl,Cl,As,Pl,Dl,Ts,Sl,zl,Cs,Nl,Fl,Gt,Pa,Ol,Ue,Il,Rl,Ht,be,Ut,U,Ps,Ll,Ml,Ds,Gl,Hl,Ss,Ul,Wl,je,Vl,Bl,Wt,$e,Vt,we,zs,Yl,Jl,Bt,Da,Yt,_a,Ns,Ql,Kl,ve,Zl,Xl,Jt,ba,Sa,Fs,ye,ao,Os,eo,Qt,We,so,Kt,aa,to,Ve,no,ro,Be,lo,oo,Zt;return j=new Js({}),va=new Kp({props:{$$slots:{default:[Xp]},$$scope:{ctx:N}}}),Na=new F({props:{code:"pip install datasets",highlighted:"pip install datasets"}}),Fa=new F({props:{code:"pip install datasets[audio]",highlighted:"pip install datasets[audio]"}}),Oa=new F({props:{code:"pip install datasets[vision]",highlighted:"pip install datasets[vision]"}}),Ia=new Js({}),Ma=new F({props:{code:`from datasets import load_dataset, Audio

dataset = load_dataset("PolyAI/minds14", "en-US", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, <span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),Ua=new F({props:{code:`from transformers import AutoModelForAudioClassification, AutoFeatureExtractor

model = AutoModelForAudioClassification.from_pretrained("facebook/wav2vec2-base")
feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForAudioClassification, AutoFeatureExtractor

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForAudioClassification.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base&quot;</span>)`}}),Va=new F({props:{code:`dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))
dataset[0]["audio"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),Ba=new F({props:{code:`def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]
    inputs = feature_extractor(
        audio_arrays,
        sampling_rate=16000,
        padding=True,
        max_length=100000,
        truncation=True,
    )
    return inputs

dataset = dataset.map(preprocess_function, batched=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays,
<span class="hljs-meta">... </span>        sampling_rate=<span class="hljs-number">16000</span>,
<span class="hljs-meta">... </span>        padding=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>        max_length=<span class="hljs-number">100000</span>,
<span class="hljs-meta">... </span>        truncation=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.<span class="hljs-built_in">map</span>(preprocess_function, batched=<span class="hljs-literal">True</span>)`}}),Ja=new F({props:{code:'dataset = dataset.rename_column("intent_class", "labels")',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.rename_column(<span class="hljs-string">&quot;intent_class&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>)'}}),qa=new ho({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[ti],pytorch:[ei]},$$scope:{ctx:N}}}),Za=new Js({}),ee=new F({props:{code:`from datasets import load_dataset, Image

dataset = load_dataset("beans", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Image

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;beans&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),le=new F({props:{code:`from torchvision.transforms import Compose, ColorJitter, ToTensor

jitter = Compose(
    [ColorJitter(brightness=0.5, hue=0.5), ToTensor()]
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> Compose, ColorJitter, ToTensor

<span class="hljs-meta">&gt;&gt;&gt; </span>jitter = Compose(
<span class="hljs-meta">... </span>    [ColorJitter(brightness=<span class="hljs-number">0.5</span>, hue=<span class="hljs-number">0.5</span>), ToTensor()]
<span class="hljs-meta">... </span>)`}}),oe=new F({props:{code:`def transforms(examples):
    examples["pixel_values"] = [jitter(image.convert("RGB")) for image in examples["image"]]
    return examples`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = [jitter(image.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),pe=new F({props:{code:"dataset = dataset.with_transform(transforms)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.with_transform(transforms)'}}),Ta=new ho({props:{pytorch:!0,tensorflow:!1,jax:!1,$$slots:{pytorch:[ri]},$$scope:{ctx:N}}}),he=new Js({}),de=new F({props:{code:`from datasets import load_dataset

dataset = load_dataset("glue", "mrpc", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;glue&quot;</span>, <span class="hljs-string">&quot;mrpc&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),_e=new Zp({props:{group1:{id:"pt",code:`from transformers import AutoModelForSequenceClassification, AutoTokenizer

model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSequenceClassification, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)`},group2:{id:"tf",code:`from transformers import TFAutoModelForSequenceClassification, AutoTokenizer

model = TFAutoModelForSequenceClassification.from_pretrained("bert-base-uncased")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForSequenceClassification, AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForSequenceClassification.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)`}}}),be=new F({props:{code:`def encode(examples):
    return tokenizer(examples["sentence1"], examples["sentence2"], truncation=True, padding="max_length")

dataset = dataset.map(encode, batched=True)
dataset[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">encode</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer(examples[<span class="hljs-string">&quot;sentence1&quot;</span>], examples[<span class="hljs-string">&quot;sentence2&quot;</span>], truncation=<span class="hljs-literal">True</span>, padding=<span class="hljs-string">&quot;max_length&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.<span class="hljs-built_in">map</span>(encode, batched=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;sentence1&#x27;</span>: <span class="hljs-string">&#x27;Amrozi accused his brother , whom he called &quot; the witness &quot; , of deliberately distorting his evidence .&#x27;</span>,
<span class="hljs-string">&#x27;sentence2&#x27;</span>: <span class="hljs-string">&#x27;Referring to him as only &quot; the witness &quot; , Amrozi accused his brother of deliberately distorting his evidence .&#x27;</span>,
<span class="hljs-string">&#x27;label&#x27;</span>: <span class="hljs-number">1</span>,
<span class="hljs-string">&#x27;idx&#x27;</span>: <span class="hljs-number">0</span>,
<span class="hljs-string">&#x27;input_ids&#x27;</span>: array([  <span class="hljs-number">101</span>,  <span class="hljs-number">7277</span>,  <span class="hljs-number">2180</span>,  <span class="hljs-number">5303</span>,  <span class="hljs-number">4806</span>,  <span class="hljs-number">1117</span>,  <span class="hljs-number">1711</span>,   <span class="hljs-number">117</span>,  <span class="hljs-number">2292</span>, <span class="hljs-number">1119</span>,  <span class="hljs-number">1270</span>,   <span class="hljs-number">107</span>,  <span class="hljs-number">1103</span>,  <span class="hljs-number">7737</span>,   <span class="hljs-number">107</span>,   <span class="hljs-number">117</span>,  <span class="hljs-number">1104</span>,  <span class="hljs-number">9938</span>, <span class="hljs-number">4267</span>, <span class="hljs-number">12223</span>, <span class="hljs-number">21811</span>,  <span class="hljs-number">1117</span>,  <span class="hljs-number">2554</span>,   <span class="hljs-number">119</span>,   <span class="hljs-number">102</span>, <span class="hljs-number">11336</span>,  <span class="hljs-number">6732</span>, <span class="hljs-number">3384</span>,  <span class="hljs-number">1106</span>,  <span class="hljs-number">1140</span>,  <span class="hljs-number">1112</span>,  <span class="hljs-number">1178</span>,   <span class="hljs-number">107</span>,  <span class="hljs-number">1103</span>,  <span class="hljs-number">7737</span>,   <span class="hljs-number">107</span>, <span class="hljs-number">117</span>,  <span class="hljs-number">7277</span>,  <span class="hljs-number">2180</span>,  <span class="hljs-number">5303</span>,  <span class="hljs-number">4806</span>,  <span class="hljs-number">1117</span>,  <span class="hljs-number">1711</span>,  <span class="hljs-number">1104</span>,  <span class="hljs-number">9938</span>, <span class="hljs-number">4267</span>, <span class="hljs-number">12223</span>, <span class="hljs-number">21811</span>,  <span class="hljs-number">1117</span>,  <span class="hljs-number">2554</span>,   <span class="hljs-number">119</span>,   <span class="hljs-number">102</span>]),
<span class="hljs-string">&#x27;token_type_ids&#x27;</span>: array([<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]),
<span class="hljs-string">&#x27;attention_mask&#x27;</span>: array([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>])}`}}),$e=new F({props:{code:'dataset = dataset.map(lambda examples: {"labels": examples["label"]}, batched=True)',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.<span class="hljs-built_in">map</span>(<span class="hljs-keyword">lambda</span> examples: {<span class="hljs-string">&quot;labels&quot;</span>: examples[<span class="hljs-string">&quot;label&quot;</span>]}, batched=<span class="hljs-literal">True</span>)'}}),Da=new ho({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[ii],pytorch:[oi]},$$scope:{ctx:N}}}),ye=new Js({}),{c(){i=r("meta"),g=f(),c=r("h1"),m=r("a"),$=r("span"),k(j.$$.fragment),S=f(),D=r("span"),C=t("Quickstart"),_=f(),y=r("p"),b=t("This quickstart is intended for developers who are ready to dive into the code and see an example of how to integrate \u{1F917} Datasets into their model training workflow. If you\u2019re a beginner, we recommend starting with our "),w=r("a"),z=t("tutorials"),O=t(", where you\u2019ll get a more thorough introduction."),v=f(),P=r("p"),ea=t("Each dataset is unique, and depending on the task, some datasets may require additional steps to prepare it for training. But you can always use \u{1F917} Datasets tools to load and process a dataset. The fastest and easiest way to get started is by loading an existing dataset from the "),M=r("a"),sa=t("Hugging Face Hub"),ta=t(". There are thousands of datasets to choose from, spanning many tasks. Choose the type of dataset you want to work with, and let\u2019s get started!"),Zs=f(),za=r("div"),Q=r("div"),na=r("a"),qe=r("div"),fn=t("Audio"),dn=f(),Ae=r("p"),mn=t("Resample an audio dataset and get it ready for a model to classify what type of banking issue a speaker is calling about."),gn=f(),ra=r("a"),Te=r("div"),_n=t("Vision"),bn=f(),Ce=r("p"),jn=t("Apply data augmentation to an image dataset and get it ready for a model to diagnose disease in bean plants."),$n=f(),la=r("a"),Pe=r("div"),wn=t("NLP"),vn=f(),De=r("p"),yn=t("Tokenize a dataset and get it ready for a model to determine whether a pair of sentences have the same meaning."),Xs=f(),k(va.$$.fragment),at=f(),Se=r("p"),kn=t("Start by installing \u{1F917} Datasets:"),et=f(),k(Na.$$.fragment),st=f(),ya=r("p"),xn=t("To work with audio datasets, install the "),ze=r("a"),En=t("Audio"),qn=t(" feature:"),tt=f(),k(Fa.$$.fragment),nt=f(),ka=r("p"),An=t("To work with image datasets, install the "),Ne=r("a"),Tn=t("Image"),Cn=t(" feature:"),rt=f(),k(Oa.$$.fragment),lt=f(),oa=r("h2"),xa=r("a"),Xe=r("span"),k(Ia.$$.fragment),Pn=f(),as=r("span"),Dn=t("Audio"),ot=f(),K=r("p"),Sn=t("Audio datasets are loaded just like text datasets. However, an audio dataset is preprocessed a bit differently. Instead of a tokenizer, you\u2019ll need a "),Ra=r("a"),zn=t("feature extractor"),Nn=t(". An audio input may also require resampling its sampling rate to match the sampling rate of the pretrained model you\u2019re using. In this quickstart, you\u2019ll prepare the "),La=r("a"),Fn=t("MInDS-14"),On=t(" dataset for a model train on and classify the banking issue a customer is having."),pt=f(),pa=r("p"),es=r("strong"),In=t("1"),Rn=t(". Load the MInDS-14 dataset by providing the "),Fe=r("a"),Ln=t("load_dataset()"),Mn=t(" function with the dataset name, dataset configuration (not all datasets will have a configuration), and a dataset split:"),it=f(),k(Ma.$$.fragment),ct=f(),W=r("p"),ss=r("strong"),Gn=t("2"),Hn=t(". Next, load a pretrained "),Ga=r("a"),Un=t("Wav2Vec2"),Wn=t(" model and its corresponding feature extractor from the "),Ha=r("a"),Vn=t("\u{1F917} Transformers"),Bn=t(" library. It is totally normal to see a warning after you load the model about some weights not being initialized. This is expected because you are loading this model checkpoint for training with another task."),ht=f(),k(Ua.$$.fragment),ut=f(),I=r("p"),ts=r("strong"),Yn=t("3"),Jn=t(". The "),Wa=r("a"),Qn=t("MInDS-14"),Kn=t(" dataset card indicates the sampling rate is 8kHz, but the Wav2Vec2 model was pretrained on a sampling rate of 16kHZ. You\u2019ll need to upsample the "),ns=r("code"),Zn=t("audio"),Xn=t(" column with the "),Oe=r("a"),ar=t("cast_column()"),er=t(" function and "),Ie=r("a"),sr=t("Audio"),tr=t(" feature to match the model\u2019s sampling rate."),ft=f(),k(Va.$$.fragment),dt=f(),G=r("p"),rs=r("strong"),nr=t("4"),rr=t(". Create a function to preprocess the audio "),ls=r("code"),lr=t("array"),or=t(" with the feature extractor, and truncate and pad the sequences into tidy rectangular tensors. The most important thing to remember is to call the audio "),os=r("code"),pr=t("array"),ir=t(" in the feature extractor since the "),ps=r("code"),cr=t("array"),hr=t(" - the actual speech signal - is the model input."),mt=f(),Ea=r("p"),ur=t("Once you have a preprocessing function, use the "),Re=r("a"),fr=t("map()"),dr=t(" function to speed up processing by applying the function to batches of examples in the dataset."),gt=f(),k(Ba.$$.fragment),_t=f(),R=r("p"),is=r("strong"),mr=t("5"),gr=t(". Use the "),Le=r("a"),_r=t("rename_column()"),br=t(" function to rename the "),cs=r("code"),jr=t("intent_class"),$r=t(" column to "),hs=r("code"),wr=t("labels"),vr=t(", which is the expected input name in "),Ya=r("a"),yr=t("Wav2Vec2ForSequenceClassification"),kr=t(":"),bt=f(),k(Ja.$$.fragment),jt=f(),Qa=r("p"),us=r("strong"),xr=t("6"),Er=t(". Set the dataset format according to the machine learning framework you\u2019re using."),$t=f(),k(qa.$$.fragment),wt=f(),ia=r("p"),fs=r("strong"),qr=t("7"),Ar=t(". Start training with your machine learning framework! Check out the \u{1F917} Transformers "),Ka=r("a"),Tr=t("audio classification guide"),Cr=t(" for an end-to-end example of how to train a model on an audio dataset."),vt=f(),ca=r("h2"),Aa=r("a"),ds=r("span"),k(Za.$$.fragment),Pr=f(),ms=r("span"),Dr=t("Vision"),yt=f(),Z=r("p"),Sr=t("Image datasets are loaded just like text datasets. However, instead of a tokenizer, you\u2019ll need a "),Xa=r("a"),zr=t("feature extractor"),Nr=t(" to preprocess the dataset. Applying data augmentation to an image is common in computer vision to make the model more robust against overfitting. You\u2019re free to use any data augmentation library you want, and then you can apply the augmentations with \u{1F917} Datasets. In this quickstart, you\u2019ll load the "),ae=r("a"),Fr=t("Beans"),Or=t(" dataset and get it ready for the model to train on and identify disease from the leaf images."),kt=f(),ha=r("p"),gs=r("strong"),Ir=t("1"),Rr=t(". Load the Beans dataset by providing the "),Me=r("a"),Lr=t("load_dataset()"),Mr=t(" function with the dataset name and a dataset split:"),xt=f(),k(ee.$$.fragment),Et=f(),L=r("p"),_s=r("strong"),Gr=t("2"),Hr=t(". Now you can add some data augmentations with any library ("),se=r("a"),Ur=t("Albumentations"),Wr=t(", "),te=r("a"),Vr=t("imgaug"),Br=t(", "),ne=r("a"),Yr=t("Kornia"),Jr=t(") you like. Here, you\u2019ll use "),re=r("a"),Qr=t("torchvision"),Kr=t(" to randomly change the color properties of an image:"),qt=f(),k(le.$$.fragment),At=f(),ua=r("p"),bs=r("strong"),Zr=t("3"),Xr=t(". Create a function to apply your transform to the dataset and generate the model input: "),js=r("code"),al=t("pixel_values"),el=t("."),Tt=f(),k(oe.$$.fragment),Ct=f(),fa=r("p"),$s=r("strong"),sl=t("4"),tl=t(". Use the "),Ge=r("a"),nl=t("with_transform()"),rl=t(" function to apply the data augmentations on-the-fly:"),Pt=f(),k(pe.$$.fragment),Dt=f(),ie=r("p"),ws=r("strong"),ll=t("5"),ol=t(". Set the dataset format according to the machine learning framework you\u2019re using."),St=f(),k(Ta.$$.fragment),zt=f(),da=r("p"),vs=r("strong"),pl=t("6"),il=t(". Start training with your machine learning framework! Check out the \u{1F917} Transformers "),ce=r("a"),cl=t("image classification guide"),hl=t(" for an end-to-end example of how to train a model on an image dataset."),Nt=f(),ma=r("h2"),Ca=r("a"),ys=r("span"),k(he.$$.fragment),ul=f(),ks=r("span"),fl=t("NLP"),Ft=f(),X=r("p"),dl=t("Text needs to be tokenized into individual tokens by a "),ue=r("a"),ml=t("tokenizer"),gl=t(". For the quickstart, you\u2019ll load the "),fe=r("a"),_l=t("Microsoft Research Paraphrase Corpus (MRPC)"),bl=t(" training dataset to train a model to determine whether a pair of sentences mean the same thing."),Ot=f(),ga=r("p"),xs=r("strong"),jl=t("1"),$l=t(". Load the MRPC dataset by providing the "),He=r("a"),wl=t("load_dataset()"),vl=t(" function with the dataset name, dataset configuration (not all datasets will have a configuration), and dataset split:"),It=f(),k(de.$$.fragment),Rt=f(),V=r("p"),Es=r("strong"),yl=t("2"),kl=t(". Next, load a pretrained "),me=r("a"),xl=t("BERT"),El=t(" model and its corresponding tokenizer from the "),ge=r("a"),ql=t("\u{1F917} Transformers"),Al=t(" library. It is totally normal to see a warning after you load the model about some weights not being initialized. This is expected because you are loading this model checkpoint for training with another task."),Lt=f(),k(_e.$$.fragment),Mt=f(),H=r("p"),qs=r("strong"),Tl=t("3"),Cl=t(". Create a function to tokenize the dataset, and you should also truncate and pad the text into tidy rectangular tensors. The tokenizer generates three new columns in the dataset: "),As=r("code"),Pl=t("input_ids"),Dl=t(", "),Ts=r("code"),Sl=t("token_type_ids"),zl=t(", and an "),Cs=r("code"),Nl=t("attention_mask"),Fl=t(". These are the model inputs."),Gt=f(),Pa=r("p"),Ol=t("Use the "),Ue=r("a"),Il=t("map()"),Rl=t(" function to speed up processing by applying your tokenization function to batches of examples in the dataset:"),Ht=f(),k(be.$$.fragment),Ut=f(),U=r("p"),Ps=r("strong"),Ll=t("4"),Ml=t(". Rename the "),Ds=r("code"),Gl=t("label"),Hl=t(" column to "),Ss=r("code"),Ul=t("labels"),Wl=t(", which is the expected input name in "),je=r("a"),Vl=t("BertForSequenceClassification"),Bl=t(":"),Wt=f(),k($e.$$.fragment),Vt=f(),we=r("p"),zs=r("strong"),Yl=t("5"),Jl=t(". Set the dataset format according to the machine learning framework you\u2019re using."),Bt=f(),k(Da.$$.fragment),Yt=f(),_a=r("p"),Ns=r("strong"),Ql=t("6"),Kl=t(". Start training with your machine learning framework! Check out the \u{1F917} Transformers "),ve=r("a"),Zl=t("text classification guide"),Xl=t(" for an end-to-end example of how to train a model on a text dataset."),Jt=f(),ba=r("h2"),Sa=r("a"),Fs=r("span"),k(ye.$$.fragment),ao=f(),Os=r("span"),eo=t("What's next?"),Qt=f(),We=r("p"),so=t("This completes the \u{1F917} Datasets quickstart! You can load any text, audio, or image dataset with a single function and get it ready for your model to train on."),Kt=f(),aa=r("p"),to=t("For your next steps, take a look at our "),Ve=r("a"),no=t("How-to guides"),ro=t(" and learn how to do more specific things like loading different dataset formats, aligning labels, and streaming large datasets. If you\u2019re interested in learning more about \u{1F917} Datasets core concepts, grab a cup of coffee and read our "),Be=r("a"),lo=t("Conceptual Guides"),oo=t("!"),this.h()},l(a){const p=Jp('[data-svelte="svelte-1phssyn"]',document.head);i=l(p,"META",{name:!0,content:!0}),p.forEach(s),g=d(a),c=l(a,"H1",{class:!0});var ke=o(c);m=l(ke,"A",{id:!0,class:!0,href:!0});var Is=o(m);$=l(Is,"SPAN",{});var Rs=o($);x(j.$$.fragment,Rs),Rs.forEach(s),Is.forEach(s),S=d(ke),D=l(ke,"SPAN",{});var Ls=o(D);C=n(Ls,"Quickstart"),Ls.forEach(s),ke.forEach(s),_=d(a),y=l(a,"P",{});var Xt=o(y);b=n(Xt,"This quickstart is intended for developers who are ready to dive into the code and see an example of how to integrate \u{1F917} Datasets into their model training workflow. If you\u2019re a beginner, we recommend starting with our "),w=l(Xt,"A",{href:!0});var uo=o(w);z=n(uo,"tutorials"),uo.forEach(s),O=n(Xt,", where you\u2019ll get a more thorough introduction."),Xt.forEach(s),v=d(a),P=l(a,"P",{});var an=o(P);ea=n(an,"Each dataset is unique, and depending on the task, some datasets may require additional steps to prepare it for training. But you can always use \u{1F917} Datasets tools to load and process a dataset. The fastest and easiest way to get started is by loading an existing dataset from the "),M=l(an,"A",{href:!0,rel:!0});var fo=o(M);sa=n(fo,"Hugging Face Hub"),fo.forEach(s),ta=n(an,". There are thousands of datasets to choose from, spanning many tasks. Choose the type of dataset you want to work with, and let\u2019s get started!"),an.forEach(s),Zs=d(a),za=l(a,"DIV",{class:!0});var mo=o(za);Q=l(mo,"DIV",{class:!0});var Ye=o(Q);na=l(Ye,"A",{class:!0,href:!0});var en=o(na);qe=l(en,"DIV",{class:!0});var go=o(qe);fn=n(go,"Audio"),go.forEach(s),dn=d(en),Ae=l(en,"P",{class:!0});var _o=o(Ae);mn=n(_o,"Resample an audio dataset and get it ready for a model to classify what type of banking issue a speaker is calling about."),_o.forEach(s),en.forEach(s),gn=d(Ye),ra=l(Ye,"A",{class:!0,href:!0});var sn=o(ra);Te=l(sn,"DIV",{class:!0});var bo=o(Te);_n=n(bo,"Vision"),bo.forEach(s),bn=d(sn),Ce=l(sn,"P",{class:!0});var jo=o(Ce);jn=n(jo,"Apply data augmentation to an image dataset and get it ready for a model to diagnose disease in bean plants."),jo.forEach(s),sn.forEach(s),$n=d(Ye),la=l(Ye,"A",{class:!0,href:!0});var tn=o(la);Pe=l(tn,"DIV",{class:!0});var $o=o(Pe);wn=n($o,"NLP"),$o.forEach(s),vn=d(tn),De=l(tn,"P",{class:!0});var wo=o(De);yn=n(wo,"Tokenize a dataset and get it ready for a model to determine whether a pair of sentences have the same meaning."),wo.forEach(s),tn.forEach(s),Ye.forEach(s),mo.forEach(s),Xs=d(a),x(va.$$.fragment,a),at=d(a),Se=l(a,"P",{});var vo=o(Se);kn=n(vo,"Start by installing \u{1F917} Datasets:"),vo.forEach(s),et=d(a),x(Na.$$.fragment,a),st=d(a),ya=l(a,"P",{});var nn=o(ya);xn=n(nn,"To work with audio datasets, install the "),ze=l(nn,"A",{href:!0});var yo=o(ze);En=n(yo,"Audio"),yo.forEach(s),qn=n(nn," feature:"),nn.forEach(s),tt=d(a),x(Fa.$$.fragment,a),nt=d(a),ka=l(a,"P",{});var rn=o(ka);An=n(rn,"To work with image datasets, install the "),Ne=l(rn,"A",{href:!0});var ko=o(Ne);Tn=n(ko,"Image"),ko.forEach(s),Cn=n(rn," feature:"),rn.forEach(s),rt=d(a),x(Oa.$$.fragment,a),lt=d(a),oa=l(a,"H2",{class:!0});var ln=o(oa);xa=l(ln,"A",{id:!0,class:!0,href:!0});var xo=o(xa);Xe=l(xo,"SPAN",{});var Eo=o(Xe);x(Ia.$$.fragment,Eo),Eo.forEach(s),xo.forEach(s),Pn=d(ln),as=l(ln,"SPAN",{});var qo=o(as);Dn=n(qo,"Audio"),qo.forEach(s),ln.forEach(s),ot=d(a),K=l(a,"P",{});var Je=o(K);Sn=n(Je,"Audio datasets are loaded just like text datasets. However, an audio dataset is preprocessed a bit differently. Instead of a tokenizer, you\u2019ll need a "),Ra=l(Je,"A",{href:!0,rel:!0});var Ao=o(Ra);zn=n(Ao,"feature extractor"),Ao.forEach(s),Nn=n(Je,". An audio input may also require resampling its sampling rate to match the sampling rate of the pretrained model you\u2019re using. In this quickstart, you\u2019ll prepare the "),La=l(Je,"A",{href:!0,rel:!0});var To=o(La);Fn=n(To,"MInDS-14"),To.forEach(s),On=n(Je," dataset for a model train on and classify the banking issue a customer is having."),Je.forEach(s),pt=d(a),pa=l(a,"P",{});var Ms=o(pa);es=l(Ms,"STRONG",{});var Co=o(es);In=n(Co,"1"),Co.forEach(s),Rn=n(Ms,". Load the MInDS-14 dataset by providing the "),Fe=l(Ms,"A",{href:!0});var Po=o(Fe);Ln=n(Po,"load_dataset()"),Po.forEach(s),Mn=n(Ms," function with the dataset name, dataset configuration (not all datasets will have a configuration), and a dataset split:"),Ms.forEach(s),it=d(a),x(Ma.$$.fragment,a),ct=d(a),W=l(a,"P",{});var xe=o(W);ss=l(xe,"STRONG",{});var Do=o(ss);Gn=n(Do,"2"),Do.forEach(s),Hn=n(xe,". Next, load a pretrained "),Ga=l(xe,"A",{href:!0,rel:!0});var So=o(Ga);Un=n(So,"Wav2Vec2"),So.forEach(s),Wn=n(xe," model and its corresponding feature extractor from the "),Ha=l(xe,"A",{href:!0,rel:!0});var zo=o(Ha);Vn=n(zo,"\u{1F917} Transformers"),zo.forEach(s),Bn=n(xe," library. It is totally normal to see a warning after you load the model about some weights not being initialized. This is expected because you are loading this model checkpoint for training with another task."),xe.forEach(s),ht=d(a),x(Ua.$$.fragment,a),ut=d(a),I=l(a,"P",{});var B=o(I);ts=l(B,"STRONG",{});var No=o(ts);Yn=n(No,"3"),No.forEach(s),Jn=n(B,". The "),Wa=l(B,"A",{href:!0,rel:!0});var Fo=o(Wa);Qn=n(Fo,"MInDS-14"),Fo.forEach(s),Kn=n(B," dataset card indicates the sampling rate is 8kHz, but the Wav2Vec2 model was pretrained on a sampling rate of 16kHZ. You\u2019ll need to upsample the "),ns=l(B,"CODE",{});var Oo=o(ns);Zn=n(Oo,"audio"),Oo.forEach(s),Xn=n(B," column with the "),Oe=l(B,"A",{href:!0});var Io=o(Oe);ar=n(Io,"cast_column()"),Io.forEach(s),er=n(B," function and "),Ie=l(B,"A",{href:!0});var Ro=o(Ie);sr=n(Ro,"Audio"),Ro.forEach(s),tr=n(B," feature to match the model\u2019s sampling rate."),B.forEach(s),ft=d(a),x(Va.$$.fragment,a),dt=d(a),G=l(a,"P",{});var ja=o(G);rs=l(ja,"STRONG",{});var Lo=o(rs);nr=n(Lo,"4"),Lo.forEach(s),rr=n(ja,". Create a function to preprocess the audio "),ls=l(ja,"CODE",{});var Mo=o(ls);lr=n(Mo,"array"),Mo.forEach(s),or=n(ja," with the feature extractor, and truncate and pad the sequences into tidy rectangular tensors. The most important thing to remember is to call the audio "),os=l(ja,"CODE",{});var Go=o(os);pr=n(Go,"array"),Go.forEach(s),ir=n(ja," in the feature extractor since the "),ps=l(ja,"CODE",{});var Ho=o(ps);cr=n(Ho,"array"),Ho.forEach(s),hr=n(ja," - the actual speech signal - is the model input."),ja.forEach(s),mt=d(a),Ea=l(a,"P",{});var on=o(Ea);ur=n(on,"Once you have a preprocessing function, use the "),Re=l(on,"A",{href:!0});var Uo=o(Re);fr=n(Uo,"map()"),Uo.forEach(s),dr=n(on," function to speed up processing by applying the function to batches of examples in the dataset."),on.forEach(s),gt=d(a),x(Ba.$$.fragment,a),_t=d(a),R=l(a,"P",{});var Y=o(R);is=l(Y,"STRONG",{});var Wo=o(is);mr=n(Wo,"5"),Wo.forEach(s),gr=n(Y,". Use the "),Le=l(Y,"A",{href:!0});var Vo=o(Le);_r=n(Vo,"rename_column()"),Vo.forEach(s),br=n(Y," function to rename the "),cs=l(Y,"CODE",{});var Bo=o(cs);jr=n(Bo,"intent_class"),Bo.forEach(s),$r=n(Y," column to "),hs=l(Y,"CODE",{});var Yo=o(hs);wr=n(Yo,"labels"),Yo.forEach(s),vr=n(Y,", which is the expected input name in "),Ya=l(Y,"A",{href:!0,rel:!0});var Jo=o(Ya);yr=n(Jo,"Wav2Vec2ForSequenceClassification"),Jo.forEach(s),kr=n(Y,":"),Y.forEach(s),bt=d(a),x(Ja.$$.fragment,a),jt=d(a),Qa=l(a,"P",{});var po=o(Qa);us=l(po,"STRONG",{});var Qo=o(us);xr=n(Qo,"6"),Qo.forEach(s),Er=n(po,". Set the dataset format according to the machine learning framework you\u2019re using."),po.forEach(s),$t=d(a),x(qa.$$.fragment,a),wt=d(a),ia=l(a,"P",{});var Gs=o(ia);fs=l(Gs,"STRONG",{});var Ko=o(fs);qr=n(Ko,"7"),Ko.forEach(s),Ar=n(Gs,". Start training with your machine learning framework! Check out the \u{1F917} Transformers "),Ka=l(Gs,"A",{href:!0,rel:!0});var Zo=o(Ka);Tr=n(Zo,"audio classification guide"),Zo.forEach(s),Cr=n(Gs," for an end-to-end example of how to train a model on an audio dataset."),Gs.forEach(s),vt=d(a),ca=l(a,"H2",{class:!0});var pn=o(ca);Aa=l(pn,"A",{id:!0,class:!0,href:!0});var Xo=o(Aa);ds=l(Xo,"SPAN",{});var ap=o(ds);x(Za.$$.fragment,ap),ap.forEach(s),Xo.forEach(s),Pr=d(pn),ms=l(pn,"SPAN",{});var ep=o(ms);Dr=n(ep,"Vision"),ep.forEach(s),pn.forEach(s),yt=d(a),Z=l(a,"P",{});var Qe=o(Z);Sr=n(Qe,"Image datasets are loaded just like text datasets. However, instead of a tokenizer, you\u2019ll need a "),Xa=l(Qe,"A",{href:!0,rel:!0});var sp=o(Xa);zr=n(sp,"feature extractor"),sp.forEach(s),Nr=n(Qe," to preprocess the dataset. Applying data augmentation to an image is common in computer vision to make the model more robust against overfitting. You\u2019re free to use any data augmentation library you want, and then you can apply the augmentations with \u{1F917} Datasets. In this quickstart, you\u2019ll load the "),ae=l(Qe,"A",{href:!0,rel:!0});var tp=o(ae);Fr=n(tp,"Beans"),tp.forEach(s),Or=n(Qe," dataset and get it ready for the model to train on and identify disease from the leaf images."),Qe.forEach(s),kt=d(a),ha=l(a,"P",{});var Hs=o(ha);gs=l(Hs,"STRONG",{});var np=o(gs);Ir=n(np,"1"),np.forEach(s),Rr=n(Hs,". Load the Beans dataset by providing the "),Me=l(Hs,"A",{href:!0});var rp=o(Me);Lr=n(rp,"load_dataset()"),rp.forEach(s),Mr=n(Hs," function with the dataset name and a dataset split:"),Hs.forEach(s),xt=d(a),x(ee.$$.fragment,a),Et=d(a),L=l(a,"P",{});var J=o(L);_s=l(J,"STRONG",{});var lp=o(_s);Gr=n(lp,"2"),lp.forEach(s),Hr=n(J,". Now you can add some data augmentations with any library ("),se=l(J,"A",{href:!0,rel:!0});var op=o(se);Ur=n(op,"Albumentations"),op.forEach(s),Wr=n(J,", "),te=l(J,"A",{href:!0,rel:!0});var pp=o(te);Vr=n(pp,"imgaug"),pp.forEach(s),Br=n(J,", "),ne=l(J,"A",{href:!0,rel:!0});var ip=o(ne);Yr=n(ip,"Kornia"),ip.forEach(s),Jr=n(J,") you like. Here, you\u2019ll use "),re=l(J,"A",{href:!0,rel:!0});var cp=o(re);Qr=n(cp,"torchvision"),cp.forEach(s),Kr=n(J," to randomly change the color properties of an image:"),J.forEach(s),qt=d(a),x(le.$$.fragment,a),At=d(a),ua=l(a,"P",{});var Us=o(ua);bs=l(Us,"STRONG",{});var hp=o(bs);Zr=n(hp,"3"),hp.forEach(s),Xr=n(Us,". Create a function to apply your transform to the dataset and generate the model input: "),js=l(Us,"CODE",{});var up=o(js);al=n(up,"pixel_values"),up.forEach(s),el=n(Us,"."),Us.forEach(s),Tt=d(a),x(oe.$$.fragment,a),Ct=d(a),fa=l(a,"P",{});var Ws=o(fa);$s=l(Ws,"STRONG",{});var fp=o($s);sl=n(fp,"4"),fp.forEach(s),tl=n(Ws,". Use the "),Ge=l(Ws,"A",{href:!0});var dp=o(Ge);nl=n(dp,"with_transform()"),dp.forEach(s),rl=n(Ws," function to apply the data augmentations on-the-fly:"),Ws.forEach(s),Pt=d(a),x(pe.$$.fragment,a),Dt=d(a),ie=l(a,"P",{});var io=o(ie);ws=l(io,"STRONG",{});var mp=o(ws);ll=n(mp,"5"),mp.forEach(s),ol=n(io,". Set the dataset format according to the machine learning framework you\u2019re using."),io.forEach(s),St=d(a),x(Ta.$$.fragment,a),zt=d(a),da=l(a,"P",{});var Vs=o(da);vs=l(Vs,"STRONG",{});var gp=o(vs);pl=n(gp,"6"),gp.forEach(s),il=n(Vs,". Start training with your machine learning framework! Check out the \u{1F917} Transformers "),ce=l(Vs,"A",{href:!0,rel:!0});var _p=o(ce);cl=n(_p,"image classification guide"),_p.forEach(s),hl=n(Vs," for an end-to-end example of how to train a model on an image dataset."),Vs.forEach(s),Nt=d(a),ma=l(a,"H2",{class:!0});var cn=o(ma);Ca=l(cn,"A",{id:!0,class:!0,href:!0});var bp=o(Ca);ys=l(bp,"SPAN",{});var jp=o(ys);x(he.$$.fragment,jp),jp.forEach(s),bp.forEach(s),ul=d(cn),ks=l(cn,"SPAN",{});var $p=o(ks);fl=n($p,"NLP"),$p.forEach(s),cn.forEach(s),Ft=d(a),X=l(a,"P",{});var Ke=o(X);dl=n(Ke,"Text needs to be tokenized into individual tokens by a "),ue=l(Ke,"A",{href:!0,rel:!0});var wp=o(ue);ml=n(wp,"tokenizer"),wp.forEach(s),gl=n(Ke,". For the quickstart, you\u2019ll load the "),fe=l(Ke,"A",{href:!0,rel:!0});var vp=o(fe);_l=n(vp,"Microsoft Research Paraphrase Corpus (MRPC)"),vp.forEach(s),bl=n(Ke," training dataset to train a model to determine whether a pair of sentences mean the same thing."),Ke.forEach(s),Ot=d(a),ga=l(a,"P",{});var Bs=o(ga);xs=l(Bs,"STRONG",{});var yp=o(xs);jl=n(yp,"1"),yp.forEach(s),$l=n(Bs,". Load the MRPC dataset by providing the "),He=l(Bs,"A",{href:!0});var kp=o(He);wl=n(kp,"load_dataset()"),kp.forEach(s),vl=n(Bs," function with the dataset name, dataset configuration (not all datasets will have a configuration), and dataset split:"),Bs.forEach(s),It=d(a),x(de.$$.fragment,a),Rt=d(a),V=l(a,"P",{});var Ee=o(V);Es=l(Ee,"STRONG",{});var xp=o(Es);yl=n(xp,"2"),xp.forEach(s),kl=n(Ee,". Next, load a pretrained "),me=l(Ee,"A",{href:!0,rel:!0});var Ep=o(me);xl=n(Ep,"BERT"),Ep.forEach(s),El=n(Ee," model and its corresponding tokenizer from the "),ge=l(Ee,"A",{href:!0,rel:!0});var qp=o(ge);ql=n(qp,"\u{1F917} Transformers"),qp.forEach(s),Al=n(Ee," library. It is totally normal to see a warning after you load the model about some weights not being initialized. This is expected because you are loading this model checkpoint for training with another task."),Ee.forEach(s),Lt=d(a),x(_e.$$.fragment,a),Mt=d(a),H=l(a,"P",{});var $a=o(H);qs=l($a,"STRONG",{});var Ap=o(qs);Tl=n(Ap,"3"),Ap.forEach(s),Cl=n($a,". Create a function to tokenize the dataset, and you should also truncate and pad the text into tidy rectangular tensors. The tokenizer generates three new columns in the dataset: "),As=l($a,"CODE",{});var Tp=o(As);Pl=n(Tp,"input_ids"),Tp.forEach(s),Dl=n($a,", "),Ts=l($a,"CODE",{});var Cp=o(Ts);Sl=n(Cp,"token_type_ids"),Cp.forEach(s),zl=n($a,", and an "),Cs=l($a,"CODE",{});var Pp=o(Cs);Nl=n(Pp,"attention_mask"),Pp.forEach(s),Fl=n($a,". These are the model inputs."),$a.forEach(s),Gt=d(a),Pa=l(a,"P",{});var hn=o(Pa);Ol=n(hn,"Use the "),Ue=l(hn,"A",{href:!0});var Dp=o(Ue);Il=n(Dp,"map()"),Dp.forEach(s),Rl=n(hn," function to speed up processing by applying your tokenization function to batches of examples in the dataset:"),hn.forEach(s),Ht=d(a),x(be.$$.fragment,a),Ut=d(a),U=l(a,"P",{});var wa=o(U);Ps=l(wa,"STRONG",{});var Sp=o(Ps);Ll=n(Sp,"4"),Sp.forEach(s),Ml=n(wa,". Rename the "),Ds=l(wa,"CODE",{});var zp=o(Ds);Gl=n(zp,"label"),zp.forEach(s),Hl=n(wa," column to "),Ss=l(wa,"CODE",{});var Np=o(Ss);Ul=n(Np,"labels"),Np.forEach(s),Wl=n(wa,", which is the expected input name in "),je=l(wa,"A",{href:!0,rel:!0});var Fp=o(je);Vl=n(Fp,"BertForSequenceClassification"),Fp.forEach(s),Bl=n(wa,":"),wa.forEach(s),Wt=d(a),x($e.$$.fragment,a),Vt=d(a),we=l(a,"P",{});var co=o(we);zs=l(co,"STRONG",{});var Op=o(zs);Yl=n(Op,"5"),Op.forEach(s),Jl=n(co,". Set the dataset format according to the machine learning framework you\u2019re using."),co.forEach(s),Bt=d(a),x(Da.$$.fragment,a),Yt=d(a),_a=l(a,"P",{});var Ys=o(_a);Ns=l(Ys,"STRONG",{});var Ip=o(Ns);Ql=n(Ip,"6"),Ip.forEach(s),Kl=n(Ys,". Start training with your machine learning framework! Check out the \u{1F917} Transformers "),ve=l(Ys,"A",{href:!0,rel:!0});var Rp=o(ve);Zl=n(Rp,"text classification guide"),Rp.forEach(s),Xl=n(Ys," for an end-to-end example of how to train a model on a text dataset."),Ys.forEach(s),Jt=d(a),ba=l(a,"H2",{class:!0});var un=o(ba);Sa=l(un,"A",{id:!0,class:!0,href:!0});var Lp=o(Sa);Fs=l(Lp,"SPAN",{});var Mp=o(Fs);x(ye.$$.fragment,Mp),Mp.forEach(s),Lp.forEach(s),ao=d(un),Os=l(un,"SPAN",{});var Gp=o(Os);eo=n(Gp,"What's next?"),Gp.forEach(s),un.forEach(s),Qt=d(a),We=l(a,"P",{});var Hp=o(We);so=n(Hp,"This completes the \u{1F917} Datasets quickstart! You can load any text, audio, or image dataset with a single function and get it ready for your model to train on."),Hp.forEach(s),Kt=d(a),aa=l(a,"P",{});var Ze=o(aa);to=n(Ze,"For your next steps, take a look at our "),Ve=l(Ze,"A",{href:!0});var Up=o(Ve);no=n(Up,"How-to guides"),Up.forEach(s),ro=n(Ze," and learn how to do more specific things like loading different dataset formats, aligning labels, and streaming large datasets. If you\u2019re interested in learning more about \u{1F917} Datasets core concepts, grab a cup of coffee and read our "),Be=l(Ze,"A",{href:!0});var Wp=o(Be);lo=n(Wp,"Conceptual Guides"),Wp.forEach(s),oo=n(Ze,"!"),Ze.forEach(s),this.h()},h(){h(i,"name","hf:doc:metadata"),h(i,"content",JSON.stringify(hi)),h(m,"id","quickstart"),h(m,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(m,"href","#quickstart"),h(c,"class","relative group"),h(w,"href","./tutorial"),h(M,"href","https://huggingface.co/datasets"),h(M,"rel","nofollow"),h(qe,"class","w-full text-center bg-gradient-to-r from-violet-300 via-sky-400 to-green-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed"),h(Ae,"class","text-gray-700"),h(na,"class","!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg"),h(na,"href","#audio"),h(Te,"class","w-full text-center bg-gradient-to-r from-pink-400 via-purple-400 to-blue-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed"),h(Ce,"class","text-gray-700"),h(ra,"class","!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg"),h(ra,"href","#vision"),h(Pe,"class","w-full text-center bg-gradient-to-r from-orange-300 via-red-400 to-violet-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed"),h(De,"class","text-gray-700"),h(la,"class","!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg"),h(la,"href","#nlp"),h(Q,"class","w-full flex flex-col space-y-4 md:space-y-0 md:grid md:grid-cols-3 md:gap-y-4 md:gap-x-5"),h(za,"class","mt-4"),h(ze,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Audio"),h(Ne,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Image"),h(xa,"id","audio"),h(xa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(xa,"href","#audio"),h(oa,"class","relative group"),h(Ra,"href","https://huggingface.co/docs/transformers/main_classes/feature_extractor#feature-extractor"),h(Ra,"rel","nofollow"),h(La,"href","https://huggingface.co/datasets/PolyAI/minds14"),h(La,"rel","nofollow"),h(Fe,"href","/docs/datasets/v2.11.0/en/package_reference/loading_methods#datasets.load_dataset"),h(Ga,"href","https://huggingface.co/facebook/wav2vec2-base"),h(Ga,"rel","nofollow"),h(Ha,"href","https://huggingface.co/transformers/"),h(Ha,"rel","nofollow"),h(Wa,"href","https://huggingface.co/datasets/PolyAI/minds14"),h(Wa,"rel","nofollow"),h(Oe,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.cast_column"),h(Ie,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Audio"),h(Re,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.map"),h(Le,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.rename_column"),h(Ya,"href","https://huggingface.co/docs/transformers/main/en/model_doc/wav2vec2#transformers.Wav2Vec2ForSequenceClassification"),h(Ya,"rel","nofollow"),h(Ka,"href","https://huggingface.co/docs/transformers/tasks/audio_classification"),h(Ka,"rel","nofollow"),h(Aa,"id","vision"),h(Aa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Aa,"href","#vision"),h(ca,"class","relative group"),h(Xa,"href","https://huggingface.co/docs/transformers/main_classes/feature_extractor#feature-extractor"),h(Xa,"rel","nofollow"),h(ae,"href","https://huggingface.co/datasets/beans"),h(ae,"rel","nofollow"),h(Me,"href","/docs/datasets/v2.11.0/en/package_reference/loading_methods#datasets.load_dataset"),h(se,"href","https://albumentations.ai/"),h(se,"rel","nofollow"),h(te,"href","https://imgaug.readthedocs.io/en/latest/"),h(te,"rel","nofollow"),h(ne,"href","https://kornia.readthedocs.io/en/latest/"),h(ne,"rel","nofollow"),h(re,"href","https://pytorch.org/vision/stable/transforms.html"),h(re,"rel","nofollow"),h(Ge,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.with_transform"),h(ce,"href","https://huggingface.co/docs/transformers/tasks/image_classification"),h(ce,"rel","nofollow"),h(Ca,"id","nlp"),h(Ca,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Ca,"href","#nlp"),h(ma,"class","relative group"),h(ue,"href","https://huggingface.co/docs/transformers/main_classes/tokenizer"),h(ue,"rel","nofollow"),h(fe,"href","https://huggingface.co/datasets/glue/viewer/mrpc"),h(fe,"rel","nofollow"),h(He,"href","/docs/datasets/v2.11.0/en/package_reference/loading_methods#datasets.load_dataset"),h(me,"href","https://huggingface.co/bert-base-uncased"),h(me,"rel","nofollow"),h(ge,"href","https://huggingface.co/transformers/"),h(ge,"rel","nofollow"),h(Ue,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.map"),h(je,"href","https://huggingface.co/docs/transformers/main/en/model_doc/bert#transformers.BertForSequenceClassification"),h(je,"rel","nofollow"),h(ve,"href","https://huggingface.co/docs/transformers/tasks/sequence_classification"),h(ve,"rel","nofollow"),h(Sa,"id","whats-next"),h(Sa,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),h(Sa,"href","#whats-next"),h(ba,"class","relative group"),h(Ve,"href","./how_to"),h(Be,"href","./about_arrow")},m(a,p){e(document.head,i),u(a,g,p),u(a,c,p),e(c,m),e(m,$),E(j,$,null),e(c,S),e(c,D),e(D,C),u(a,_,p),u(a,y,p),e(y,b),e(y,w),e(w,z),e(y,O),u(a,v,p),u(a,P,p),e(P,ea),e(P,M),e(M,sa),e(P,ta),u(a,Zs,p),u(a,za,p),e(za,Q),e(Q,na),e(na,qe),e(qe,fn),e(na,dn),e(na,Ae),e(Ae,mn),e(Q,gn),e(Q,ra),e(ra,Te),e(Te,_n),e(ra,bn),e(ra,Ce),e(Ce,jn),e(Q,$n),e(Q,la),e(la,Pe),e(Pe,wn),e(la,vn),e(la,De),e(De,yn),u(a,Xs,p),E(va,a,p),u(a,at,p),u(a,Se,p),e(Se,kn),u(a,et,p),E(Na,a,p),u(a,st,p),u(a,ya,p),e(ya,xn),e(ya,ze),e(ze,En),e(ya,qn),u(a,tt,p),E(Fa,a,p),u(a,nt,p),u(a,ka,p),e(ka,An),e(ka,Ne),e(Ne,Tn),e(ka,Cn),u(a,rt,p),E(Oa,a,p),u(a,lt,p),u(a,oa,p),e(oa,xa),e(xa,Xe),E(Ia,Xe,null),e(oa,Pn),e(oa,as),e(as,Dn),u(a,ot,p),u(a,K,p),e(K,Sn),e(K,Ra),e(Ra,zn),e(K,Nn),e(K,La),e(La,Fn),e(K,On),u(a,pt,p),u(a,pa,p),e(pa,es),e(es,In),e(pa,Rn),e(pa,Fe),e(Fe,Ln),e(pa,Mn),u(a,it,p),E(Ma,a,p),u(a,ct,p),u(a,W,p),e(W,ss),e(ss,Gn),e(W,Hn),e(W,Ga),e(Ga,Un),e(W,Wn),e(W,Ha),e(Ha,Vn),e(W,Bn),u(a,ht,p),E(Ua,a,p),u(a,ut,p),u(a,I,p),e(I,ts),e(ts,Yn),e(I,Jn),e(I,Wa),e(Wa,Qn),e(I,Kn),e(I,ns),e(ns,Zn),e(I,Xn),e(I,Oe),e(Oe,ar),e(I,er),e(I,Ie),e(Ie,sr),e(I,tr),u(a,ft,p),E(Va,a,p),u(a,dt,p),u(a,G,p),e(G,rs),e(rs,nr),e(G,rr),e(G,ls),e(ls,lr),e(G,or),e(G,os),e(os,pr),e(G,ir),e(G,ps),e(ps,cr),e(G,hr),u(a,mt,p),u(a,Ea,p),e(Ea,ur),e(Ea,Re),e(Re,fr),e(Ea,dr),u(a,gt,p),E(Ba,a,p),u(a,_t,p),u(a,R,p),e(R,is),e(is,mr),e(R,gr),e(R,Le),e(Le,_r),e(R,br),e(R,cs),e(cs,jr),e(R,$r),e(R,hs),e(hs,wr),e(R,vr),e(R,Ya),e(Ya,yr),e(R,kr),u(a,bt,p),E(Ja,a,p),u(a,jt,p),u(a,Qa,p),e(Qa,us),e(us,xr),e(Qa,Er),u(a,$t,p),E(qa,a,p),u(a,wt,p),u(a,ia,p),e(ia,fs),e(fs,qr),e(ia,Ar),e(ia,Ka),e(Ka,Tr),e(ia,Cr),u(a,vt,p),u(a,ca,p),e(ca,Aa),e(Aa,ds),E(Za,ds,null),e(ca,Pr),e(ca,ms),e(ms,Dr),u(a,yt,p),u(a,Z,p),e(Z,Sr),e(Z,Xa),e(Xa,zr),e(Z,Nr),e(Z,ae),e(ae,Fr),e(Z,Or),u(a,kt,p),u(a,ha,p),e(ha,gs),e(gs,Ir),e(ha,Rr),e(ha,Me),e(Me,Lr),e(ha,Mr),u(a,xt,p),E(ee,a,p),u(a,Et,p),u(a,L,p),e(L,_s),e(_s,Gr),e(L,Hr),e(L,se),e(se,Ur),e(L,Wr),e(L,te),e(te,Vr),e(L,Br),e(L,ne),e(ne,Yr),e(L,Jr),e(L,re),e(re,Qr),e(L,Kr),u(a,qt,p),E(le,a,p),u(a,At,p),u(a,ua,p),e(ua,bs),e(bs,Zr),e(ua,Xr),e(ua,js),e(js,al),e(ua,el),u(a,Tt,p),E(oe,a,p),u(a,Ct,p),u(a,fa,p),e(fa,$s),e($s,sl),e(fa,tl),e(fa,Ge),e(Ge,nl),e(fa,rl),u(a,Pt,p),E(pe,a,p),u(a,Dt,p),u(a,ie,p),e(ie,ws),e(ws,ll),e(ie,ol),u(a,St,p),E(Ta,a,p),u(a,zt,p),u(a,da,p),e(da,vs),e(vs,pl),e(da,il),e(da,ce),e(ce,cl),e(da,hl),u(a,Nt,p),u(a,ma,p),e(ma,Ca),e(Ca,ys),E(he,ys,null),e(ma,ul),e(ma,ks),e(ks,fl),u(a,Ft,p),u(a,X,p),e(X,dl),e(X,ue),e(ue,ml),e(X,gl),e(X,fe),e(fe,_l),e(X,bl),u(a,Ot,p),u(a,ga,p),e(ga,xs),e(xs,jl),e(ga,$l),e(ga,He),e(He,wl),e(ga,vl),u(a,It,p),E(de,a,p),u(a,Rt,p),u(a,V,p),e(V,Es),e(Es,yl),e(V,kl),e(V,me),e(me,xl),e(V,El),e(V,ge),e(ge,ql),e(V,Al),u(a,Lt,p),E(_e,a,p),u(a,Mt,p),u(a,H,p),e(H,qs),e(qs,Tl),e(H,Cl),e(H,As),e(As,Pl),e(H,Dl),e(H,Ts),e(Ts,Sl),e(H,zl),e(H,Cs),e(Cs,Nl),e(H,Fl),u(a,Gt,p),u(a,Pa,p),e(Pa,Ol),e(Pa,Ue),e(Ue,Il),e(Pa,Rl),u(a,Ht,p),E(be,a,p),u(a,Ut,p),u(a,U,p),e(U,Ps),e(Ps,Ll),e(U,Ml),e(U,Ds),e(Ds,Gl),e(U,Hl),e(U,Ss),e(Ss,Ul),e(U,Wl),e(U,je),e(je,Vl),e(U,Bl),u(a,Wt,p),E($e,a,p),u(a,Vt,p),u(a,we,p),e(we,zs),e(zs,Yl),e(we,Jl),u(a,Bt,p),E(Da,a,p),u(a,Yt,p),u(a,_a,p),e(_a,Ns),e(Ns,Ql),e(_a,Kl),e(_a,ve),e(ve,Zl),e(_a,Xl),u(a,Jt,p),u(a,ba,p),e(ba,Sa),e(Sa,Fs),E(ye,Fs,null),e(ba,ao),e(ba,Os),e(Os,eo),u(a,Qt,p),u(a,We,p),e(We,so),u(a,Kt,p),u(a,aa,p),e(aa,to),e(aa,Ve),e(Ve,no),e(aa,ro),e(aa,Be),e(Be,lo),e(aa,oo),Zt=!0},p(a,[p]){const ke={};p&2&&(ke.$$scope={dirty:p,ctx:a}),va.$set(ke);const Is={};p&2&&(Is.$$scope={dirty:p,ctx:a}),qa.$set(Is);const Rs={};p&2&&(Rs.$$scope={dirty:p,ctx:a}),Ta.$set(Rs);const Ls={};p&2&&(Ls.$$scope={dirty:p,ctx:a}),Da.$set(Ls)},i(a){Zt||(q(j.$$.fragment,a),q(va.$$.fragment,a),q(Na.$$.fragment,a),q(Fa.$$.fragment,a),q(Oa.$$.fragment,a),q(Ia.$$.fragment,a),q(Ma.$$.fragment,a),q(Ua.$$.fragment,a),q(Va.$$.fragment,a),q(Ba.$$.fragment,a),q(Ja.$$.fragment,a),q(qa.$$.fragment,a),q(Za.$$.fragment,a),q(ee.$$.fragment,a),q(le.$$.fragment,a),q(oe.$$.fragment,a),q(pe.$$.fragment,a),q(Ta.$$.fragment,a),q(he.$$.fragment,a),q(de.$$.fragment,a),q(_e.$$.fragment,a),q(be.$$.fragment,a),q($e.$$.fragment,a),q(Da.$$.fragment,a),q(ye.$$.fragment,a),Zt=!0)},o(a){A(j.$$.fragment,a),A(va.$$.fragment,a),A(Na.$$.fragment,a),A(Fa.$$.fragment,a),A(Oa.$$.fragment,a),A(Ia.$$.fragment,a),A(Ma.$$.fragment,a),A(Ua.$$.fragment,a),A(Va.$$.fragment,a),A(Ba.$$.fragment,a),A(Ja.$$.fragment,a),A(qa.$$.fragment,a),A(Za.$$.fragment,a),A(ee.$$.fragment,a),A(le.$$.fragment,a),A(oe.$$.fragment,a),A(pe.$$.fragment,a),A(Ta.$$.fragment,a),A(he.$$.fragment,a),A(de.$$.fragment,a),A(_e.$$.fragment,a),A(be.$$.fragment,a),A($e.$$.fragment,a),A(Da.$$.fragment,a),A(ye.$$.fragment,a),Zt=!1},d(a){s(i),a&&s(g),a&&s(c),T(j),a&&s(_),a&&s(y),a&&s(v),a&&s(P),a&&s(Zs),a&&s(za),a&&s(Xs),T(va,a),a&&s(at),a&&s(Se),a&&s(et),T(Na,a),a&&s(st),a&&s(ya),a&&s(tt),T(Fa,a),a&&s(nt),a&&s(ka),a&&s(rt),T(Oa,a),a&&s(lt),a&&s(oa),T(Ia),a&&s(ot),a&&s(K),a&&s(pt),a&&s(pa),a&&s(it),T(Ma,a),a&&s(ct),a&&s(W),a&&s(ht),T(Ua,a),a&&s(ut),a&&s(I),a&&s(ft),T(Va,a),a&&s(dt),a&&s(G),a&&s(mt),a&&s(Ea),a&&s(gt),T(Ba,a),a&&s(_t),a&&s(R),a&&s(bt),T(Ja,a),a&&s(jt),a&&s(Qa),a&&s($t),T(qa,a),a&&s(wt),a&&s(ia),a&&s(vt),a&&s(ca),T(Za),a&&s(yt),a&&s(Z),a&&s(kt),a&&s(ha),a&&s(xt),T(ee,a),a&&s(Et),a&&s(L),a&&s(qt),T(le,a),a&&s(At),a&&s(ua),a&&s(Tt),T(oe,a),a&&s(Ct),a&&s(fa),a&&s(Pt),T(pe,a),a&&s(Dt),a&&s(ie),a&&s(St),T(Ta,a),a&&s(zt),a&&s(da),a&&s(Nt),a&&s(ma),T(he),a&&s(Ft),a&&s(X),a&&s(Ot),a&&s(ga),a&&s(It),T(de,a),a&&s(Rt),a&&s(V),a&&s(Lt),T(_e,a),a&&s(Mt),a&&s(H),a&&s(Gt),a&&s(Pa),a&&s(Ht),T(be,a),a&&s(Ut),a&&s(U),a&&s(Wt),T($e,a),a&&s(Vt),a&&s(we),a&&s(Bt),T(Da,a),a&&s(Yt),a&&s(_a),a&&s(Jt),a&&s(ba),T(ye),a&&s(Qt),a&&s(We),a&&s(Kt),a&&s(aa)}}}const hi={local:"quickstart",sections:[{local:"audio",title:"Audio"},{local:"vision",title:"Vision"},{local:"nlp",title:"NLP"},{local:"whats-next",title:"What's next?"}],title:"Quickstart"};function ui(N){return Qp(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class $i extends Vp{constructor(i){super();Bp(this,i,ui,ci,Yp,{})}}export{$i as default,hi as metadata};
