import{S as _r,i as yr,s as vr,e as l,k as h,w as v,t,M as $r,c as r,d as e,m as c,a as o,x as $,h as n,b as f,G as a,g as i,y as k,q as w,o as x,B as E,v as kr,L as br}from"../chunks/vendor-hf-doc-builder.js";import{T as wr}from"../chunks/Tip-hf-doc-builder.js";import{I as He}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as D}from"../chunks/CodeBlock-hf-doc-builder.js";import{F as xr,M as jr}from"../chunks/Markdown-hf-doc-builder.js";import"../chunks/IconTensorflow-hf-doc-builder.js";function Er(N){let m,g,u,d,_;return{c(){m=l("p"),g=t("Check out the "),u=l("a"),d=t("Tokenizers"),_=t(" section in Chapter 2 of the Hugging Face course to learn more about tokenization and different tokenization algorithms."),this.h()},l(b){m=r(b,"P",{});var y=o(m);g=n(y,"Check out the "),u=r(y,"A",{href:!0,rel:!0});var A=o(u);d=n(A,"Tokenizers"),A.forEach(e),_=n(y," section in Chapter 2 of the Hugging Face course to learn more about tokenization and different tokenization algorithms."),y.forEach(e),this.h()},h(){f(u,"href","https://huggingface.co/course/chapter2/4?fw=pt"),f(u,"rel","nofollow")},m(b,y){i(b,m,y),a(m,g),a(m,u),a(u,d),a(m,_)},d(b){b&&e(m)}}}function Tr(N){let m,g,u,d,_,b,y,A;return y=new D({props:{code:`dataset.set_format(type="torch", columns=["input_ids", "token_type_ids", "attention_mask", "labels"])
dataset.format['type']`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_format(<span class="hljs-built_in">type</span>=<span class="hljs-string">&quot;torch&quot;</span>, columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>, <span class="hljs-string">&quot;attention_mask&quot;</span>, <span class="hljs-string">&quot;labels&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.<span class="hljs-built_in">format</span>[<span class="hljs-string">&#x27;type&#x27;</span>]
<span class="hljs-string">&#x27;torch&#x27;</span>`}}),{c(){m=l("p"),g=t("Use the "),u=l("a"),d=t("set_format()"),_=t(" function to set the dataset format to be compatible with PyTorch:"),b=h(),v(y.$$.fragment),this.h()},l(j){m=r(j,"P",{});var T=o(m);g=n(T,"Use the "),u=r(T,"A",{href:!0});var P=o(u);d=n(P,"set_format()"),P.forEach(e),_=n(T," function to set the dataset format to be compatible with PyTorch:"),T.forEach(e),b=c(j),$(y.$$.fragment,j),this.h()},h(){f(u,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.set_format")},m(j,T){i(j,m,T),a(m,g),a(m,u),a(u,d),a(m,_),i(j,b,T),k(y,j,T),A=!0},p:br,i(j){A||(w(y.$$.fragment,j),A=!0)},o(j){x(y.$$.fragment,j),A=!1},d(j){j&&e(m),j&&e(b),E(y,j)}}}function qr(N){let m,g;return m=new jr({props:{$$slots:{default:[Tr]},$$scope:{ctx:N}}}),{c(){v(m.$$.fragment)},l(u){$(m.$$.fragment,u)},m(u,d){k(m,u,d),g=!0},p(u,d){const _={};d&2&&(_.$$scope={dirty:d,ctx:u}),m.$set(_)},i(u){g||(w(m.$$.fragment,u),g=!0)},o(u){x(m.$$.fragment,u),g=!1},d(u){E(m,u)}}}function Ar(N){let m,g,u,d,_,b,y,A,j,T,P;return T=new D({props:{code:`from transformers import DataCollatorWithPadding

data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors="tf")
tf_dataset = dataset.to_tf_dataset(
    columns=["input_ids", "token_type_ids", "attention_mask"],
    label_cols=["labels"],
    batch_size=2,
    collate_fn=data_collator,
    shuffle=True
)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorWithPadding

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorWithPadding(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tf_dataset = dataset.to_tf_dataset(
<span class="hljs-meta">... </span>    columns=[<span class="hljs-string">&quot;input_ids&quot;</span>, <span class="hljs-string">&quot;token_type_ids&quot;</span>, <span class="hljs-string">&quot;attention_mask&quot;</span>],
<span class="hljs-meta">... </span>    label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>
<span class="hljs-meta">... </span>)`}}),{c(){m=l("p"),g=t("Use the "),u=l("a"),d=t("to_tf_dataset()"),_=t(" function to set the dataset format to be compatible with TensorFlow. You\u2019ll also need to import a "),b=l("a"),y=t("data collator"),A=t(" from \u{1F917} Transformers to combine the varying sequence lengths into a single batch of equal lengths:"),j=h(),v(T.$$.fragment),this.h()},l(q){m=r(q,"P",{});var z=o(m);g=n(z,"Use the "),u=r(z,"A",{href:!0});var L=o(u);d=n(L,"to_tf_dataset()"),L.forEach(e),_=n(z," function to set the dataset format to be compatible with TensorFlow. You\u2019ll also need to import a "),b=r(z,"A",{href:!0,rel:!0});var Gs=o(b);y=n(Gs,"data collator"),Gs.forEach(e),A=n(z," from \u{1F917} Transformers to combine the varying sequence lengths into a single batch of equal lengths:"),z.forEach(e),j=c(q),$(T.$$.fragment,q),this.h()},h(){f(u,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.to_tf_dataset"),f(b,"href","https://huggingface.co/docs/transformers/main_classes/data_collator#transformers.DataCollatorWithPadding"),f(b,"rel","nofollow")},m(q,z){i(q,m,z),a(m,g),a(m,u),a(u,d),a(m,_),a(m,b),a(b,y),a(m,A),i(q,j,z),k(T,q,z),P=!0},p:br,i(q){P||(w(T.$$.fragment,q),P=!0)},o(q){x(T.$$.fragment,q),P=!1},d(q){q&&e(m),q&&e(j),E(T,q)}}}function zr(N){let m,g;return m=new jr({props:{$$slots:{default:[Ar]},$$scope:{ctx:N}}}),{c(){v(m.$$.fragment)},l(u){$(m.$$.fragment,u)},m(u,d){k(m,u,d),g=!0},p(u,d){const _={};d&2&&(_.$$scope={dirty:d,ctx:u}),m.$set(_)},i(u){g||(w(m.$$.fragment,u),g=!0)},o(u){x(m.$$.fragment,u),g=!1},d(u){E(m,u)}}}function Pr(N){let m,g,u,d,_,b,y,A,j,T,P,q,z,L,Gs,Ja,W,ta,Be,Ve,na,Ye,Je,la,Ke,Ka,Ws,Qe,Qa,Fs,Xe,Xa,ms,Za,Ms,Ze,se,H,es,ra,hs,st,oa,at,ae,ts,et,pa,tt,nt,ee,ns,te,U,ia,lt,rt,cs,ot,pt,us,it,mt,ne,fs,le,B,ma,ht,ct,ha,ut,ft,re,ds,oe,Ls,dt,pe,F,Hs,ca,gt,bt,jt,Bs,ua,_t,yt,vt,Vs,fa,$t,kt,ie,Ys,wt,me,I,da,xt,Et,Js,Tt,qt,ga,At,zt,ba,Pt,St,he,gs,ce,bs,ja,Dt,It,ue,ls,fe,js,_a,Rt,Ct,de,V,rs,ya,_s,Ot,va,Nt,ge,os,Ut,$a,Gt,Wt,be,R,ka,Ft,Mt,ys,Lt,Ht,Ks,Bt,Vt,vs,Yt,Jt,je,$s,_e,Y,wa,Kt,Qt,xa,Xt,Zt,ye,ks,ve,ws,Ea,sn,an,$e,S,en,Qs,tn,nn,Ta,ln,rn,Xs,on,pn,qa,mn,hn,ke,xs,we,C,Aa,cn,un,Zs,fn,dn,za,gn,bn,Pa,jn,_n,xe,Es,Ee,Ts,Sa,yn,vn,Te,J,ps,Da,qs,$n,Ia,kn,qe,is,wn,Ra,xn,En,Ae,O,Ca,Tn,qn,As,An,zn,Oa,Pn,Sn,zs,Dn,In,ze,Ps,Pe,K,Na,Rn,Cn,Ua,On,Nn,Se,Ss,De,Q,Ga,Un,Gn,Ds,Wn,Fn,Ie,Is,Re,G,Wa,Mn,Ln,sa,Hn,Bn,Fa,Vn,Yn,Ce,Rs,Oe,Cs,Ma,Jn,Kn,Ne;return b=new He({}),ms=new D({props:{code:"pip install transformers",highlighted:"pip install transformers"}}),hs=new He({}),ns=new wr({props:{$$slots:{default:[Er]},$$scope:{ctx:N}}}),fs=new D({props:{code:`from transformers import AutoTokenizer
from datasets import load_dataset

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
dataset = load_dataset("rotten_tomatoes", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;rotten_tomatoes&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),ds=new D({props:{code:'tokenizer(dataset[0]["text"])',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>])
{<span class="hljs-string">&#x27;input_ids&#x27;</span>: [<span class="hljs-number">101</span>, <span class="hljs-number">1103</span>, <span class="hljs-number">2067</span>, <span class="hljs-number">1110</span>, <span class="hljs-number">17348</span>, <span class="hljs-number">1106</span>, <span class="hljs-number">1129</span>, <span class="hljs-number">1103</span>, <span class="hljs-number">6880</span>, <span class="hljs-number">1432</span>, <span class="hljs-number">112</span>, <span class="hljs-number">188</span>, <span class="hljs-number">1207</span>, <span class="hljs-number">107</span>, <span class="hljs-number">14255</span>, <span class="hljs-number">1389</span>, <span class="hljs-number">107</span>, <span class="hljs-number">1105</span>, <span class="hljs-number">1115</span>, <span class="hljs-number">1119</span>, <span class="hljs-number">112</span>, <span class="hljs-number">188</span>, <span class="hljs-number">1280</span>, <span class="hljs-number">1106</span>, <span class="hljs-number">1294</span>, <span class="hljs-number">170</span>, <span class="hljs-number">24194</span>, <span class="hljs-number">1256</span>, <span class="hljs-number">3407</span>, <span class="hljs-number">1190</span>, <span class="hljs-number">170</span>, <span class="hljs-number">11791</span>, <span class="hljs-number">5253</span>, <span class="hljs-number">188</span>, <span class="hljs-number">1732</span>, <span class="hljs-number">7200</span>, <span class="hljs-number">10947</span>, <span class="hljs-number">12606</span>, <span class="hljs-number">2895</span>, <span class="hljs-number">117</span>, <span class="hljs-number">179</span>, <span class="hljs-number">7766</span>, <span class="hljs-number">118</span>, <span class="hljs-number">172</span>, <span class="hljs-number">15554</span>, <span class="hljs-number">1181</span>, <span class="hljs-number">3498</span>, <span class="hljs-number">6961</span>, <span class="hljs-number">3263</span>, <span class="hljs-number">1137</span>, <span class="hljs-number">188</span>, <span class="hljs-number">1566</span>, <span class="hljs-number">7912</span>, <span class="hljs-number">14516</span>, <span class="hljs-number">6997</span>, <span class="hljs-number">119</span>, <span class="hljs-number">102</span>], 
 <span class="hljs-string">&#x27;token_type_ids&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>], 
 <span class="hljs-string">&#x27;attention_mask&#x27;</span>: [<span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>]}`}}),gs=new D({props:{code:`def tokenization(example):
    return tokenizer(example["text"])

dataset = dataset.map(tokenization, batched=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenization</span>(<span class="hljs-params">example</span>):
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenizer(example[<span class="hljs-string">&quot;text&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.<span class="hljs-built_in">map</span>(tokenization, batched=<span class="hljs-literal">True</span>)`}}),ls=new xr({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[zr],pytorch:[qr]},$$scope:{ctx:N}}}),_s=new He({}),$s=new D({props:{code:`from transformers import AutoFeatureExtractor
from datasets import load_dataset, Audio

feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base-960h")
dataset = load_dataset("PolyAI/minds14", "en-US", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Audio

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;facebook/wav2vec2-base-960h&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;PolyAI/minds14&quot;</span>, <span class="hljs-string">&quot;en-US&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),ks=new D({props:{code:'dataset[0]["audio"]',highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.00024414</span>, -<span class="hljs-number">0.00024414</span>, ..., -<span class="hljs-number">0.00024414</span>,
         <span class="hljs-number">0.</span>        ,  <span class="hljs-number">0.</span>        ], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">8000</span>}`}}),xs=new D({props:{code:`dataset = dataset.cast_column("audio", Audio(sampling_rate=16_000))
dataset[0]["audio"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.cast_column(<span class="hljs-string">&quot;audio&quot;</span>, Audio(sampling_rate=<span class="hljs-number">16_000</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>]
{<span class="hljs-string">&#x27;array&#x27;</span>: array([ <span class="hljs-number">2.3443763e-05</span>,  <span class="hljs-number">2.1729663e-04</span>,  <span class="hljs-number">2.2145823e-04</span>, ...,
         <span class="hljs-number">3.8356509e-05</span>, -<span class="hljs-number">7.3497440e-06</span>, -<span class="hljs-number">2.1754686e-05</span>], dtype=float32),
 <span class="hljs-string">&#x27;path&#x27;</span>: <span class="hljs-string">&#x27;/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav&#x27;</span>,
 <span class="hljs-string">&#x27;sampling_rate&#x27;</span>: <span class="hljs-number">16000</span>}`}}),Es=new D({props:{code:`def preprocess_function(examples):
    audio_arrays = [x["array"] for x in examples["audio"]]
    inputs = feature_extractor(
        audio_arrays, sampling_rate=feature_extractor.sampling_rate, max_length=16000, truncation=True
    )
    return inputs

dataset = dataset.map(preprocess_function, batched=True)`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">preprocess_function</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    audio_arrays = [x[<span class="hljs-string">&quot;array&quot;</span>] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;audio&quot;</span>]]
<span class="hljs-meta">... </span>    inputs = feature_extractor(
<span class="hljs-meta">... </span>        audio_arrays, sampling_rate=feature_extractor.sampling_rate, max_length=<span class="hljs-number">16000</span>, truncation=<span class="hljs-literal">True</span>
<span class="hljs-meta">... </span>    )
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> inputs

<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = dataset.<span class="hljs-built_in">map</span>(preprocess_function, batched=<span class="hljs-literal">True</span>)`}}),qs=new He({}),Ps=new D({props:{code:`from transformers import AutoFeatureExtractor
from datasets import load_dataset, Image

feature_extractor = AutoFeatureExtractor.from_pretrained("google/vit-base-patch16-224-in21k")
dataset = load_dataset("beans", split="train")`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoFeatureExtractor
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Image

<span class="hljs-meta">&gt;&gt;&gt; </span>feature_extractor = AutoFeatureExtractor.from_pretrained(<span class="hljs-string">&quot;google/vit-base-patch16-224-in21k&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = load_dataset(<span class="hljs-string">&quot;beans&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)`}}),Ss=new D({props:{code:'dataset[0]["image"]',highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;image&quot;</span>]'}}),Is=new D({props:{code:`from torchvision.transforms import RandomRotation

rotate = RandomRotation(degrees=(0, 90))
def transforms(examples):
    examples["pixel_values"] = [rotate(image.convert("RGB")) for image in examples["image"]]
    return examples`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torchvision.transforms <span class="hljs-keyword">import</span> RandomRotation

<span class="hljs-meta">&gt;&gt;&gt; </span>rotate = RandomRotation(degrees=(<span class="hljs-number">0</span>, <span class="hljs-number">90</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">transforms</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    examples[<span class="hljs-string">&quot;pixel_values&quot;</span>] = [rotate(image.convert(<span class="hljs-string">&quot;RGB&quot;</span>)) <span class="hljs-keyword">for</span> image <span class="hljs-keyword">in</span> examples[<span class="hljs-string">&quot;image&quot;</span>]]
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> examples`}}),Rs=new D({props:{code:`dataset.set_transform(transforms)
dataset[0]["pixel_values"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>dataset.set_transform(transforms)
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;pixel_values&quot;</span>]`}}),{c(){m=l("meta"),g=h(),u=l("h1"),d=l("a"),_=l("span"),v(b.$$.fragment),y=h(),A=l("span"),j=t("Preprocess"),T=h(),P=l("p"),q=t("In addition to loading datasets, \u{1F917} Datasets other main goal is to offer a diverse set of preprocessing functions to get a dataset into an appropriate format for training with your machine learning framework."),z=h(),L=l("p"),Gs=t("There are many possible ways to preprocess a dataset, and it all depends on your specific dataset. Sometimes you may need to rename a column, and other times you might need to unflatten nested fields. \u{1F917} Datasets provides a way to do most of these things. But in nearly all preprocessing cases, depending on your dataset modality, you\u2019ll need to:"),Ja=h(),W=l("ul"),ta=l("li"),Be=t("Tokenize a text dataset."),Ve=h(),na=l("li"),Ye=t("Resample an audio dataset."),Je=h(),la=l("li"),Ke=t("Apply transforms to an image dataset."),Ka=h(),Ws=l("p"),Qe=t("The last preprocessing step is usually setting your dataset format to be compatible with your machine learning framework\u2019s expected input format."),Qa=h(),Fs=l("p"),Xe=t("In this tutorial, you\u2019ll also need to install the \u{1F917} Transformers library:"),Xa=h(),v(ms.$$.fragment),Za=h(),Ms=l("p"),Ze=t("Grab a dataset of your choice and follow along!"),se=h(),H=l("h2"),es=l("a"),ra=l("span"),v(hs.$$.fragment),st=h(),oa=l("span"),at=t("Tokenize text"),ae=h(),ts=l("p"),et=t("Models cannot process raw text, so you\u2019ll need to convert the text into numbers. Tokenization provides a way to do this by dividing text into individual words called "),pa=l("em"),tt=t("tokens"),nt=t(". Tokens are finally converted to numbers."),ee=h(),v(ns.$$.fragment),te=h(),U=l("p"),ia=l("strong"),lt=t("1"),rt=t(". Start by loading the "),cs=l("a"),ot=t("rotten_tomatoes"),pt=t(" dataset and the tokenizer corresponding to a pretrained "),us=l("a"),it=t("BERT"),mt=t(" model. Using the same tokenizer as the pretrained model is important because you want to make sure the text is split in the same way."),ne=h(),v(fs.$$.fragment),le=h(),B=l("p"),ma=l("strong"),ht=t("2"),ct=t(". Call your tokenizer on the first row of "),ha=l("code"),ut=t("text"),ft=t(" in the dataset:"),re=h(),v(ds.$$.fragment),oe=h(),Ls=l("p"),dt=t("The tokenizer returns a dictionary with three items:"),pe=h(),F=l("ul"),Hs=l("li"),ca=l("code"),gt=t("input_ids"),bt=t(": the numbers representing the tokens in the text."),jt=h(),Bs=l("li"),ua=l("code"),_t=t("token_type_ids"),yt=t(": indicates which sequence a token belongs to if there is more than one sequence."),vt=h(),Vs=l("li"),fa=l("code"),$t=t("attention_mask"),kt=t(": indicates whether a token should be masked or not."),ie=h(),Ys=l("p"),wt=t("These values are actually the model inputs."),me=h(),I=l("p"),da=l("strong"),xt=t("3"),Et=t(". The fastest way to tokenize your entire dataset is to use the "),Js=l("a"),Tt=t("map()"),qt=t(" function. This function speeds up tokenization by applying the tokenizer to batches of examples instead of individual examples. Set the "),ga=l("code"),At=t("batched"),zt=t(" parameter to "),ba=l("code"),Pt=t("True"),St=t(":"),he=h(),v(gs.$$.fragment),ce=h(),bs=l("p"),ja=l("strong"),Dt=t("4"),It=t(". Set the format of your dataset to be compatible with your machine learning framework:"),ue=h(),v(ls.$$.fragment),fe=h(),js=l("p"),_a=l("strong"),Rt=t("5"),Ct=t(". The dataset is now ready for training with your machine learning framework!"),de=h(),V=l("h2"),rs=l("a"),ya=l("span"),v(_s.$$.fragment),Ot=h(),va=l("span"),Nt=t("Resample audio signals"),ge=h(),os=l("p"),Ut=t("Audio inputs like text datasets need to be divided into discrete data points. This is known as "),$a=l("em"),Gt=t("sampling"),Wt=t("; the sampling rate tells you how much of the speech signal is captured per second. It is important to make sure the sampling rate of your dataset matches the sampling rate of the data used to pretrain the model you\u2019re using. If the sampling rates are different, the pretrained model may perform poorly on your dataset because it doesn\u2019t recognize the differences in the sampling rate."),be=h(),R=l("p"),ka=l("strong"),Ft=t("1"),Mt=t(". Start by loading the "),ys=l("a"),Lt=t("MInDS-14"),Ht=t(" dataset, the "),Ks=l("a"),Bt=t("Audio"),Vt=t(" feature, and the feature extractor corresponding to a pretrained "),vs=l("a"),Yt=t("Wav2Vec2"),Jt=t(" model:"),je=h(),v($s.$$.fragment),_e=h(),Y=l("p"),wa=l("strong"),Kt=t("2"),Qt=t(". Index into the first row of the dataset. When you call the "),xa=l("code"),Xt=t("audio"),Zt=t(" column of the dataset, it is automatically decoded and resampled:"),ye=h(),v(ks.$$.fragment),ve=h(),ws=l("p"),Ea=l("strong"),sn=t("3"),an=t(". Reading a dataset card is incredibly useful and can give you a lot of information about the dataset. A quick look at the MInDS-14 dataset card tells you the sampling rate is 8kHz. Likewise, you can get many details about a model from its model card. The Wav2Vec2 model card says it was sampled on 16kHz speech audio. This means you\u2019ll need to upsample the MInDS-14 dataset to match the sampling rate of the model."),$e=h(),S=l("p"),en=t("Use the "),Qs=l("a"),tn=t("cast_column()"),nn=t(" function and set the "),Ta=l("code"),ln=t("sampling_rate"),rn=t(" parameter in the "),Xs=l("a"),on=t("Audio"),pn=t(" feature to upsample the audio signal. When you call the "),qa=l("code"),mn=t("audio"),hn=t(" column now, it is decoded and resampled to 16kHz:"),ke=h(),v(xs.$$.fragment),we=h(),C=l("p"),Aa=l("strong"),cn=t("4"),un=t(". Use the "),Zs=l("a"),fn=t("map()"),dn=t(" function to resample the entire dataset to 16kHz. This function speeds up resampling by applying the feature extractor to batches of examples instead of individual examples. Set the "),za=l("code"),gn=t("batched"),bn=t(" parameter to "),Pa=l("code"),jn=t("True"),_n=t(":"),xe=h(),v(Es.$$.fragment),Ee=h(),Ts=l("p"),Sa=l("strong"),yn=t("5"),vn=t(". The dataset is now ready for training with your machine learning framework!"),Te=h(),J=l("h2"),ps=l("a"),Da=l("span"),v(qs.$$.fragment),$n=h(),Ia=l("span"),kn=t("Apply data augmentations"),qe=h(),is=l("p"),wn=t("The most common preprocessing you\u2019ll do with image datasets is "),Ra=l("em"),xn=t("data augmentation"),En=t(", a process that introduces random variations to an image without changing the meaning of the data. This can mean changing the color properties of an image or randomly cropping an image. You are free to use any data augmentation library you like, and \u{1F917} Datasets will help you apply your data augmentations to your dataset."),Ae=h(),O=l("p"),Ca=l("strong"),Tn=t("1"),qn=t(". Start by loading the "),As=l("a"),An=t("Beans"),zn=t(" dataset, the "),Oa=l("code"),Pn=t("Image"),Sn=t(" feature, and the feature extractor corresponding to a pretrained "),zs=l("a"),Dn=t("ViT"),In=t(" model:"),ze=h(),v(Ps.$$.fragment),Pe=h(),K=l("p"),Na=l("strong"),Rn=t("2"),Cn=t(". Index into the first row of the dataset. When you call the "),Ua=l("code"),On=t("image"),Nn=t(" column of the dataset, the underlying PIL object is automatically decoded into an image."),Se=h(),v(Ss.$$.fragment),De=h(),Q=l("p"),Ga=l("strong"),Un=t("3"),Gn=t(". Now, you can apply some transforms to the image. Feel free to take a look at the "),Ds=l("a"),Wn=t("various transforms available"),Fn=t(" in torchvision and choose one you\u2019d like to experiment with. This example applies a transform that randomly rotates the image:"),Ie=h(),v(Is.$$.fragment),Re=h(),G=l("p"),Wa=l("strong"),Mn=t("4"),Ln=t(". Use the "),sa=l("a"),Hn=t("set_transform()"),Bn=t(" function to apply the transform on-the-fly. When you index into the image "),Fa=l("code"),Vn=t("pixel_values"),Yn=t(", the transform is applied, and your image gets rotated."),Ce=h(),v(Rs.$$.fragment),Oe=h(),Cs=l("p"),Ma=l("strong"),Jn=t("5"),Kn=t(". The dataset is now ready for training with your machine learning framework!"),this.h()},l(s){const p=$r('[data-svelte="svelte-1phssyn"]',document.head);m=r(p,"META",{name:!0,content:!0}),p.forEach(e),g=c(s),u=r(s,"H1",{class:!0});var Os=o(u);d=r(Os,"A",{id:!0,class:!0,href:!0});var La=o(d);_=r(La,"SPAN",{});var ll=o(_);$(b.$$.fragment,ll),ll.forEach(e),La.forEach(e),y=c(Os),A=r(Os,"SPAN",{});var rl=o(A);j=n(rl,"Preprocess"),rl.forEach(e),Os.forEach(e),T=c(s),P=r(s,"P",{});var ol=o(P);q=n(ol,"In addition to loading datasets, \u{1F917} Datasets other main goal is to offer a diverse set of preprocessing functions to get a dataset into an appropriate format for training with your machine learning framework."),ol.forEach(e),z=c(s),L=r(s,"P",{});var pl=o(L);Gs=n(pl,"There are many possible ways to preprocess a dataset, and it all depends on your specific dataset. Sometimes you may need to rename a column, and other times you might need to unflatten nested fields. \u{1F917} Datasets provides a way to do most of these things. But in nearly all preprocessing cases, depending on your dataset modality, you\u2019ll need to:"),pl.forEach(e),Ja=c(s),W=r(s,"UL",{});var aa=o(W);ta=r(aa,"LI",{});var il=o(ta);Be=n(il,"Tokenize a text dataset."),il.forEach(e),Ve=c(aa),na=r(aa,"LI",{});var ml=o(na);Ye=n(ml,"Resample an audio dataset."),ml.forEach(e),Je=c(aa),la=r(aa,"LI",{});var hl=o(la);Ke=n(hl,"Apply transforms to an image dataset."),hl.forEach(e),aa.forEach(e),Ka=c(s),Ws=r(s,"P",{});var cl=o(Ws);Qe=n(cl,"The last preprocessing step is usually setting your dataset format to be compatible with your machine learning framework\u2019s expected input format."),cl.forEach(e),Qa=c(s),Fs=r(s,"P",{});var ul=o(Fs);Xe=n(ul,"In this tutorial, you\u2019ll also need to install the \u{1F917} Transformers library:"),ul.forEach(e),Xa=c(s),$(ms.$$.fragment,s),Za=c(s),Ms=r(s,"P",{});var fl=o(Ms);Ze=n(fl,"Grab a dataset of your choice and follow along!"),fl.forEach(e),se=c(s),H=r(s,"H2",{class:!0});var Ue=o(H);es=r(Ue,"A",{id:!0,class:!0,href:!0});var dl=o(es);ra=r(dl,"SPAN",{});var gl=o(ra);$(hs.$$.fragment,gl),gl.forEach(e),dl.forEach(e),st=c(Ue),oa=r(Ue,"SPAN",{});var bl=o(oa);at=n(bl,"Tokenize text"),bl.forEach(e),Ue.forEach(e),ae=c(s),ts=r(s,"P",{});var Ge=o(ts);et=n(Ge,"Models cannot process raw text, so you\u2019ll need to convert the text into numbers. Tokenization provides a way to do this by dividing text into individual words called "),pa=r(Ge,"EM",{});var jl=o(pa);tt=n(jl,"tokens"),jl.forEach(e),nt=n(Ge,". Tokens are finally converted to numbers."),Ge.forEach(e),ee=c(s),$(ns.$$.fragment,s),te=c(s),U=r(s,"P",{});var Ns=o(U);ia=r(Ns,"STRONG",{});var _l=o(ia);lt=n(_l,"1"),_l.forEach(e),rt=n(Ns,". Start by loading the "),cs=r(Ns,"A",{href:!0,rel:!0});var yl=o(cs);ot=n(yl,"rotten_tomatoes"),yl.forEach(e),pt=n(Ns," dataset and the tokenizer corresponding to a pretrained "),us=r(Ns,"A",{href:!0,rel:!0});var vl=o(us);it=n(vl,"BERT"),vl.forEach(e),mt=n(Ns," model. Using the same tokenizer as the pretrained model is important because you want to make sure the text is split in the same way."),Ns.forEach(e),ne=c(s),$(fs.$$.fragment,s),le=c(s),B=r(s,"P",{});var Ha=o(B);ma=r(Ha,"STRONG",{});var $l=o(ma);ht=n($l,"2"),$l.forEach(e),ct=n(Ha,". Call your tokenizer on the first row of "),ha=r(Ha,"CODE",{});var kl=o(ha);ut=n(kl,"text"),kl.forEach(e),ft=n(Ha," in the dataset:"),Ha.forEach(e),re=c(s),$(ds.$$.fragment,s),oe=c(s),Ls=r(s,"P",{});var wl=o(Ls);dt=n(wl,"The tokenizer returns a dictionary with three items:"),wl.forEach(e),pe=c(s),F=r(s,"UL",{});var ea=o(F);Hs=r(ea,"LI",{});var Qn=o(Hs);ca=r(Qn,"CODE",{});var xl=o(ca);gt=n(xl,"input_ids"),xl.forEach(e),bt=n(Qn,": the numbers representing the tokens in the text."),Qn.forEach(e),jt=c(ea),Bs=r(ea,"LI",{});var Xn=o(Bs);ua=r(Xn,"CODE",{});var El=o(ua);_t=n(El,"token_type_ids"),El.forEach(e),yt=n(Xn,": indicates which sequence a token belongs to if there is more than one sequence."),Xn.forEach(e),vt=c(ea),Vs=r(ea,"LI",{});var Zn=o(Vs);fa=r(Zn,"CODE",{});var Tl=o(fa);$t=n(Tl,"attention_mask"),Tl.forEach(e),kt=n(Zn,": indicates whether a token should be masked or not."),Zn.forEach(e),ea.forEach(e),ie=c(s),Ys=r(s,"P",{});var ql=o(Ys);wt=n(ql,"These values are actually the model inputs."),ql.forEach(e),me=c(s),I=r(s,"P",{});var X=o(I);da=r(X,"STRONG",{});var Al=o(da);xt=n(Al,"3"),Al.forEach(e),Et=n(X,". The fastest way to tokenize your entire dataset is to use the "),Js=r(X,"A",{href:!0});var zl=o(Js);Tt=n(zl,"map()"),zl.forEach(e),qt=n(X," function. This function speeds up tokenization by applying the tokenizer to batches of examples instead of individual examples. Set the "),ga=r(X,"CODE",{});var Pl=o(ga);At=n(Pl,"batched"),Pl.forEach(e),zt=n(X," parameter to "),ba=r(X,"CODE",{});var Sl=o(ba);Pt=n(Sl,"True"),Sl.forEach(e),St=n(X,":"),X.forEach(e),he=c(s),$(gs.$$.fragment,s),ce=c(s),bs=r(s,"P",{});var sl=o(bs);ja=r(sl,"STRONG",{});var Dl=o(ja);Dt=n(Dl,"4"),Dl.forEach(e),It=n(sl,". Set the format of your dataset to be compatible with your machine learning framework:"),sl.forEach(e),ue=c(s),$(ls.$$.fragment,s),fe=c(s),js=r(s,"P",{});var al=o(js);_a=r(al,"STRONG",{});var Il=o(_a);Rt=n(Il,"5"),Il.forEach(e),Ct=n(al,". The dataset is now ready for training with your machine learning framework!"),al.forEach(e),de=c(s),V=r(s,"H2",{class:!0});var We=o(V);rs=r(We,"A",{id:!0,class:!0,href:!0});var Rl=o(rs);ya=r(Rl,"SPAN",{});var Cl=o(ya);$(_s.$$.fragment,Cl),Cl.forEach(e),Rl.forEach(e),Ot=c(We),va=r(We,"SPAN",{});var Ol=o(va);Nt=n(Ol,"Resample audio signals"),Ol.forEach(e),We.forEach(e),ge=c(s),os=r(s,"P",{});var Fe=o(os);Ut=n(Fe,"Audio inputs like text datasets need to be divided into discrete data points. This is known as "),$a=r(Fe,"EM",{});var Nl=o($a);Gt=n(Nl,"sampling"),Nl.forEach(e),Wt=n(Fe,"; the sampling rate tells you how much of the speech signal is captured per second. It is important to make sure the sampling rate of your dataset matches the sampling rate of the data used to pretrain the model you\u2019re using. If the sampling rates are different, the pretrained model may perform poorly on your dataset because it doesn\u2019t recognize the differences in the sampling rate."),Fe.forEach(e),be=c(s),R=r(s,"P",{});var Z=o(R);ka=r(Z,"STRONG",{});var Ul=o(ka);Ft=n(Ul,"1"),Ul.forEach(e),Mt=n(Z,". Start by loading the "),ys=r(Z,"A",{href:!0,rel:!0});var Gl=o(ys);Lt=n(Gl,"MInDS-14"),Gl.forEach(e),Ht=n(Z," dataset, the "),Ks=r(Z,"A",{href:!0});var Wl=o(Ks);Bt=n(Wl,"Audio"),Wl.forEach(e),Vt=n(Z," feature, and the feature extractor corresponding to a pretrained "),vs=r(Z,"A",{href:!0,rel:!0});var Fl=o(vs);Yt=n(Fl,"Wav2Vec2"),Fl.forEach(e),Jt=n(Z," model:"),Z.forEach(e),je=c(s),$($s.$$.fragment,s),_e=c(s),Y=r(s,"P",{});var Ba=o(Y);wa=r(Ba,"STRONG",{});var Ml=o(wa);Kt=n(Ml,"2"),Ml.forEach(e),Qt=n(Ba,". Index into the first row of the dataset. When you call the "),xa=r(Ba,"CODE",{});var Ll=o(xa);Xt=n(Ll,"audio"),Ll.forEach(e),Zt=n(Ba," column of the dataset, it is automatically decoded and resampled:"),Ba.forEach(e),ye=c(s),$(ks.$$.fragment,s),ve=c(s),ws=r(s,"P",{});var el=o(ws);Ea=r(el,"STRONG",{});var Hl=o(Ea);sn=n(Hl,"3"),Hl.forEach(e),an=n(el,". Reading a dataset card is incredibly useful and can give you a lot of information about the dataset. A quick look at the MInDS-14 dataset card tells you the sampling rate is 8kHz. Likewise, you can get many details about a model from its model card. The Wav2Vec2 model card says it was sampled on 16kHz speech audio. This means you\u2019ll need to upsample the MInDS-14 dataset to match the sampling rate of the model."),el.forEach(e),$e=c(s),S=r(s,"P",{});var M=o(S);en=n(M,"Use the "),Qs=r(M,"A",{href:!0});var Bl=o(Qs);tn=n(Bl,"cast_column()"),Bl.forEach(e),nn=n(M," function and set the "),Ta=r(M,"CODE",{});var Vl=o(Ta);ln=n(Vl,"sampling_rate"),Vl.forEach(e),rn=n(M," parameter in the "),Xs=r(M,"A",{href:!0});var Yl=o(Xs);on=n(Yl,"Audio"),Yl.forEach(e),pn=n(M," feature to upsample the audio signal. When you call the "),qa=r(M,"CODE",{});var Jl=o(qa);mn=n(Jl,"audio"),Jl.forEach(e),hn=n(M," column now, it is decoded and resampled to 16kHz:"),M.forEach(e),ke=c(s),$(xs.$$.fragment,s),we=c(s),C=r(s,"P",{});var ss=o(C);Aa=r(ss,"STRONG",{});var Kl=o(Aa);cn=n(Kl,"4"),Kl.forEach(e),un=n(ss,". Use the "),Zs=r(ss,"A",{href:!0});var Ql=o(Zs);fn=n(Ql,"map()"),Ql.forEach(e),dn=n(ss," function to resample the entire dataset to 16kHz. This function speeds up resampling by applying the feature extractor to batches of examples instead of individual examples. Set the "),za=r(ss,"CODE",{});var Xl=o(za);gn=n(Xl,"batched"),Xl.forEach(e),bn=n(ss," parameter to "),Pa=r(ss,"CODE",{});var Zl=o(Pa);jn=n(Zl,"True"),Zl.forEach(e),_n=n(ss,":"),ss.forEach(e),xe=c(s),$(Es.$$.fragment,s),Ee=c(s),Ts=r(s,"P",{});var tl=o(Ts);Sa=r(tl,"STRONG",{});var sr=o(Sa);yn=n(sr,"5"),sr.forEach(e),vn=n(tl,". The dataset is now ready for training with your machine learning framework!"),tl.forEach(e),Te=c(s),J=r(s,"H2",{class:!0});var Me=o(J);ps=r(Me,"A",{id:!0,class:!0,href:!0});var ar=o(ps);Da=r(ar,"SPAN",{});var er=o(Da);$(qs.$$.fragment,er),er.forEach(e),ar.forEach(e),$n=c(Me),Ia=r(Me,"SPAN",{});var tr=o(Ia);kn=n(tr,"Apply data augmentations"),tr.forEach(e),Me.forEach(e),qe=c(s),is=r(s,"P",{});var Le=o(is);wn=n(Le,"The most common preprocessing you\u2019ll do with image datasets is "),Ra=r(Le,"EM",{});var nr=o(Ra);xn=n(nr,"data augmentation"),nr.forEach(e),En=n(Le,", a process that introduces random variations to an image without changing the meaning of the data. This can mean changing the color properties of an image or randomly cropping an image. You are free to use any data augmentation library you like, and \u{1F917} Datasets will help you apply your data augmentations to your dataset."),Le.forEach(e),Ae=c(s),O=r(s,"P",{});var as=o(O);Ca=r(as,"STRONG",{});var lr=o(Ca);Tn=n(lr,"1"),lr.forEach(e),qn=n(as,". Start by loading the "),As=r(as,"A",{href:!0,rel:!0});var rr=o(As);An=n(rr,"Beans"),rr.forEach(e),zn=n(as," dataset, the "),Oa=r(as,"CODE",{});var or=o(Oa);Pn=n(or,"Image"),or.forEach(e),Sn=n(as," feature, and the feature extractor corresponding to a pretrained "),zs=r(as,"A",{href:!0,rel:!0});var pr=o(zs);Dn=n(pr,"ViT"),pr.forEach(e),In=n(as," model:"),as.forEach(e),ze=c(s),$(Ps.$$.fragment,s),Pe=c(s),K=r(s,"P",{});var Va=o(K);Na=r(Va,"STRONG",{});var ir=o(Na);Rn=n(ir,"2"),ir.forEach(e),Cn=n(Va,". Index into the first row of the dataset. When you call the "),Ua=r(Va,"CODE",{});var mr=o(Ua);On=n(mr,"image"),mr.forEach(e),Nn=n(Va," column of the dataset, the underlying PIL object is automatically decoded into an image."),Va.forEach(e),Se=c(s),$(Ss.$$.fragment,s),De=c(s),Q=r(s,"P",{});var Ya=o(Q);Ga=r(Ya,"STRONG",{});var hr=o(Ga);Un=n(hr,"3"),hr.forEach(e),Gn=n(Ya,". Now, you can apply some transforms to the image. Feel free to take a look at the "),Ds=r(Ya,"A",{href:!0,rel:!0});var cr=o(Ds);Wn=n(cr,"various transforms available"),cr.forEach(e),Fn=n(Ya," in torchvision and choose one you\u2019d like to experiment with. This example applies a transform that randomly rotates the image:"),Ya.forEach(e),Ie=c(s),$(Is.$$.fragment,s),Re=c(s),G=r(s,"P",{});var Us=o(G);Wa=r(Us,"STRONG",{});var ur=o(Wa);Mn=n(ur,"4"),ur.forEach(e),Ln=n(Us,". Use the "),sa=r(Us,"A",{href:!0});var fr=o(sa);Hn=n(fr,"set_transform()"),fr.forEach(e),Bn=n(Us," function to apply the transform on-the-fly. When you index into the image "),Fa=r(Us,"CODE",{});var dr=o(Fa);Vn=n(dr,"pixel_values"),dr.forEach(e),Yn=n(Us,", the transform is applied, and your image gets rotated."),Us.forEach(e),Ce=c(s),$(Rs.$$.fragment,s),Oe=c(s),Cs=r(s,"P",{});var nl=o(Cs);Ma=r(nl,"STRONG",{});var gr=o(Ma);Jn=n(gr,"5"),gr.forEach(e),Kn=n(nl,". The dataset is now ready for training with your machine learning framework!"),nl.forEach(e),this.h()},h(){f(m,"name","hf:doc:metadata"),f(m,"content",JSON.stringify(Sr)),f(d,"id","preprocess"),f(d,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(d,"href","#preprocess"),f(u,"class","relative group"),f(es,"id","tokenize-text"),f(es,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(es,"href","#tokenize-text"),f(H,"class","relative group"),f(cs,"href","https://huggingface.co/datasets/rotten_tomatoes"),f(cs,"rel","nofollow"),f(us,"href","https://huggingface.co/bert-base-uncased"),f(us,"rel","nofollow"),f(Js,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.map"),f(rs,"id","resample-audio-signals"),f(rs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(rs,"href","#resample-audio-signals"),f(V,"class","relative group"),f(ys,"href","https://huggingface.co/datasets/PolyAI/minds14"),f(ys,"rel","nofollow"),f(Ks,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Audio"),f(vs,"href","https://huggingface.co/facebook/wav2vec2-base-960h"),f(vs,"rel","nofollow"),f(Qs,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.cast_column"),f(Xs,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Audio"),f(Zs,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.map"),f(ps,"id","apply-data-augmentations"),f(ps,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),f(ps,"href","#apply-data-augmentations"),f(J,"class","relative group"),f(As,"href","https://huggingface.co/datasets/beans"),f(As,"rel","nofollow"),f(zs,"href","https://huggingface.co/google/vit-base-patch16-224-in21k"),f(zs,"rel","nofollow"),f(Ds,"href","https://pytorch.org/vision/stable/auto_examples/plot_transforms.html#sphx-glr-auto-examples-plot-transforms-py"),f(Ds,"rel","nofollow"),f(sa,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.set_transform")},m(s,p){a(document.head,m),i(s,g,p),i(s,u,p),a(u,d),a(d,_),k(b,_,null),a(u,y),a(u,A),a(A,j),i(s,T,p),i(s,P,p),a(P,q),i(s,z,p),i(s,L,p),a(L,Gs),i(s,Ja,p),i(s,W,p),a(W,ta),a(ta,Be),a(W,Ve),a(W,na),a(na,Ye),a(W,Je),a(W,la),a(la,Ke),i(s,Ka,p),i(s,Ws,p),a(Ws,Qe),i(s,Qa,p),i(s,Fs,p),a(Fs,Xe),i(s,Xa,p),k(ms,s,p),i(s,Za,p),i(s,Ms,p),a(Ms,Ze),i(s,se,p),i(s,H,p),a(H,es),a(es,ra),k(hs,ra,null),a(H,st),a(H,oa),a(oa,at),i(s,ae,p),i(s,ts,p),a(ts,et),a(ts,pa),a(pa,tt),a(ts,nt),i(s,ee,p),k(ns,s,p),i(s,te,p),i(s,U,p),a(U,ia),a(ia,lt),a(U,rt),a(U,cs),a(cs,ot),a(U,pt),a(U,us),a(us,it),a(U,mt),i(s,ne,p),k(fs,s,p),i(s,le,p),i(s,B,p),a(B,ma),a(ma,ht),a(B,ct),a(B,ha),a(ha,ut),a(B,ft),i(s,re,p),k(ds,s,p),i(s,oe,p),i(s,Ls,p),a(Ls,dt),i(s,pe,p),i(s,F,p),a(F,Hs),a(Hs,ca),a(ca,gt),a(Hs,bt),a(F,jt),a(F,Bs),a(Bs,ua),a(ua,_t),a(Bs,yt),a(F,vt),a(F,Vs),a(Vs,fa),a(fa,$t),a(Vs,kt),i(s,ie,p),i(s,Ys,p),a(Ys,wt),i(s,me,p),i(s,I,p),a(I,da),a(da,xt),a(I,Et),a(I,Js),a(Js,Tt),a(I,qt),a(I,ga),a(ga,At),a(I,zt),a(I,ba),a(ba,Pt),a(I,St),i(s,he,p),k(gs,s,p),i(s,ce,p),i(s,bs,p),a(bs,ja),a(ja,Dt),a(bs,It),i(s,ue,p),k(ls,s,p),i(s,fe,p),i(s,js,p),a(js,_a),a(_a,Rt),a(js,Ct),i(s,de,p),i(s,V,p),a(V,rs),a(rs,ya),k(_s,ya,null),a(V,Ot),a(V,va),a(va,Nt),i(s,ge,p),i(s,os,p),a(os,Ut),a(os,$a),a($a,Gt),a(os,Wt),i(s,be,p),i(s,R,p),a(R,ka),a(ka,Ft),a(R,Mt),a(R,ys),a(ys,Lt),a(R,Ht),a(R,Ks),a(Ks,Bt),a(R,Vt),a(R,vs),a(vs,Yt),a(R,Jt),i(s,je,p),k($s,s,p),i(s,_e,p),i(s,Y,p),a(Y,wa),a(wa,Kt),a(Y,Qt),a(Y,xa),a(xa,Xt),a(Y,Zt),i(s,ye,p),k(ks,s,p),i(s,ve,p),i(s,ws,p),a(ws,Ea),a(Ea,sn),a(ws,an),i(s,$e,p),i(s,S,p),a(S,en),a(S,Qs),a(Qs,tn),a(S,nn),a(S,Ta),a(Ta,ln),a(S,rn),a(S,Xs),a(Xs,on),a(S,pn),a(S,qa),a(qa,mn),a(S,hn),i(s,ke,p),k(xs,s,p),i(s,we,p),i(s,C,p),a(C,Aa),a(Aa,cn),a(C,un),a(C,Zs),a(Zs,fn),a(C,dn),a(C,za),a(za,gn),a(C,bn),a(C,Pa),a(Pa,jn),a(C,_n),i(s,xe,p),k(Es,s,p),i(s,Ee,p),i(s,Ts,p),a(Ts,Sa),a(Sa,yn),a(Ts,vn),i(s,Te,p),i(s,J,p),a(J,ps),a(ps,Da),k(qs,Da,null),a(J,$n),a(J,Ia),a(Ia,kn),i(s,qe,p),i(s,is,p),a(is,wn),a(is,Ra),a(Ra,xn),a(is,En),i(s,Ae,p),i(s,O,p),a(O,Ca),a(Ca,Tn),a(O,qn),a(O,As),a(As,An),a(O,zn),a(O,Oa),a(Oa,Pn),a(O,Sn),a(O,zs),a(zs,Dn),a(O,In),i(s,ze,p),k(Ps,s,p),i(s,Pe,p),i(s,K,p),a(K,Na),a(Na,Rn),a(K,Cn),a(K,Ua),a(Ua,On),a(K,Nn),i(s,Se,p),k(Ss,s,p),i(s,De,p),i(s,Q,p),a(Q,Ga),a(Ga,Un),a(Q,Gn),a(Q,Ds),a(Ds,Wn),a(Q,Fn),i(s,Ie,p),k(Is,s,p),i(s,Re,p),i(s,G,p),a(G,Wa),a(Wa,Mn),a(G,Ln),a(G,sa),a(sa,Hn),a(G,Bn),a(G,Fa),a(Fa,Vn),a(G,Yn),i(s,Ce,p),k(Rs,s,p),i(s,Oe,p),i(s,Cs,p),a(Cs,Ma),a(Ma,Jn),a(Cs,Kn),Ne=!0},p(s,[p]){const Os={};p&2&&(Os.$$scope={dirty:p,ctx:s}),ns.$set(Os);const La={};p&2&&(La.$$scope={dirty:p,ctx:s}),ls.$set(La)},i(s){Ne||(w(b.$$.fragment,s),w(ms.$$.fragment,s),w(hs.$$.fragment,s),w(ns.$$.fragment,s),w(fs.$$.fragment,s),w(ds.$$.fragment,s),w(gs.$$.fragment,s),w(ls.$$.fragment,s),w(_s.$$.fragment,s),w($s.$$.fragment,s),w(ks.$$.fragment,s),w(xs.$$.fragment,s),w(Es.$$.fragment,s),w(qs.$$.fragment,s),w(Ps.$$.fragment,s),w(Ss.$$.fragment,s),w(Is.$$.fragment,s),w(Rs.$$.fragment,s),Ne=!0)},o(s){x(b.$$.fragment,s),x(ms.$$.fragment,s),x(hs.$$.fragment,s),x(ns.$$.fragment,s),x(fs.$$.fragment,s),x(ds.$$.fragment,s),x(gs.$$.fragment,s),x(ls.$$.fragment,s),x(_s.$$.fragment,s),x($s.$$.fragment,s),x(ks.$$.fragment,s),x(xs.$$.fragment,s),x(Es.$$.fragment,s),x(qs.$$.fragment,s),x(Ps.$$.fragment,s),x(Ss.$$.fragment,s),x(Is.$$.fragment,s),x(Rs.$$.fragment,s),Ne=!1},d(s){e(m),s&&e(g),s&&e(u),E(b),s&&e(T),s&&e(P),s&&e(z),s&&e(L),s&&e(Ja),s&&e(W),s&&e(Ka),s&&e(Ws),s&&e(Qa),s&&e(Fs),s&&e(Xa),E(ms,s),s&&e(Za),s&&e(Ms),s&&e(se),s&&e(H),E(hs),s&&e(ae),s&&e(ts),s&&e(ee),E(ns,s),s&&e(te),s&&e(U),s&&e(ne),E(fs,s),s&&e(le),s&&e(B),s&&e(re),E(ds,s),s&&e(oe),s&&e(Ls),s&&e(pe),s&&e(F),s&&e(ie),s&&e(Ys),s&&e(me),s&&e(I),s&&e(he),E(gs,s),s&&e(ce),s&&e(bs),s&&e(ue),E(ls,s),s&&e(fe),s&&e(js),s&&e(de),s&&e(V),E(_s),s&&e(ge),s&&e(os),s&&e(be),s&&e(R),s&&e(je),E($s,s),s&&e(_e),s&&e(Y),s&&e(ye),E(ks,s),s&&e(ve),s&&e(ws),s&&e($e),s&&e(S),s&&e(ke),E(xs,s),s&&e(we),s&&e(C),s&&e(xe),E(Es,s),s&&e(Ee),s&&e(Ts),s&&e(Te),s&&e(J),E(qs),s&&e(qe),s&&e(is),s&&e(Ae),s&&e(O),s&&e(ze),E(Ps,s),s&&e(Pe),s&&e(K),s&&e(Se),E(Ss,s),s&&e(De),s&&e(Q),s&&e(Ie),E(Is,s),s&&e(Re),s&&e(G),s&&e(Ce),E(Rs,s),s&&e(Oe),s&&e(Cs)}}}const Sr={local:"preprocess",sections:[{local:"tokenize-text",title:"Tokenize text"},{local:"resample-audio-signals",title:"Resample audio signals"},{local:"apply-data-augmentations",title:"Apply data augmentations"}],title:"Preprocess"};function Dr(N){return kr(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Gr extends _r{constructor(m){super();yr(this,m,Dr,Pr,vr,{})}}export{Gr as default,Sr as metadata};
