import{S as ap,i as tp,s as ep,e as l,k as h,w as b,t as e,M as np,c as o,d as t,m as d,a as r,x as _,h as n,b as c,G as a,g as i,y as v,q as w,o as j,B as $,v as lp}from"../chunks/vendor-hf-doc-builder.js";import{T as $o}from"../chunks/Tip-hf-doc-builder.js";import{I as Ts}from"../chunks/IconCopyLink-hf-doc-builder.js";import{C as B}from"../chunks/CodeBlock-hf-doc-builder.js";function op(is){let u,A,f,E,C;return{c(){u=l("p"),A=e("A "),f=l("a"),E=e("Dataset"),C=e(" object is a wrapper of an Arrow table, which allows fast reads from arrays in the dataset to TensorFlow tensors."),this.h()},l(g){u=o(g,"P",{});var k=r(u);A=n(k,"A "),f=o(k,"A",{href:!0});var O=r(f);E=n(O,"Dataset"),O.forEach(t),C=n(k," object is a wrapper of an Arrow table, which allows fast reads from arrays in the dataset to TensorFlow tensors."),k.forEach(t),this.h()},h(){c(f,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset")},m(g,k){i(g,u,k),a(u,A),a(u,f),a(f,E),a(u,C)},d(g){g&&t(u)}}}function rp(is){let u,A,f,E,C,g,k,O,I,Y,y;return{c(){u=l("p"),A=e("To use the "),f=l("a"),E=e("Image"),C=e(" feature type, you\u2019ll need to install the "),g=l("code"),k=e("vision"),O=e(` extra as
`),I=l("code"),Y=e("pip install datasets[vision]"),y=e("."),this.h()},l(N){u=o(N,"P",{});var m=r(u);A=n(m,"To use the "),f=o(m,"A",{href:!0});var ss=r(f);E=n(ss,"Image"),ss.forEach(t),C=n(m," feature type, you\u2019ll need to install the "),g=o(m,"CODE",{});var as=r(g);k=n(as,"vision"),as.forEach(t),O=n(m,` extra as
`),I=o(m,"CODE",{});var M=r(I);Y=n(M,"pip install datasets[vision]"),M.forEach(t),y=n(m,"."),m.forEach(t),this.h()},h(){c(f,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Image")},m(N,m){i(N,u,m),a(u,A),a(u,f),a(f,E),a(u,C),a(u,g),a(g,k),a(u,O),a(u,I),a(I,Y),a(u,y)},d(N){N&&t(u)}}}function pp(is){let u,A,f,E,C,g,k,O,I,Y,y;return{c(){u=l("p"),A=e("To use the "),f=l("a"),E=e("Audio"),C=e(" feature type, you\u2019ll need to install the "),g=l("code"),k=e("audio"),O=e(` extra as
`),I=l("code"),Y=e("pip install datasets[audio]"),y=e("."),this.h()},l(N){u=o(N,"P",{});var m=r(u);A=n(m,"To use the "),f=o(m,"A",{href:!0});var ss=r(f);E=n(ss,"Audio"),ss.forEach(t),C=n(m," feature type, you\u2019ll need to install the "),g=o(m,"CODE",{});var as=r(g);k=n(as,"audio"),as.forEach(t),O=n(m,` extra as
`),I=o(m,"CODE",{});var M=r(I);Y=n(M,"pip install datasets[audio]"),M.forEach(t),y=n(m,"."),m.forEach(t),this.h()},h(){c(f,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Audio")},m(N,m){i(N,u,m),a(u,A),a(u,f),a(f,E),a(u,C),a(u,g),a(g,k),a(u,O),a(u,I),a(I,Y),a(u,y)},d(N){N&&t(u)}}}function ip(is){let u,A,f,E,C,g,k,O,I,Y,y,N,m,ss,as,M,ke,Te,fa,xe,qe,ma,Ce,Ae,Et,ts,hs,ga,xs,Oe,ya,Pe,Dt,sa,Fe,kt,ds,Ie,ba,Ne,Se,Tt,qs,xt,cs,qt,K,He,_a,Le,Ue,va,Ye,Re,Ct,Cs,At,es,us,wa,As,We,ja,Me,Ot,fs,ze,$a,Be,Ke,Pt,Os,Ft,ms,Ge,Ea,Je,Qe,It,Ps,Nt,ns,gs,Da,Fs,Ve,ka,Xe,St,Is,aa,Ze,sn,Ht,Ns,Lt,ta,an,Ut,Ss,Yt,ea,tn,Rt,Hs,Wt,na,en,Mt,G,nn,la,ln,on,oa,rn,pn,zt,ys,Bt,Ls,Kt,bs,Gt,Us,Jt,ls,_s,Ta,Ys,hn,xa,dn,Qt,P,cn,qa,un,fn,Ca,mn,gn,Aa,yn,bn,Oa,_n,vn,Pa,wn,jn,Vt,T,$n,Fa,En,Dn,Ia,kn,Tn,Na,xn,qn,Sa,Cn,An,Ha,On,Pn,La,Fn,In,Xt,x,Nn,Ua,Sn,Hn,Ya,Ln,Un,Ra,Yn,Rn,Rs,Wn,Mn,Ws,zn,Bn,Wa,Kn,Gn,Zt,os,vs,Ma,Ms,Jn,ra,Qn,za,Vn,se,ws,Xn,Ba,Zn,sl,ae,zs,te,R,al,Ka,tl,el,Ga,nl,ll,Ja,ol,rl,ee,Bs,ne,q,pl,pa,il,hl,Qa,dl,cl,Ks,ul,fl,Va,ml,gl,Gs,yl,bl,Js,_l,vl,le,J,wl,Xa,jl,$l,Za,El,Dl,oe,rs,js,st,Qs,kl,at,Tl,re,D,xl,tt,ql,Cl,et,Al,Ol,nt,Pl,Fl,lt,Il,Nl,ot,Sl,Hl,rt,Ll,Ul,pt,Yl,Rl,pe,Q,Wl,it,Ml,zl,ht,Bl,Kl,ie,V,Vs,Gl,dt,Jl,Ql,Vl,z,Xl,ct,Zl,so,ut,ao,to,ft,eo,no,lo,U,oo,mt,ro,po,gt,io,ho,yt,co,uo,bt,fo,mo,he,ps,$s,_t,Xs,go,vt,yo,de,Es,bo,wt,_o,vo,ce;return g=new Ts({}),xs=new Ts({}),qs=new B({props:{code:`from datasets import Dataset
data = [[1, 2],[3, 4]]
ds = Dataset.from_dict({"data": data})
ds = ds.with_format("tf")
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>,), dtype=int64, numpy=array([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>])&gt;}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=int64, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
       [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])&gt;}`}}),cs=new $o({props:{$$slots:{default:[op]},$$scope:{ctx:is}}}),Cs=new B({props:{code:"ds[:]",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=int64, numpy=
array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
       [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])&gt;}`}}),As=new Ts({}),Os=new B({props:{code:`from datasets import Dataset
data = [[[1, 2],[3, 4]],[[5, 6],[7, 8]]]
ds = Dataset.from_dict({"data": data})
ds = ds.with_format("tf")
ds[0]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.RaggedTensor [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>], [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]]&gt;}`}}),Ps=new B({props:{code:`from datasets import Dataset, Features, Array2D
data = [[[1, 2],[3, 4]],[[5, 6],[7, 8]]]
features = Features({"data": Array2D(shape=(2, 2), dtype='int32')})
ds = Dataset.from_dict({"data": data}, features=features)
ds = ds.with_format("tf")
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Array2D
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],[[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],[<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]]
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;data&quot;</span>: Array2D(shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=<span class="hljs-string">&#x27;int32&#x27;</span>)})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;data&quot;</span>: data}, features=features)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=int64, numpy=
 array([[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
        [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]])&gt;}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">2</span>, <span class="hljs-number">2</span>), dtype=int64, numpy=
 array([[[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],
         [<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]],
 
        [[<span class="hljs-number">5</span>, <span class="hljs-number">6</span>],
         [<span class="hljs-number">7</span>, <span class="hljs-number">8</span>]]])&gt;}`}}),Fs=new Ts({}),Ns=new B({props:{code:`from datasets import Dataset, Features, ClassLabel
labels = [0, 0, 1]
features = Features({"label": ClassLabel(names=["negative", "positive"])})
ds = Dataset.from_dict({"label": labels}, features=features) 
ds = ds.with_format("tf")  
ds[:3]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, ClassLabel
<span class="hljs-meta">&gt;&gt;&gt; </span>labels = [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;label&quot;</span>: ClassLabel(names=[<span class="hljs-string">&quot;negative&quot;</span>, <span class="hljs-string">&quot;positive&quot;</span>])})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;label&quot;</span>: labels}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">3</span>]
{<span class="hljs-string">&#x27;label&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">3</span>,), dtype=int64, numpy=array([<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">1</span>])&gt;}`}}),Ss=new B({props:{code:`from datasets import Dataset, Features 
text = ["foo", "bar"]
data = [0, 1] 
ds = Dataset.from_dict({"text": text, "data": data})  
ds = ds.with_format("tf") 
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features 
<span class="hljs-meta">&gt;&gt;&gt; </span>text = [<span class="hljs-string">&quot;foo&quot;</span>, <span class="hljs-string">&quot;bar&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>data = [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>] 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;text&quot;</span>: text, <span class="hljs-string">&quot;data&quot;</span>: data})  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;text&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>,), dtype=string, numpy=array([<span class="hljs-string">b&#x27;foo&#x27;</span>, <span class="hljs-string">b&#x27;bar&#x27;</span>], dtype=<span class="hljs-built_in">object</span>)&gt;,
 <span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>,), dtype=int64, numpy=array([<span class="hljs-number">0</span>, <span class="hljs-number">1</span>])&gt;}`}}),Hs=new B({props:{code:`ds = ds.with_format("tf", columns=["data"], output_all_columns=True)
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>, columns=[<span class="hljs-string">&quot;data&quot;</span>], output_all_columns=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;data&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>,), dtype=int64, numpy=array([<span class="hljs-number">0</span>, <span class="hljs-number">1</span>])&gt;,
 <span class="hljs-string">&#x27;text&#x27;</span>: [<span class="hljs-string">&#x27;foo&#x27;</span>, <span class="hljs-string">&#x27;bar&#x27;</span>]}`}}),ys=new $o({props:{$$slots:{default:[rp]},$$scope:{ctx:is}}}),Ls=new B({props:{code:`from datasets import Dataset, Features, Audio, Image
images = ["path/to/image.png"] * 10
features = Features({"image": Image()})
ds = Dataset.from_dict({"image": images}, features=features) 
ds = ds.with_format("tf")  
ds[0]
ds[:2]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio, Image
<span class="hljs-meta">&gt;&gt;&gt; </span>images = [<span class="hljs-string">&quot;path/to/image.png&quot;</span>] * <span class="hljs-number">10</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;image&quot;</span>: Image()})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;image&quot;</span>: images}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">4</span>), dtype=uint8, numpy=
 array([[[<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
         [<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
         ...,
         [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
         [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]], dtype=uint8)&gt;}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[:<span class="hljs-number">2</span>]
{<span class="hljs-string">&#x27;image&#x27;</span>: &lt;tf.Tensor: shape=(<span class="hljs-number">2</span>, <span class="hljs-number">512</span>, <span class="hljs-number">512</span>, <span class="hljs-number">4</span>), dtype=uint8, numpy=
 array([[[[<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
          [<span class="hljs-number">255</span>, <span class="hljs-number">215</span>, <span class="hljs-number">106</span>, <span class="hljs-number">255</span>],
          ...,
          [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>],
          [<span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>, <span class="hljs-number">255</span>]]]], dtype=uint8)&gt;}`}}),bs=new $o({props:{$$slots:{default:[pp]},$$scope:{ctx:is}}}),Us=new B({props:{code:`from datasets import Dataset, Features, Audio, Image
audio = ["path/to/audio.wav"] * 10
features = Features({"audio": Audio()})
ds = Dataset.from_dict({"audio": audio}, features=features) 
ds = ds.with_format("tf")  
ds[0]["audio"]["array"]
ds[0]["audio"]["sampling_rate"]`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset, Features, Audio, Image
<span class="hljs-meta">&gt;&gt;&gt; </span>audio = [<span class="hljs-string">&quot;path/to/audio.wav&quot;</span>] * <span class="hljs-number">10</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>features = Features({<span class="hljs-string">&quot;audio&quot;</span>: Audio()})
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict({<span class="hljs-string">&quot;audio&quot;</span>: audio}, features=features) 
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = ds.with_format(<span class="hljs-string">&quot;tf&quot;</span>)  
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;array&quot;</span>]
&lt;tf.Tensor: shape=(<span class="hljs-number">202311</span>,), dtype=float32, numpy=
array([ <span class="hljs-number">6.1035156e-05</span>,  <span class="hljs-number">1.5258789e-05</span>,  <span class="hljs-number">1.6784668e-04</span>, ...,
       -<span class="hljs-number">1.5258789e-05</span>, -<span class="hljs-number">1.5258789e-05</span>,  <span class="hljs-number">1.5258789e-05</span>], dtype=float32)&gt;
<span class="hljs-meta">&gt;&gt;&gt; </span>ds[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;audio&quot;</span>][<span class="hljs-string">&quot;sampling_rate&quot;</span>]
&lt;tf.Tensor: shape=(), dtype=int32, numpy=<span class="hljs-number">44100</span>&gt;`}}),Ys=new Ts({}),Ms=new Ts({}),zs=new B({props:{code:`from datasets import Dataset
data = {"inputs": [[1, 2],[3, 4]], "labels": [0, 1]}
ds = Dataset.from_dict(data)
tf_ds = ds.to_tf_dataset(`,highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> Dataset
<span class="hljs-meta">&gt;&gt;&gt; </span>data = {<span class="hljs-string">&quot;inputs&quot;</span>: [[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>],[<span class="hljs-number">3</span>, <span class="hljs-number">4</span>]], <span class="hljs-string">&quot;labels&quot;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">1</span>]}
<span class="hljs-meta">&gt;&gt;&gt; </span>ds = Dataset.from_dict(data)
<span class="hljs-meta">&gt;&gt;&gt; </span>tf_ds = ds.to_tf_dataset(
            columns=[<span class="hljs-string">&quot;inputs&quot;</span>],
            label_cols=[<span class="hljs-string">&quot;labels&quot;</span>],
            batch_size=<span class="hljs-number">2</span>,
            shuffle=<span class="hljs-literal">True</span>
            )`}}),Bs=new B({props:{code:"model.fit(tf_ds, epochs=2)",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(tf_ds, epochs=<span class="hljs-number">2</span>)'}}),Qs=new Ts({}),Xs=new Ts({}),{c(){u=l("meta"),A=h(),f=l("h1"),E=l("a"),C=l("span"),b(g.$$.fragment),k=h(),O=l("span"),I=e("Using Datasets with TensorFlow"),Y=h(),y=l("p"),N=e("This document is a quick introduction to using "),m=l("code"),ss=e("datasets"),as=e(` with TensorFlow, with a particular focus on how to get
`),M=l("code"),ke=e("tf.Tensor"),Te=e(" objects out of our datasets, and how to stream data from Hugging Face "),fa=l("code"),xe=e("Dataset"),qe=e(` objects to Keras methods
like `),ma=l("code"),Ce=e("model.fit()"),Ae=e("."),Et=h(),ts=l("h2"),hs=l("a"),ga=l("span"),b(xs.$$.fragment),Oe=h(),ya=l("span"),Pe=e("Dataset format"),Dt=h(),sa=l("p"),Fe=e("By default, datasets return regular Python objects: integers, floats, strings, lists, etc."),kt=h(),ds=l("p"),Ie=e("To get TensorFlow tensors instead, you can set the format of the dataset to "),ba=l("code"),Ne=e("tf"),Se=e(":"),Tt=h(),b(qs.$$.fragment),xt=h(),b(cs.$$.fragment),qt=h(),K=l("p"),He=e("This can be useful for converting your dataset to a dict of "),_a=l("code"),Le=e("Tensor"),Ue=e(` objects, or for writing a generator to load TF
samples from it. If you wish to convert the entire dataset to `),va=l("code"),Ye=e("Tensor"),Re=e(", simply query the full dataset:"),Ct=h(),b(Cs.$$.fragment),At=h(),es=l("h2"),us=l("a"),wa=l("span"),b(As.$$.fragment),We=h(),ja=l("span"),Me=e("N-dimensional arrays"),Ot=h(),fs=l("p"),ze=e(`If your dataset consists of N-dimensional arrays, you will see that by default they are considered as nested lists.
In particular, a TensorFlow formatted dataset outputs a `),$a=l("code"),Be=e("RaggedTensor"),Ke=e(" instead of a single tensor:"),Pt=h(),b(Os.$$.fragment),Ft=h(),ms=l("p"),Ge=e("To get a single tensor, you must explicitly use the "),Ea=l("code"),Je=e("Array"),Qe=e(" feature type and specify the shape of your tensors:"),It=h(),b(Ps.$$.fragment),Nt=h(),ns=l("h2"),gs=l("a"),Da=l("span"),b(Fs.$$.fragment),Ve=h(),ka=l("span"),Xe=e("Other feature types"),St=h(),Is=l("p"),aa=l("a"),Ze=e("ClassLabel"),sn=e(" data are properly converted to tensors:"),Ht=h(),b(Ns.$$.fragment),Lt=h(),ta=l("p"),an=e("Strings and binary objects are also supported:"),Ut=h(),b(Ss.$$.fragment),Yt=h(),ea=l("p"),tn=e("You can also explicitly format certain columns and leave the other columns unformatted:"),Rt=h(),b(Hs.$$.fragment),Wt=h(),na=l("p"),en=e("String and binary objects are unchanged, since PyTorch only supports numbers."),Mt=h(),G=l("p"),nn=e("The "),la=l("a"),ln=e("Image"),on=e(" and "),oa=l("a"),rn=e("Audio"),pn=e(" feature types are also supported."),zt=h(),b(ys.$$.fragment),Bt=h(),b(Ls.$$.fragment),Kt=h(),b(bs.$$.fragment),Gt=h(),b(Us.$$.fragment),Jt=h(),ls=l("h2"),_s=l("a"),Ta=l("span"),b(Ys.$$.fragment),hn=h(),xa=l("span"),dn=e("Data loading"),Qt=h(),P=l("p"),cn=e(`Although you can load individual samples and batches just by indexing into your dataset, this won\u2019t work if you want
to use Keras methods like `),qa=l("code"),un=e("fit()"),fn=e(" and "),Ca=l("code"),mn=e("predict()"),gn=e(`. You could write a generator function that shuffles and loads batches
from your dataset and `),Aa=l("code"),yn=e("fit()"),bn=e(` on that, but that sounds like a lot of unnecessary work. Instead, if you want to stream
data from your dataset on-the-fly, we recommend converting your dataset to a `),Oa=l("code"),_n=e("tf.data.Dataset"),vn=e(` using the
`),Pa=l("code"),wn=e("to_tf_dataset()"),jn=e(" method."),Vt=h(),T=l("p"),$n=e("The "),Fa=l("code"),En=e("tf.data.Dataset"),Dn=e(` class covers a wide range of use-cases - it is often created from Tensors in memory, or using a load function to read files on disc
or external storage. The dataset can be transformed arbitrarily with the `),Ia=l("code"),kn=e("map()"),Tn=e(" method, or methods like "),Na=l("code"),xn=e("batch()"),qn=e(`
and `),Sa=l("code"),Cn=e("shuffle()"),An=e(` can be used to create a dataset that\u2019s ready for training. These methods do not modify the stored data
in any way - instead, the methods build a data pipeline graph that will be executed when the dataset is iterated over,
usually during model training or inference. This is different from the `),Ha=l("code"),On=e("map()"),Pn=e(" method of Hugging Face "),La=l("code"),Fn=e("Dataset"),In=e(` objects,
which runs the map function immediately and saves the new or changed columns.`),Xt=h(),x=l("p"),Nn=e("Since the entire data preprocessing pipeline can be compiled in a "),Ua=l("code"),Sn=e("tf.data.Dataset"),Hn=e(`, this approach allows for massively
parallel, asynchronous data loading and training. However, the requirement for graph compilation can be a limitation,
particularly for Hugging Face tokenizers, which are usually not (yet!) compilable as part of a TF graph. As a result,
we usually advise pre-processing the dataset as a Hugging Face dataset, where arbitrary Python functions can be
used, and then converting to `),Ya=l("code"),Ln=e("tf.data.Dataset"),Un=e(" afterwards using "),Ra=l("code"),Yn=e("to_tf_dataset()"),Rn=e(` to get a batched dataset ready for
training. To see examples of this approach, please see the `),Rs=l("a"),Wn=e("examples"),Mn=e(" or "),Ws=l("a"),zn=e("notebooks"),Bn=e(" for "),Wa=l("code"),Kn=e("transformers"),Gn=e("."),Zt=h(),os=l("h3"),vs=l("a"),Ma=l("span"),b(Ms.$$.fragment),Jn=h(),ra=l("span"),Qn=e("Using "),za=l("code"),Vn=e("to_tf_dataset()"),se=h(),ws=l("p"),Xn=e("Using "),Ba=l("code"),Zn=e("to_tf_dataset()"),sl=e(" is straightforward. Once your dataset is preprocessed and ready, simply call it like so:"),ae=h(),b(zs.$$.fragment),te=h(),R=l("p"),al=e("The returned "),Ka=l("code"),tl=e("tf_ds"),el=e(" object here is now fully ready to train on, and can be passed directly to "),Ga=l("code"),nl=e("model.fit()"),ll=e(`. Note
that you set the batch size when creating the dataset, and so you don\u2019t need to specify it when calling `),Ja=l("code"),ol=e("fit()"),rl=e(":"),ee=h(),b(Bs.$$.fragment),ne=h(),q=l("p"),pl=e("For a full description of the arguments, please see the "),pa=l("a"),il=e("to_tf_dataset()"),hl=e(` documentation. In many cases,
you will also need to add a `),Qa=l("code"),dl=e("collate_fn"),cl=e(` to your call. This is a function that takes multiple elements of the dataset
and combines them into a single batch. When all elements have the same length, the built-in default collator will
suffice, but for more complex tasks a custom collator may be necessary. In particular, many tasks have samples
with varying sequence lengths which will require a `),Ks=l("a"),ul=e("data collator"),fl=e(` that can pad batches correctly. You can see examples
of this in the `),Va=l("code"),ml=e("transformers"),gl=e(" NLP "),Gs=l("a"),yl=e("examples"),bl=e(` and
`),Js=l("a"),_l=e("notebooks"),vl=e(", where variable sequence lengths are very common."),le=h(),J=l("p"),wl=e("If you find that loading with "),Xa=l("code"),jl=e("to_tf_dataset"),$l=e(" is slow, you can also use the "),Za=l("code"),El=e("num_workers"),Dl=e(` argument. This spins
up multiple subprocesses to load data in parallel. This feature is recent and still somewhat experimental - please file
an issue if you encounter any bugs while using it!`),oe=h(),rs=l("h3"),js=l("a"),st=l("span"),b(Qs.$$.fragment),kl=h(),at=l("span"),Tl=e("When to use to_tf_dataset"),re=h(),D=l("p"),xl=e(`The astute reader may have noticed at this point that we have offered two approaches to achieve the same goal - if you
want to pass your dataset to a TensorFlow model, you can either convert the dataset to a `),tt=l("code"),ql=e("Tensor"),Cl=e(" or "),et=l("code"),Al=e("dict"),Ol=e(" of "),nt=l("code"),Pl=e("Tensors"),Fl=e(`
using `),lt=l("code"),Il=e(".with_format('tf')"),Nl=e(", or you can convert the dataset to a "),ot=l("code"),Sl=e("tf.data.Dataset"),Hl=e(" with "),rt=l("code"),Ll=e("to_tf_dataset()"),Ul=e(`. Either of these
can be passed to `),pt=l("code"),Yl=e("model.fit()"),Rl=e(", so which should you choose?"),pe=h(),Q=l("p"),Wl=e("The key thing to recognize is that when you convert the whole dataset to "),it=l("code"),Ml=e("Tensor"),zl=e(`s, it is static and fully loaded into
RAM. This is simple and convenient, but if any of the following apply, you should probably use `),ht=l("code"),Bl=e("to_tf_dataset()"),Kl=e(`
instead:`),ie=h(),V=l("ul"),Vs=l("li"),Gl=e("Your dataset is too large to fit in RAM. "),dt=l("code"),Jl=e("to_tf_dataset()"),Ql=e(` streams only one batch at a time, so even very large
datasets can be handled with this method.`),Vl=h(),z=l("li"),Xl=e("You want to apply random transformations using "),ct=l("code"),Zl=e("dataset.with_transform()"),so=e(" or the "),ut=l("code"),ao=e("collate_fn"),to=e(`. This is
common in several modalities, such as image augmentations when training vision models, or random masking when training
masked language models. Using `),ft=l("code"),eo=e("to_tf_dataset()"),no=e(` will apply those transformations
at the moment when a batch is loaded, which means the same samples will get different augmentations each time
they are loaded. This is usually what you want.`),lo=h(),U=l("li"),oo=e(`Your data has a variable dimension, such as input texts in NLP that consist of varying
numbers of tokens. When you create a batch with samples with a variable dimension, the standard solution is to
pad the shorter samples to the length of the longest one. When you stream samples from a dataset with `),mt=l("code"),ro=e("to_tf_dataset"),po=e(`,
you can apply this padding to each batch via your `),gt=l("code"),io=e("collate_fn"),ho=e(`. However, if you want to convert
such a dataset to dense `),yt=l("code"),co=e("Tensor"),uo=e("s, then you will have to pad samples to the length of the longest sample in "),bt=l("em"),fo=e(`the
entire dataset!`),mo=e(" This can result in huge amounts of padding, which wastes memory and reduces your model\u2019s speed."),he=h(),ps=l("h3"),$s=l("a"),_t=l("span"),b(Xs.$$.fragment),go=h(),vt=l("span"),yo=e("Caveats and limitations"),de=h(),Es=l("p"),bo=e("Right now, "),wt=l("code"),_o=e("to_tf_dataset()"),vo=e(" always returns a batched dataset - we will add support for unbatched datasets soon!"),this.h()},l(s){const p=np('[data-svelte="svelte-1phssyn"]',document.head);u=o(p,"META",{name:!0,content:!0}),p.forEach(t),A=d(s),f=o(s,"H1",{class:!0});var Zs=r(f);E=o(Zs,"A",{id:!0,class:!0,href:!0});var jt=r(E);C=o(jt,"SPAN",{});var $t=r(C);_(g.$$.fragment,$t),$t.forEach(t),jt.forEach(t),k=d(Zs),O=o(Zs,"SPAN",{});var Eo=r(O);I=n(Eo,"Using Datasets with TensorFlow"),Eo.forEach(t),Zs.forEach(t),Y=d(s),y=o(s,"P",{});var X=r(y);N=n(X,"This document is a quick introduction to using "),m=o(X,"CODE",{});var Do=r(m);ss=n(Do,"datasets"),Do.forEach(t),as=n(X,` with TensorFlow, with a particular focus on how to get
`),M=o(X,"CODE",{});var ko=r(M);ke=n(ko,"tf.Tensor"),ko.forEach(t),Te=n(X," objects out of our datasets, and how to stream data from Hugging Face "),fa=o(X,"CODE",{});var To=r(fa);xe=n(To,"Dataset"),To.forEach(t),qe=n(X,` objects to Keras methods
like `),ma=o(X,"CODE",{});var xo=r(ma);Ce=n(xo,"model.fit()"),xo.forEach(t),Ae=n(X,"."),X.forEach(t),Et=d(s),ts=o(s,"H2",{class:!0});var ue=r(ts);hs=o(ue,"A",{id:!0,class:!0,href:!0});var qo=r(hs);ga=o(qo,"SPAN",{});var Co=r(ga);_(xs.$$.fragment,Co),Co.forEach(t),qo.forEach(t),Oe=d(ue),ya=o(ue,"SPAN",{});var Ao=r(ya);Pe=n(Ao,"Dataset format"),Ao.forEach(t),ue.forEach(t),Dt=d(s),sa=o(s,"P",{});var Oo=r(sa);Fe=n(Oo,"By default, datasets return regular Python objects: integers, floats, strings, lists, etc."),Oo.forEach(t),kt=d(s),ds=o(s,"P",{});var fe=r(ds);Ie=n(fe,"To get TensorFlow tensors instead, you can set the format of the dataset to "),ba=o(fe,"CODE",{});var Po=r(ba);Ne=n(Po,"tf"),Po.forEach(t),Se=n(fe,":"),fe.forEach(t),Tt=d(s),_(qs.$$.fragment,s),xt=d(s),_(cs.$$.fragment,s),qt=d(s),K=o(s,"P",{});var ia=r(K);He=n(ia,"This can be useful for converting your dataset to a dict of "),_a=o(ia,"CODE",{});var Fo=r(_a);Le=n(Fo,"Tensor"),Fo.forEach(t),Ue=n(ia,` objects, or for writing a generator to load TF
samples from it. If you wish to convert the entire dataset to `),va=o(ia,"CODE",{});var Io=r(va);Ye=n(Io,"Tensor"),Io.forEach(t),Re=n(ia,", simply query the full dataset:"),ia.forEach(t),Ct=d(s),_(Cs.$$.fragment,s),At=d(s),es=o(s,"H2",{class:!0});var me=r(es);us=o(me,"A",{id:!0,class:!0,href:!0});var No=r(us);wa=o(No,"SPAN",{});var So=r(wa);_(As.$$.fragment,So),So.forEach(t),No.forEach(t),We=d(me),ja=o(me,"SPAN",{});var Ho=r(ja);Me=n(Ho,"N-dimensional arrays"),Ho.forEach(t),me.forEach(t),Ot=d(s),fs=o(s,"P",{});var ge=r(fs);ze=n(ge,`If your dataset consists of N-dimensional arrays, you will see that by default they are considered as nested lists.
In particular, a TensorFlow formatted dataset outputs a `),$a=o(ge,"CODE",{});var Lo=r($a);Be=n(Lo,"RaggedTensor"),Lo.forEach(t),Ke=n(ge," instead of a single tensor:"),ge.forEach(t),Pt=d(s),_(Os.$$.fragment,s),Ft=d(s),ms=o(s,"P",{});var ye=r(ms);Ge=n(ye,"To get a single tensor, you must explicitly use the "),Ea=o(ye,"CODE",{});var Uo=r(Ea);Je=n(Uo,"Array"),Uo.forEach(t),Qe=n(ye," feature type and specify the shape of your tensors:"),ye.forEach(t),It=d(s),_(Ps.$$.fragment,s),Nt=d(s),ns=o(s,"H2",{class:!0});var be=r(ns);gs=o(be,"A",{id:!0,class:!0,href:!0});var Yo=r(gs);Da=o(Yo,"SPAN",{});var Ro=r(Da);_(Fs.$$.fragment,Ro),Ro.forEach(t),Yo.forEach(t),Ve=d(be),ka=o(be,"SPAN",{});var Wo=r(ka);Xe=n(Wo,"Other feature types"),Wo.forEach(t),be.forEach(t),St=d(s),Is=o(s,"P",{});var wo=r(Is);aa=o(wo,"A",{href:!0});var Mo=r(aa);Ze=n(Mo,"ClassLabel"),Mo.forEach(t),sn=n(wo," data are properly converted to tensors:"),wo.forEach(t),Ht=d(s),_(Ns.$$.fragment,s),Lt=d(s),ta=o(s,"P",{});var zo=r(ta);an=n(zo,"Strings and binary objects are also supported:"),zo.forEach(t),Ut=d(s),_(Ss.$$.fragment,s),Yt=d(s),ea=o(s,"P",{});var Bo=r(ea);tn=n(Bo,"You can also explicitly format certain columns and leave the other columns unformatted:"),Bo.forEach(t),Rt=d(s),_(Hs.$$.fragment,s),Wt=d(s),na=o(s,"P",{});var Ko=r(na);en=n(Ko,"String and binary objects are unchanged, since PyTorch only supports numbers."),Ko.forEach(t),Mt=d(s),G=o(s,"P",{});var ha=r(G);nn=n(ha,"The "),la=o(ha,"A",{href:!0});var Go=r(la);ln=n(Go,"Image"),Go.forEach(t),on=n(ha," and "),oa=o(ha,"A",{href:!0});var Jo=r(oa);rn=n(Jo,"Audio"),Jo.forEach(t),pn=n(ha," feature types are also supported."),ha.forEach(t),zt=d(s),_(ys.$$.fragment,s),Bt=d(s),_(Ls.$$.fragment,s),Kt=d(s),_(bs.$$.fragment,s),Gt=d(s),_(Us.$$.fragment,s),Jt=d(s),ls=o(s,"H2",{class:!0});var _e=r(ls);_s=o(_e,"A",{id:!0,class:!0,href:!0});var Qo=r(_s);Ta=o(Qo,"SPAN",{});var Vo=r(Ta);_(Ys.$$.fragment,Vo),Vo.forEach(t),Qo.forEach(t),hn=d(_e),xa=o(_e,"SPAN",{});var Xo=r(xa);dn=n(Xo,"Data loading"),Xo.forEach(t),_e.forEach(t),Qt=d(s),P=o(s,"P",{});var W=r(P);cn=n(W,`Although you can load individual samples and batches just by indexing into your dataset, this won\u2019t work if you want
to use Keras methods like `),qa=o(W,"CODE",{});var Zo=r(qa);un=n(Zo,"fit()"),Zo.forEach(t),fn=n(W," and "),Ca=o(W,"CODE",{});var sr=r(Ca);mn=n(sr,"predict()"),sr.forEach(t),gn=n(W,`. You could write a generator function that shuffles and loads batches
from your dataset and `),Aa=o(W,"CODE",{});var ar=r(Aa);yn=n(ar,"fit()"),ar.forEach(t),bn=n(W,` on that, but that sounds like a lot of unnecessary work. Instead, if you want to stream
data from your dataset on-the-fly, we recommend converting your dataset to a `),Oa=o(W,"CODE",{});var tr=r(Oa);_n=n(tr,"tf.data.Dataset"),tr.forEach(t),vn=n(W,` using the
`),Pa=o(W,"CODE",{});var er=r(Pa);wn=n(er,"to_tf_dataset()"),er.forEach(t),jn=n(W," method."),W.forEach(t),Vt=d(s),T=o(s,"P",{});var S=r(T);$n=n(S,"The "),Fa=o(S,"CODE",{});var nr=r(Fa);En=n(nr,"tf.data.Dataset"),nr.forEach(t),Dn=n(S,` class covers a wide range of use-cases - it is often created from Tensors in memory, or using a load function to read files on disc
or external storage. The dataset can be transformed arbitrarily with the `),Ia=o(S,"CODE",{});var lr=r(Ia);kn=n(lr,"map()"),lr.forEach(t),Tn=n(S," method, or methods like "),Na=o(S,"CODE",{});var or=r(Na);xn=n(or,"batch()"),or.forEach(t),qn=n(S,`
and `),Sa=o(S,"CODE",{});var rr=r(Sa);Cn=n(rr,"shuffle()"),rr.forEach(t),An=n(S,` can be used to create a dataset that\u2019s ready for training. These methods do not modify the stored data
in any way - instead, the methods build a data pipeline graph that will be executed when the dataset is iterated over,
usually during model training or inference. This is different from the `),Ha=o(S,"CODE",{});var pr=r(Ha);On=n(pr,"map()"),pr.forEach(t),Pn=n(S," method of Hugging Face "),La=o(S,"CODE",{});var ir=r(La);Fn=n(ir,"Dataset"),ir.forEach(t),In=n(S,` objects,
which runs the map function immediately and saves the new or changed columns.`),S.forEach(t),Xt=d(s),x=o(s,"P",{});var H=r(x);Nn=n(H,"Since the entire data preprocessing pipeline can be compiled in a "),Ua=o(H,"CODE",{});var hr=r(Ua);Sn=n(hr,"tf.data.Dataset"),hr.forEach(t),Hn=n(H,`, this approach allows for massively
parallel, asynchronous data loading and training. However, the requirement for graph compilation can be a limitation,
particularly for Hugging Face tokenizers, which are usually not (yet!) compilable as part of a TF graph. As a result,
we usually advise pre-processing the dataset as a Hugging Face dataset, where arbitrary Python functions can be
used, and then converting to `),Ya=o(H,"CODE",{});var dr=r(Ya);Ln=n(dr,"tf.data.Dataset"),dr.forEach(t),Un=n(H," afterwards using "),Ra=o(H,"CODE",{});var cr=r(Ra);Yn=n(cr,"to_tf_dataset()"),cr.forEach(t),Rn=n(H,` to get a batched dataset ready for
training. To see examples of this approach, please see the `),Rs=o(H,"A",{href:!0,rel:!0});var ur=r(Rs);Wn=n(ur,"examples"),ur.forEach(t),Mn=n(H," or "),Ws=o(H,"A",{href:!0,rel:!0});var fr=r(Ws);zn=n(fr,"notebooks"),fr.forEach(t),Bn=n(H," for "),Wa=o(H,"CODE",{});var mr=r(Wa);Kn=n(mr,"transformers"),mr.forEach(t),Gn=n(H,"."),H.forEach(t),Zt=d(s),os=o(s,"H3",{class:!0});var ve=r(os);vs=o(ve,"A",{id:!0,class:!0,href:!0});var gr=r(vs);Ma=o(gr,"SPAN",{});var yr=r(Ma);_(Ms.$$.fragment,yr),yr.forEach(t),gr.forEach(t),Jn=d(ve),ra=o(ve,"SPAN",{});var jo=r(ra);Qn=n(jo,"Using "),za=o(jo,"CODE",{});var br=r(za);Vn=n(br,"to_tf_dataset()"),br.forEach(t),jo.forEach(t),ve.forEach(t),se=d(s),ws=o(s,"P",{});var we=r(ws);Xn=n(we,"Using "),Ba=o(we,"CODE",{});var _r=r(Ba);Zn=n(_r,"to_tf_dataset()"),_r.forEach(t),sl=n(we," is straightforward. Once your dataset is preprocessed and ready, simply call it like so:"),we.forEach(t),ae=d(s),_(zs.$$.fragment,s),te=d(s),R=o(s,"P",{});var Ds=r(R);al=n(Ds,"The returned "),Ka=o(Ds,"CODE",{});var vr=r(Ka);tl=n(vr,"tf_ds"),vr.forEach(t),el=n(Ds," object here is now fully ready to train on, and can be passed directly to "),Ga=o(Ds,"CODE",{});var wr=r(Ga);nl=n(wr,"model.fit()"),wr.forEach(t),ll=n(Ds,`. Note
that you set the batch size when creating the dataset, and so you don\u2019t need to specify it when calling `),Ja=o(Ds,"CODE",{});var jr=r(Ja);ol=n(jr,"fit()"),jr.forEach(t),rl=n(Ds,":"),Ds.forEach(t),ee=d(s),_(Bs.$$.fragment,s),ne=d(s),q=o(s,"P",{});var L=r(q);pl=n(L,"For a full description of the arguments, please see the "),pa=o(L,"A",{href:!0});var $r=r(pa);il=n($r,"to_tf_dataset()"),$r.forEach(t),hl=n(L,` documentation. In many cases,
you will also need to add a `),Qa=o(L,"CODE",{});var Er=r(Qa);dl=n(Er,"collate_fn"),Er.forEach(t),cl=n(L,` to your call. This is a function that takes multiple elements of the dataset
and combines them into a single batch. When all elements have the same length, the built-in default collator will
suffice, but for more complex tasks a custom collator may be necessary. In particular, many tasks have samples
with varying sequence lengths which will require a `),Ks=o(L,"A",{href:!0,rel:!0});var Dr=r(Ks);ul=n(Dr,"data collator"),Dr.forEach(t),fl=n(L,` that can pad batches correctly. You can see examples
of this in the `),Va=o(L,"CODE",{});var kr=r(Va);ml=n(kr,"transformers"),kr.forEach(t),gl=n(L," NLP "),Gs=o(L,"A",{href:!0,rel:!0});var Tr=r(Gs);yl=n(Tr,"examples"),Tr.forEach(t),bl=n(L,` and
`),Js=o(L,"A",{href:!0,rel:!0});var xr=r(Js);_l=n(xr,"notebooks"),xr.forEach(t),vl=n(L,", where variable sequence lengths are very common."),L.forEach(t),le=d(s),J=o(s,"P",{});var da=r(J);wl=n(da,"If you find that loading with "),Xa=o(da,"CODE",{});var qr=r(Xa);jl=n(qr,"to_tf_dataset"),qr.forEach(t),$l=n(da," is slow, you can also use the "),Za=o(da,"CODE",{});var Cr=r(Za);El=n(Cr,"num_workers"),Cr.forEach(t),Dl=n(da,` argument. This spins
up multiple subprocesses to load data in parallel. This feature is recent and still somewhat experimental - please file
an issue if you encounter any bugs while using it!`),da.forEach(t),oe=d(s),rs=o(s,"H3",{class:!0});var je=r(rs);js=o(je,"A",{id:!0,class:!0,href:!0});var Ar=r(js);st=o(Ar,"SPAN",{});var Or=r(st);_(Qs.$$.fragment,Or),Or.forEach(t),Ar.forEach(t),kl=d(je),at=o(je,"SPAN",{});var Pr=r(at);Tl=n(Pr,"When to use to_tf_dataset"),Pr.forEach(t),je.forEach(t),re=d(s),D=o(s,"P",{});var F=r(D);xl=n(F,`The astute reader may have noticed at this point that we have offered two approaches to achieve the same goal - if you
want to pass your dataset to a TensorFlow model, you can either convert the dataset to a `),tt=o(F,"CODE",{});var Fr=r(tt);ql=n(Fr,"Tensor"),Fr.forEach(t),Cl=n(F," or "),et=o(F,"CODE",{});var Ir=r(et);Al=n(Ir,"dict"),Ir.forEach(t),Ol=n(F," of "),nt=o(F,"CODE",{});var Nr=r(nt);Pl=n(Nr,"Tensors"),Nr.forEach(t),Fl=n(F,`
using `),lt=o(F,"CODE",{});var Sr=r(lt);Il=n(Sr,".with_format('tf')"),Sr.forEach(t),Nl=n(F,", or you can convert the dataset to a "),ot=o(F,"CODE",{});var Hr=r(ot);Sl=n(Hr,"tf.data.Dataset"),Hr.forEach(t),Hl=n(F," with "),rt=o(F,"CODE",{});var Lr=r(rt);Ll=n(Lr,"to_tf_dataset()"),Lr.forEach(t),Ul=n(F,`. Either of these
can be passed to `),pt=o(F,"CODE",{});var Ur=r(pt);Yl=n(Ur,"model.fit()"),Ur.forEach(t),Rl=n(F,", so which should you choose?"),F.forEach(t),pe=d(s),Q=o(s,"P",{});var ca=r(Q);Wl=n(ca,"The key thing to recognize is that when you convert the whole dataset to "),it=o(ca,"CODE",{});var Yr=r(it);Ml=n(Yr,"Tensor"),Yr.forEach(t),zl=n(ca,`s, it is static and fully loaded into
RAM. This is simple and convenient, but if any of the following apply, you should probably use `),ht=o(ca,"CODE",{});var Rr=r(ht);Bl=n(Rr,"to_tf_dataset()"),Rr.forEach(t),Kl=n(ca,`
instead:`),ca.forEach(t),ie=d(s),V=o(s,"UL",{});var ua=r(V);Vs=o(ua,"LI",{});var $e=r(Vs);Gl=n($e,"Your dataset is too large to fit in RAM. "),dt=o($e,"CODE",{});var Wr=r(dt);Jl=n(Wr,"to_tf_dataset()"),Wr.forEach(t),Ql=n($e,` streams only one batch at a time, so even very large
datasets can be handled with this method.`),$e.forEach(t),Vl=d(ua),z=o(ua,"LI",{});var ks=r(z);Xl=n(ks,"You want to apply random transformations using "),ct=o(ks,"CODE",{});var Mr=r(ct);Zl=n(Mr,"dataset.with_transform()"),Mr.forEach(t),so=n(ks," or the "),ut=o(ks,"CODE",{});var zr=r(ut);ao=n(zr,"collate_fn"),zr.forEach(t),to=n(ks,`. This is
common in several modalities, such as image augmentations when training vision models, or random masking when training
masked language models. Using `),ft=o(ks,"CODE",{});var Br=r(ft);eo=n(Br,"to_tf_dataset()"),Br.forEach(t),no=n(ks,` will apply those transformations
at the moment when a batch is loaded, which means the same samples will get different augmentations each time
they are loaded. This is usually what you want.`),ks.forEach(t),lo=d(ua),U=o(ua,"LI",{});var Z=r(U);oo=n(Z,`Your data has a variable dimension, such as input texts in NLP that consist of varying
numbers of tokens. When you create a batch with samples with a variable dimension, the standard solution is to
pad the shorter samples to the length of the longest one. When you stream samples from a dataset with `),mt=o(Z,"CODE",{});var Kr=r(mt);ro=n(Kr,"to_tf_dataset"),Kr.forEach(t),po=n(Z,`,
you can apply this padding to each batch via your `),gt=o(Z,"CODE",{});var Gr=r(gt);io=n(Gr,"collate_fn"),Gr.forEach(t),ho=n(Z,`. However, if you want to convert
such a dataset to dense `),yt=o(Z,"CODE",{});var Jr=r(yt);co=n(Jr,"Tensor"),Jr.forEach(t),uo=n(Z,"s, then you will have to pad samples to the length of the longest sample in "),bt=o(Z,"EM",{});var Qr=r(bt);fo=n(Qr,`the
entire dataset!`),Qr.forEach(t),mo=n(Z," This can result in huge amounts of padding, which wastes memory and reduces your model\u2019s speed."),Z.forEach(t),ua.forEach(t),he=d(s),ps=o(s,"H3",{class:!0});var Ee=r(ps);$s=o(Ee,"A",{id:!0,class:!0,href:!0});var Vr=r($s);_t=o(Vr,"SPAN",{});var Xr=r(_t);_(Xs.$$.fragment,Xr),Xr.forEach(t),Vr.forEach(t),go=d(Ee),vt=o(Ee,"SPAN",{});var Zr=r(vt);yo=n(Zr,"Caveats and limitations"),Zr.forEach(t),Ee.forEach(t),de=d(s),Es=o(s,"P",{});var De=r(Es);bo=n(De,"Right now, "),wt=o(De,"CODE",{});var sp=r(wt);_o=n(sp,"to_tf_dataset()"),sp.forEach(t),vo=n(De," always returns a batched dataset - we will add support for unbatched datasets soon!"),De.forEach(t),this.h()},h(){c(u,"name","hf:doc:metadata"),c(u,"content",JSON.stringify(hp)),c(E,"id","using-datasets-with-tensorflow"),c(E,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(E,"href","#using-datasets-with-tensorflow"),c(f,"class","relative group"),c(hs,"id","dataset-format"),c(hs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(hs,"href","#dataset-format"),c(ts,"class","relative group"),c(us,"id","ndimensional-arrays"),c(us,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(us,"href","#ndimensional-arrays"),c(es,"class","relative group"),c(gs,"id","other-feature-types"),c(gs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(gs,"href","#other-feature-types"),c(ns,"class","relative group"),c(aa,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.ClassLabel"),c(la,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Image"),c(oa,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Audio"),c(_s,"id","data-loading"),c(_s,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(_s,"href","#data-loading"),c(ls,"class","relative group"),c(Rs,"href","https://github.com/huggingface/transformers/tree/main/examples"),c(Rs,"rel","nofollow"),c(Ws,"href","https://huggingface.co/docs/transformers/notebooks"),c(Ws,"rel","nofollow"),c(vs,"id","using-totfdataset"),c(vs,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(vs,"href","#using-totfdataset"),c(os,"class","relative group"),c(pa,"href","/docs/datasets/v2.11.0/en/package_reference/main_classes#datasets.Dataset.to_tf_dataset"),c(Ks,"href","https://huggingface.co/docs/transformers/main/en/main_classes/data_collator"),c(Ks,"rel","nofollow"),c(Gs,"href","https://github.com/huggingface/transformers/tree/main/examples"),c(Gs,"rel","nofollow"),c(Js,"href","https://huggingface.co/docs/transformers/notebooks"),c(Js,"rel","nofollow"),c(js,"id","when-to-use-totfdataset"),c(js,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c(js,"href","#when-to-use-totfdataset"),c(rs,"class","relative group"),c($s,"id","caveats-and-limitations"),c($s,"class","header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full"),c($s,"href","#caveats-and-limitations"),c(ps,"class","relative group")},m(s,p){a(document.head,u),i(s,A,p),i(s,f,p),a(f,E),a(E,C),v(g,C,null),a(f,k),a(f,O),a(O,I),i(s,Y,p),i(s,y,p),a(y,N),a(y,m),a(m,ss),a(y,as),a(y,M),a(M,ke),a(y,Te),a(y,fa),a(fa,xe),a(y,qe),a(y,ma),a(ma,Ce),a(y,Ae),i(s,Et,p),i(s,ts,p),a(ts,hs),a(hs,ga),v(xs,ga,null),a(ts,Oe),a(ts,ya),a(ya,Pe),i(s,Dt,p),i(s,sa,p),a(sa,Fe),i(s,kt,p),i(s,ds,p),a(ds,Ie),a(ds,ba),a(ba,Ne),a(ds,Se),i(s,Tt,p),v(qs,s,p),i(s,xt,p),v(cs,s,p),i(s,qt,p),i(s,K,p),a(K,He),a(K,_a),a(_a,Le),a(K,Ue),a(K,va),a(va,Ye),a(K,Re),i(s,Ct,p),v(Cs,s,p),i(s,At,p),i(s,es,p),a(es,us),a(us,wa),v(As,wa,null),a(es,We),a(es,ja),a(ja,Me),i(s,Ot,p),i(s,fs,p),a(fs,ze),a(fs,$a),a($a,Be),a(fs,Ke),i(s,Pt,p),v(Os,s,p),i(s,Ft,p),i(s,ms,p),a(ms,Ge),a(ms,Ea),a(Ea,Je),a(ms,Qe),i(s,It,p),v(Ps,s,p),i(s,Nt,p),i(s,ns,p),a(ns,gs),a(gs,Da),v(Fs,Da,null),a(ns,Ve),a(ns,ka),a(ka,Xe),i(s,St,p),i(s,Is,p),a(Is,aa),a(aa,Ze),a(Is,sn),i(s,Ht,p),v(Ns,s,p),i(s,Lt,p),i(s,ta,p),a(ta,an),i(s,Ut,p),v(Ss,s,p),i(s,Yt,p),i(s,ea,p),a(ea,tn),i(s,Rt,p),v(Hs,s,p),i(s,Wt,p),i(s,na,p),a(na,en),i(s,Mt,p),i(s,G,p),a(G,nn),a(G,la),a(la,ln),a(G,on),a(G,oa),a(oa,rn),a(G,pn),i(s,zt,p),v(ys,s,p),i(s,Bt,p),v(Ls,s,p),i(s,Kt,p),v(bs,s,p),i(s,Gt,p),v(Us,s,p),i(s,Jt,p),i(s,ls,p),a(ls,_s),a(_s,Ta),v(Ys,Ta,null),a(ls,hn),a(ls,xa),a(xa,dn),i(s,Qt,p),i(s,P,p),a(P,cn),a(P,qa),a(qa,un),a(P,fn),a(P,Ca),a(Ca,mn),a(P,gn),a(P,Aa),a(Aa,yn),a(P,bn),a(P,Oa),a(Oa,_n),a(P,vn),a(P,Pa),a(Pa,wn),a(P,jn),i(s,Vt,p),i(s,T,p),a(T,$n),a(T,Fa),a(Fa,En),a(T,Dn),a(T,Ia),a(Ia,kn),a(T,Tn),a(T,Na),a(Na,xn),a(T,qn),a(T,Sa),a(Sa,Cn),a(T,An),a(T,Ha),a(Ha,On),a(T,Pn),a(T,La),a(La,Fn),a(T,In),i(s,Xt,p),i(s,x,p),a(x,Nn),a(x,Ua),a(Ua,Sn),a(x,Hn),a(x,Ya),a(Ya,Ln),a(x,Un),a(x,Ra),a(Ra,Yn),a(x,Rn),a(x,Rs),a(Rs,Wn),a(x,Mn),a(x,Ws),a(Ws,zn),a(x,Bn),a(x,Wa),a(Wa,Kn),a(x,Gn),i(s,Zt,p),i(s,os,p),a(os,vs),a(vs,Ma),v(Ms,Ma,null),a(os,Jn),a(os,ra),a(ra,Qn),a(ra,za),a(za,Vn),i(s,se,p),i(s,ws,p),a(ws,Xn),a(ws,Ba),a(Ba,Zn),a(ws,sl),i(s,ae,p),v(zs,s,p),i(s,te,p),i(s,R,p),a(R,al),a(R,Ka),a(Ka,tl),a(R,el),a(R,Ga),a(Ga,nl),a(R,ll),a(R,Ja),a(Ja,ol),a(R,rl),i(s,ee,p),v(Bs,s,p),i(s,ne,p),i(s,q,p),a(q,pl),a(q,pa),a(pa,il),a(q,hl),a(q,Qa),a(Qa,dl),a(q,cl),a(q,Ks),a(Ks,ul),a(q,fl),a(q,Va),a(Va,ml),a(q,gl),a(q,Gs),a(Gs,yl),a(q,bl),a(q,Js),a(Js,_l),a(q,vl),i(s,le,p),i(s,J,p),a(J,wl),a(J,Xa),a(Xa,jl),a(J,$l),a(J,Za),a(Za,El),a(J,Dl),i(s,oe,p),i(s,rs,p),a(rs,js),a(js,st),v(Qs,st,null),a(rs,kl),a(rs,at),a(at,Tl),i(s,re,p),i(s,D,p),a(D,xl),a(D,tt),a(tt,ql),a(D,Cl),a(D,et),a(et,Al),a(D,Ol),a(D,nt),a(nt,Pl),a(D,Fl),a(D,lt),a(lt,Il),a(D,Nl),a(D,ot),a(ot,Sl),a(D,Hl),a(D,rt),a(rt,Ll),a(D,Ul),a(D,pt),a(pt,Yl),a(D,Rl),i(s,pe,p),i(s,Q,p),a(Q,Wl),a(Q,it),a(it,Ml),a(Q,zl),a(Q,ht),a(ht,Bl),a(Q,Kl),i(s,ie,p),i(s,V,p),a(V,Vs),a(Vs,Gl),a(Vs,dt),a(dt,Jl),a(Vs,Ql),a(V,Vl),a(V,z),a(z,Xl),a(z,ct),a(ct,Zl),a(z,so),a(z,ut),a(ut,ao),a(z,to),a(z,ft),a(ft,eo),a(z,no),a(V,lo),a(V,U),a(U,oo),a(U,mt),a(mt,ro),a(U,po),a(U,gt),a(gt,io),a(U,ho),a(U,yt),a(yt,co),a(U,uo),a(U,bt),a(bt,fo),a(U,mo),i(s,he,p),i(s,ps,p),a(ps,$s),a($s,_t),v(Xs,_t,null),a(ps,go),a(ps,vt),a(vt,yo),i(s,de,p),i(s,Es,p),a(Es,bo),a(Es,wt),a(wt,_o),a(Es,vo),ce=!0},p(s,[p]){const Zs={};p&2&&(Zs.$$scope={dirty:p,ctx:s}),cs.$set(Zs);const jt={};p&2&&(jt.$$scope={dirty:p,ctx:s}),ys.$set(jt);const $t={};p&2&&($t.$$scope={dirty:p,ctx:s}),bs.$set($t)},i(s){ce||(w(g.$$.fragment,s),w(xs.$$.fragment,s),w(qs.$$.fragment,s),w(cs.$$.fragment,s),w(Cs.$$.fragment,s),w(As.$$.fragment,s),w(Os.$$.fragment,s),w(Ps.$$.fragment,s),w(Fs.$$.fragment,s),w(Ns.$$.fragment,s),w(Ss.$$.fragment,s),w(Hs.$$.fragment,s),w(ys.$$.fragment,s),w(Ls.$$.fragment,s),w(bs.$$.fragment,s),w(Us.$$.fragment,s),w(Ys.$$.fragment,s),w(Ms.$$.fragment,s),w(zs.$$.fragment,s),w(Bs.$$.fragment,s),w(Qs.$$.fragment,s),w(Xs.$$.fragment,s),ce=!0)},o(s){j(g.$$.fragment,s),j(xs.$$.fragment,s),j(qs.$$.fragment,s),j(cs.$$.fragment,s),j(Cs.$$.fragment,s),j(As.$$.fragment,s),j(Os.$$.fragment,s),j(Ps.$$.fragment,s),j(Fs.$$.fragment,s),j(Ns.$$.fragment,s),j(Ss.$$.fragment,s),j(Hs.$$.fragment,s),j(ys.$$.fragment,s),j(Ls.$$.fragment,s),j(bs.$$.fragment,s),j(Us.$$.fragment,s),j(Ys.$$.fragment,s),j(Ms.$$.fragment,s),j(zs.$$.fragment,s),j(Bs.$$.fragment,s),j(Qs.$$.fragment,s),j(Xs.$$.fragment,s),ce=!1},d(s){t(u),s&&t(A),s&&t(f),$(g),s&&t(Y),s&&t(y),s&&t(Et),s&&t(ts),$(xs),s&&t(Dt),s&&t(sa),s&&t(kt),s&&t(ds),s&&t(Tt),$(qs,s),s&&t(xt),$(cs,s),s&&t(qt),s&&t(K),s&&t(Ct),$(Cs,s),s&&t(At),s&&t(es),$(As),s&&t(Ot),s&&t(fs),s&&t(Pt),$(Os,s),s&&t(Ft),s&&t(ms),s&&t(It),$(Ps,s),s&&t(Nt),s&&t(ns),$(Fs),s&&t(St),s&&t(Is),s&&t(Ht),$(Ns,s),s&&t(Lt),s&&t(ta),s&&t(Ut),$(Ss,s),s&&t(Yt),s&&t(ea),s&&t(Rt),$(Hs,s),s&&t(Wt),s&&t(na),s&&t(Mt),s&&t(G),s&&t(zt),$(ys,s),s&&t(Bt),$(Ls,s),s&&t(Kt),$(bs,s),s&&t(Gt),$(Us,s),s&&t(Jt),s&&t(ls),$(Ys),s&&t(Qt),s&&t(P),s&&t(Vt),s&&t(T),s&&t(Xt),s&&t(x),s&&t(Zt),s&&t(os),$(Ms),s&&t(se),s&&t(ws),s&&t(ae),$(zs,s),s&&t(te),s&&t(R),s&&t(ee),$(Bs,s),s&&t(ne),s&&t(q),s&&t(le),s&&t(J),s&&t(oe),s&&t(rs),$(Qs),s&&t(re),s&&t(D),s&&t(pe),s&&t(Q),s&&t(ie),s&&t(V),s&&t(he),s&&t(ps),$(Xs),s&&t(de),s&&t(Es)}}}const hp={local:"using-datasets-with-tensorflow",sections:[{local:"dataset-format",title:"Dataset format"},{local:"ndimensional-arrays",title:"N-dimensional arrays"},{local:"other-feature-types",title:"Other feature types"},{local:"data-loading",sections:[{local:"using-totfdataset",title:"Using `to_tf_dataset()`"},{local:"when-to-use-totfdataset",title:"When to use to_tf_dataset"},{local:"caveats-and-limitations",title:"Caveats and limitations"}],title:"Data loading"}],title:"Using Datasets with TensorFlow"};function dp(is){return lp(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class gp extends ap{constructor(u){super();tp(this,u,dp,ip,ep,{})}}export{gp as default,hp as metadata};
